ZslProcessor3.cpp revision 491e341211b4772c75f719158f6b397e1c40497d
1/*
2 * Copyright (C) 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "Camera2-ZslProcessor3"
18#define ATRACE_TAG ATRACE_TAG_CAMERA
19//#define LOG_NDEBUG 0
20//#define LOG_NNDEBUG 0
21
22#ifdef LOG_NNDEBUG
23#define ALOGVV(...) ALOGV(__VA_ARGS__)
24#else
25#define ALOGVV(...) ((void)0)
26#endif
27
28#include <utils/Log.h>
29#include <utils/Trace.h>
30#include <gui/Surface.h>
31
32#include "common/CameraDeviceBase.h"
33#include "api1/Camera2Client.h"
34#include "api1/client2/CaptureSequencer.h"
35#include "api1/client2/ZslProcessor3.h"
36#include "device3/Camera3Device.h"
37
38namespace android {
39namespace camera2 {
40
41ZslProcessor3::ZslProcessor3(
42    sp<Camera2Client> client,
43    wp<CaptureSequencer> sequencer):
44        Thread(false),
45        mState(RUNNING),
46        mClient(client),
47        mSequencer(sequencer),
48        mId(client->getCameraId()),
49        mZslStreamId(NO_STREAM),
50        mFrameListHead(0),
51        mZslQueueHead(0),
52        mZslQueueTail(0) {
53    mZslQueue.insertAt(0, kZslBufferDepth);
54    mFrameList.insertAt(0, kFrameListDepth);
55    sp<CaptureSequencer> captureSequencer = mSequencer.promote();
56    if (captureSequencer != 0) captureSequencer->setZslProcessor(this);
57}
58
59ZslProcessor3::~ZslProcessor3() {
60    ALOGV("%s: Exit", __FUNCTION__);
61    deleteStream();
62}
63
64void ZslProcessor3::onFrameAvailable(int32_t /*requestId*/,
65                                     const CameraMetadata &frame) {
66    Mutex::Autolock l(mInputMutex);
67    camera_metadata_ro_entry_t entry;
68    entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
69    nsecs_t timestamp = entry.data.i64[0];
70    (void)timestamp;
71    ALOGVV("Got preview metadata for timestamp %lld", timestamp);
72
73    if (mState != RUNNING) return;
74
75    mFrameList.editItemAt(mFrameListHead) = frame;
76    mFrameListHead = (mFrameListHead + 1) % kFrameListDepth;
77}
78
79status_t ZslProcessor3::updateStream(const Parameters &params) {
80    ATRACE_CALL();
81    ALOGV("%s: Configuring ZSL streams", __FUNCTION__);
82    status_t res;
83
84    Mutex::Autolock l(mInputMutex);
85
86    sp<Camera2Client> client = mClient.promote();
87    if (client == 0) {
88        ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
89        return INVALID_OPERATION;
90    }
91    sp<Camera3Device> device =
92        static_cast<Camera3Device*>(client->getCameraDevice().get());
93    if (device == 0) {
94        ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
95        return INVALID_OPERATION;
96    }
97
98    if (mZslStreamId != NO_STREAM) {
99        // Check if stream parameters have to change
100        uint32_t currentWidth, currentHeight;
101        res = device->getStreamInfo(mZslStreamId,
102                &currentWidth, &currentHeight, 0);
103        if (res != OK) {
104            ALOGE("%s: Camera %d: Error querying capture output stream info: "
105                    "%s (%d)", __FUNCTION__,
106                    client->getCameraId(), strerror(-res), res);
107            return res;
108        }
109        if (currentWidth != (uint32_t)params.fastInfo.arrayWidth ||
110                currentHeight != (uint32_t)params.fastInfo.arrayHeight) {
111            ALOGV("%s: Camera %d: Deleting stream %d since the buffer "
112                  "dimensions changed",
113                __FUNCTION__, client->getCameraId(), mZslStreamId);
114            res = device->deleteStream(mZslStreamId);
115            if (res == -EBUSY) {
116                ALOGV("%s: Camera %d: Device is busy, call updateStream again "
117                      " after it becomes idle", __FUNCTION__, mId);
118                return res;
119            } else if(res != OK) {
120                ALOGE("%s: Camera %d: Unable to delete old output stream "
121                        "for ZSL: %s (%d)", __FUNCTION__,
122                        client->getCameraId(), strerror(-res), res);
123                return res;
124            }
125            mZslStreamId = NO_STREAM;
126        }
127    }
128
129    if (mZslStreamId == NO_STREAM) {
130        // Create stream for HAL production
131        // TODO: Sort out better way to select resolution for ZSL
132
133        // Note that format specified internally in Camera3ZslStream
134        res = device->createZslStream(
135                params.fastInfo.arrayWidth, params.fastInfo.arrayHeight,
136                kZslBufferDepth,
137                &mZslStreamId,
138                &mZslStream);
139        if (res != OK) {
140            ALOGE("%s: Camera %d: Can't create ZSL stream: "
141                    "%s (%d)", __FUNCTION__, client->getCameraId(),
142                    strerror(-res), res);
143            return res;
144        }
145    }
146    client->registerFrameListener(Camera2Client::kPreviewRequestIdStart,
147            Camera2Client::kPreviewRequestIdEnd,
148            this);
149
150    return OK;
151}
152
153status_t ZslProcessor3::deleteStream() {
154    ATRACE_CALL();
155    status_t res;
156
157    Mutex::Autolock l(mInputMutex);
158
159    if (mZslStreamId != NO_STREAM) {
160        sp<Camera2Client> client = mClient.promote();
161        if (client == 0) {
162            ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
163            return INVALID_OPERATION;
164        }
165
166        sp<Camera3Device> device =
167            reinterpret_cast<Camera3Device*>(client->getCameraDevice().get());
168        if (device == 0) {
169            ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
170            return INVALID_OPERATION;
171        }
172
173        res = device->deleteStream(mZslStreamId);
174        if (res != OK) {
175            ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: "
176                    "%s (%d)", __FUNCTION__, client->getCameraId(),
177                    mZslStreamId, strerror(-res), res);
178            return res;
179        }
180
181        mZslStreamId = NO_STREAM;
182    }
183    return OK;
184}
185
186int ZslProcessor3::getStreamId() const {
187    Mutex::Autolock l(mInputMutex);
188    return mZslStreamId;
189}
190
191status_t ZslProcessor3::pushToReprocess(int32_t requestId) {
192    ALOGV("%s: Send in reprocess request with id %d",
193            __FUNCTION__, requestId);
194    Mutex::Autolock l(mInputMutex);
195    status_t res;
196    sp<Camera2Client> client = mClient.promote();
197
198    if (client == 0) {
199        ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
200        return INVALID_OPERATION;
201    }
202
203    IF_ALOGV() {
204        dumpZslQueue(-1);
205    }
206
207    size_t metadataIdx;
208    nsecs_t candidateTimestamp = getCandidateTimestampLocked(&metadataIdx);
209
210    if (candidateTimestamp == -1) {
211        ALOGE("%s: Could not find good candidate for ZSL reprocessing",
212              __FUNCTION__);
213        return NOT_ENOUGH_DATA;
214    }
215
216    res = mZslStream->enqueueInputBufferByTimestamp(candidateTimestamp,
217                                                    /*actualTimestamp*/NULL);
218
219    if (res == mZslStream->NO_BUFFER_AVAILABLE) {
220        ALOGV("%s: No ZSL buffers yet", __FUNCTION__);
221        return NOT_ENOUGH_DATA;
222    } else if (res != OK) {
223        ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)",
224                __FUNCTION__, strerror(-res), res);
225        return res;
226    }
227
228    {
229        CameraMetadata request = mFrameList[metadataIdx];
230
231        // Verify that the frame is reasonable for reprocessing
232
233        camera_metadata_entry_t entry;
234        entry = request.find(ANDROID_CONTROL_AE_STATE);
235        if (entry.count == 0) {
236            ALOGE("%s: ZSL queue frame has no AE state field!",
237                    __FUNCTION__);
238            return BAD_VALUE;
239        }
240        if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
241                entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
242            ALOGV("%s: ZSL queue frame AE state is %d, need full capture",
243                    __FUNCTION__, entry.data.u8[0]);
244            return NOT_ENOUGH_DATA;
245        }
246
247        uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS;
248        res = request.update(ANDROID_REQUEST_TYPE,
249                &requestType, 1);
250        int32_t inputStreams[1] =
251                { mZslStreamId };
252        if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS,
253                inputStreams, 1);
254        // TODO: Shouldn't we also update the latest preview frame?
255        int32_t outputStreams[1] =
256                { client->getCaptureStreamId() };
257        if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS,
258                outputStreams, 1);
259        res = request.update(ANDROID_REQUEST_ID,
260                &requestId, 1);
261
262        if (res != OK ) {
263            ALOGE("%s: Unable to update frame to a reprocess request",
264                  __FUNCTION__);
265            return INVALID_OPERATION;
266        }
267
268        res = client->stopStream();
269        if (res != OK) {
270            ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: "
271                "%s (%d)",
272                __FUNCTION__, client->getCameraId(), strerror(-res), res);
273            return INVALID_OPERATION;
274        }
275
276        // Flush device to clear out all in-flight requests pending in HAL.
277        res = client->getCameraDevice()->flush();
278        if (res != OK) {
279            ALOGE("%s: Camera %d: Failed to flush device: "
280                "%s (%d)",
281                __FUNCTION__, client->getCameraId(), strerror(-res), res);
282            return res;
283        }
284
285        // Update JPEG settings
286        {
287            SharedParameters::Lock l(client->getParameters());
288            res = l.mParameters.updateRequestJpeg(&request);
289            if (res != OK) {
290                ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL "
291                        "capture request: %s (%d)", __FUNCTION__,
292                        client->getCameraId(),
293                        strerror(-res), res);
294                return res;
295            }
296        }
297
298        mLatestCapturedRequest = request;
299        res = client->getCameraDevice()->capture(request);
300        if (res != OK ) {
301            ALOGE("%s: Unable to send ZSL reprocess request to capture: %s"
302                  " (%d)", __FUNCTION__, strerror(-res), res);
303            return res;
304        }
305
306        mState = LOCKED;
307    }
308
309    return OK;
310}
311
312status_t ZslProcessor3::clearZslQueue() {
313    Mutex::Autolock l(mInputMutex);
314    // If in middle of capture, can't clear out queue
315    if (mState == LOCKED) return OK;
316
317    return clearZslQueueLocked();
318}
319
320status_t ZslProcessor3::clearZslQueueLocked() {
321    if (mZslStream != 0) {
322        return mZslStream->clearInputRingBuffer();
323    }
324    return OK;
325}
326
327void ZslProcessor3::dump(int fd, const Vector<String16>& /*args*/) const {
328    Mutex::Autolock l(mInputMutex);
329    if (!mLatestCapturedRequest.isEmpty()) {
330        String8 result("    Latest ZSL capture request:\n");
331        write(fd, result.string(), result.size());
332        mLatestCapturedRequest.dump(fd, 2, 6);
333    } else {
334        String8 result("    Latest ZSL capture request: none yet\n");
335        write(fd, result.string(), result.size());
336    }
337    dumpZslQueue(fd);
338}
339
340bool ZslProcessor3::threadLoop() {
341    // TODO: remove dependency on thread. For now, shut thread down right
342    // away.
343    return false;
344}
345
346void ZslProcessor3::dumpZslQueue(int fd) const {
347    String8 header("ZSL queue contents:");
348    String8 indent("    ");
349    ALOGV("%s", header.string());
350    if (fd != -1) {
351        header = indent + header + "\n";
352        write(fd, header.string(), header.size());
353    }
354    for (size_t i = 0; i < mZslQueue.size(); i++) {
355        const ZslPair &queueEntry = mZslQueue[i];
356        nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
357        camera_metadata_ro_entry_t entry;
358        nsecs_t frameTimestamp = 0;
359        int frameAeState = -1;
360        if (!queueEntry.frame.isEmpty()) {
361            entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
362            if (entry.count > 0) frameTimestamp = entry.data.i64[0];
363            entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE);
364            if (entry.count > 0) frameAeState = entry.data.u8[0];
365        }
366        String8 result =
367                String8::format("   %d: b: %lld\tf: %lld, AE state: %d", i,
368                        bufferTimestamp, frameTimestamp, frameAeState);
369        ALOGV("%s", result.string());
370        if (fd != -1) {
371            result = indent + result + "\n";
372            write(fd, result.string(), result.size());
373        }
374
375    }
376}
377
378nsecs_t ZslProcessor3::getCandidateTimestampLocked(size_t* metadataIdx) const {
379    /**
380     * Find the smallest timestamp we know about so far
381     * - ensure that aeState is either converged or locked
382     */
383
384    size_t idx = 0;
385    nsecs_t minTimestamp = -1;
386
387    size_t emptyCount = mFrameList.size();
388
389    for (size_t j = 0; j < mFrameList.size(); j++) {
390        const CameraMetadata &frame = mFrameList[j];
391        if (!frame.isEmpty()) {
392
393            emptyCount--;
394
395            camera_metadata_ro_entry_t entry;
396            entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
397            if (entry.count == 0) {
398                ALOGE("%s: Can't find timestamp in frame!",
399                        __FUNCTION__);
400                continue;
401            }
402            nsecs_t frameTimestamp = entry.data.i64[0];
403            if (minTimestamp > frameTimestamp || minTimestamp == -1) {
404
405                entry = frame.find(ANDROID_CONTROL_AE_STATE);
406
407                if (entry.count == 0) {
408                    /**
409                     * This is most likely a HAL bug. The aeState field is
410                     * mandatory, so it should always be in a metadata packet.
411                     */
412                    ALOGW("%s: ZSL queue frame has no AE state field!",
413                            __FUNCTION__);
414                    continue;
415                }
416                if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
417                        entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
418                    ALOGVV("%s: ZSL queue frame AE state is %d, need "
419                           "full capture",  __FUNCTION__, entry.data.u8[0]);
420                    continue;
421                }
422
423                minTimestamp = frameTimestamp;
424                idx = j;
425            }
426
427            ALOGVV("%s: Saw timestamp %lld", __FUNCTION__, frameTimestamp);
428        }
429    }
430
431    if (emptyCount == mFrameList.size()) {
432        /**
433         * This could be mildly bad and means our ZSL was triggered before
434         * there were any frames yet received by the camera framework.
435         *
436         * This is a fairly corner case which can happen under:
437         * + a user presses the shutter button real fast when the camera starts
438         *     (startPreview followed immediately by takePicture).
439         * + burst capture case (hitting shutter button as fast possible)
440         *
441         * If this happens in steady case (preview running for a while, call
442         *     a single takePicture) then this might be a fwk bug.
443         */
444        ALOGW("%s: ZSL queue has no metadata frames", __FUNCTION__);
445    }
446
447    ALOGV("%s: Candidate timestamp %lld (idx %d), empty frames: %d",
448          __FUNCTION__, minTimestamp, idx, emptyCount);
449
450    if (metadataIdx) {
451        *metadataIdx = idx;
452    }
453
454    return minTimestamp;
455}
456
457void ZslProcessor3::onBufferAcquired(const BufferInfo& /*bufferInfo*/) {
458    // Intentionally left empty
459    // Although theoretically we could use this to get better dump info
460}
461
462void ZslProcessor3::onBufferReleased(const BufferInfo& bufferInfo) {
463    Mutex::Autolock l(mInputMutex);
464
465    // ignore output buffers
466    if (bufferInfo.mOutput) {
467        return;
468    }
469
470    // TODO: Verify that the buffer is in our queue by looking at timestamp
471    // theoretically unnecessary unless we change the following assumptions:
472    // -- only 1 buffer reprocessed at a time (which is the case now)
473
474    // Erase entire ZSL queue since we've now completed the capture and preview
475    // is stopped.
476    //
477    // We need to guarantee that if we do two back-to-back captures,
478    // the second won't use a buffer that's older/the same as the first, which
479    // is theoretically possible if we don't clear out the queue and the
480    // selection criteria is something like 'newest'. Clearing out the queue
481    // on a completed capture ensures we'll only use new data.
482    ALOGV("%s: Memory optimization, clearing ZSL queue",
483          __FUNCTION__);
484    clearZslQueueLocked();
485
486    // Required so we accept more ZSL requests
487    mState = RUNNING;
488}
489
490}; // namespace camera2
491}; // namespace android
492