ZslProcessor.cpp revision d1d6467d3bcbc1305eeba0176a2edf04925c368e 
1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "Camera2-ZslProcessor"
18#define ATRACE_TAG ATRACE_TAG_CAMERA
19//#define LOG_NDEBUG 0
20//#define LOG_NNDEBUG 0
21
22#ifdef LOG_NNDEBUG
23#define ALOGVV(...) ALOGV(__VA_ARGS__)
24#else
25#define ALOGVV(...) ((void)0)
26#endif
27
28#include <utils/Log.h>
29#include <utils/Trace.h>
30#include <gui/Surface.h>
31
32#include "common/CameraDeviceBase.h"
33#include "api1/Camera2Client.h"
34#include "api1/client2/CaptureSequencer.h"
35#include "api1/client2/ZslProcessor.h"
36
37namespace android {
38namespace camera2 {
39
40ZslProcessor::ZslProcessor(
41    sp<Camera2Client> client,
42    wp<CaptureSequencer> sequencer):
43        Thread(false),
44        mState(RUNNING),
45        mClient(client),
46        mDevice(client->getCameraDevice()),
47        mSequencer(sequencer),
48        mId(client->getCameraId()),
49        mZslBufferAvailable(false),
50        mZslStreamId(NO_STREAM),
51        mZslReprocessStreamId(NO_STREAM),
52        mFrameListHead(0),
53        mZslQueueHead(0),
54        mZslQueueTail(0) {
55    mZslQueue.insertAt(0, kZslBufferDepth);
56    mFrameList.insertAt(0, kFrameListDepth);
57    sp<CaptureSequencer> captureSequencer = mSequencer.promote();
58    if (captureSequencer != 0) captureSequencer->setZslProcessor(this);
59}
60
61ZslProcessor::~ZslProcessor() {
62    ALOGV("%s: Exit", __FUNCTION__);
63    deleteStream();
64}
65
66void ZslProcessor::onFrameAvailable() {
67    Mutex::Autolock l(mInputMutex);
68    if (!mZslBufferAvailable) {
69        mZslBufferAvailable = true;
70        mZslBufferAvailableSignal.signal();
71    }
72}
73
74void ZslProcessor::onFrameAvailable(int32_t /*frameId*/,
75        const CameraMetadata &frame) {
76    Mutex::Autolock l(mInputMutex);
77    camera_metadata_ro_entry_t entry;
78    entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
79    nsecs_t timestamp = entry.data.i64[0];
80    (void)timestamp;
81    ALOGVV("Got preview frame for timestamp %lld", timestamp);
82
83    if (mState != RUNNING) return;
84
85    mFrameList.editItemAt(mFrameListHead) = frame;
86    mFrameListHead = (mFrameListHead + 1) % kFrameListDepth;
87
88    findMatchesLocked();
89}
90
91void ZslProcessor::onBufferReleased(buffer_handle_t *handle) {
92    Mutex::Autolock l(mInputMutex);
93
94    // Verify that the buffer is in our queue
95    size_t i = 0;
96    for (; i < mZslQueue.size(); i++) {
97        if (&(mZslQueue[i].buffer.mGraphicBuffer->handle) == handle) break;
98    }
99    if (i == mZslQueue.size()) {
100        ALOGW("%s: Released buffer %p not found in queue",
101                __FUNCTION__, handle);
102    }
103
104    // Erase entire ZSL queue since we've now completed the capture and preview
105    // is stopped.
106    clearZslQueueLocked();
107
108    mState = RUNNING;
109}
110
111status_t ZslProcessor::updateStream(const Parameters &params) {
112    ATRACE_CALL();
113    ALOGV("%s: Configuring ZSL streams", __FUNCTION__);
114    status_t res;
115
116    Mutex::Autolock l(mInputMutex);
117
118    sp<Camera2Client> client = mClient.promote();
119    if (client == 0) {
120        ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
121        return INVALID_OPERATION;
122    }
123    sp<CameraDeviceBase> device = mDevice.promote();
124    if (device == 0) {
125        ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
126        return INVALID_OPERATION;
127    }
128
129    if (mZslConsumer == 0) {
130        // Create CPU buffer queue endpoint
131        sp<BufferQueue> bq = new BufferQueue();
132        mZslConsumer = new BufferItemConsumer(bq,
133            GRALLOC_USAGE_HW_CAMERA_ZSL,
134            kZslBufferDepth);
135        mZslConsumer->setFrameAvailableListener(this);
136        mZslConsumer->setName(String8("Camera2Client::ZslConsumer"));
137        mZslWindow = new Surface(bq);
138    }
139
140    if (mZslStreamId != NO_STREAM) {
141        // Check if stream parameters have to change
142        uint32_t currentWidth, currentHeight;
143        res = device->getStreamInfo(mZslStreamId,
144                &currentWidth, &currentHeight, 0);
145        if (res != OK) {
146            ALOGE("%s: Camera %d: Error querying capture output stream info: "
147                    "%s (%d)", __FUNCTION__,
148                    mId, strerror(-res), res);
149            return res;
150        }
151        if (currentWidth != (uint32_t)params.fastInfo.arrayWidth ||
152                currentHeight != (uint32_t)params.fastInfo.arrayHeight) {
153            res = device->deleteReprocessStream(mZslReprocessStreamId);
154            if (res != OK) {
155                ALOGE("%s: Camera %d: Unable to delete old reprocess stream "
156                        "for ZSL: %s (%d)", __FUNCTION__,
157                        mId, strerror(-res), res);
158                return res;
159            }
160            ALOGV("%s: Camera %d: Deleting stream %d since the buffer dimensions changed",
161                __FUNCTION__, mId, mZslStreamId);
162            res = device->deleteStream(mZslStreamId);
163            if (res != OK) {
164                ALOGE("%s: Camera %d: Unable to delete old output stream "
165                        "for ZSL: %s (%d)", __FUNCTION__,
166                        mId, strerror(-res), res);
167                return res;
168            }
169            mZslStreamId = NO_STREAM;
170        }
171    }
172
173    if (mZslStreamId == NO_STREAM) {
174        // Create stream for HAL production
175        // TODO: Sort out better way to select resolution for ZSL
176        int streamType = params.quirks.useZslFormat ?
177                (int)CAMERA2_HAL_PIXEL_FORMAT_ZSL :
178                (int)HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
179        res = device->createStream(mZslWindow,
180                params.fastInfo.arrayWidth, params.fastInfo.arrayHeight,
181                streamType, 0,
182                &mZslStreamId);
183        if (res != OK) {
184            ALOGE("%s: Camera %d: Can't create output stream for ZSL: "
185                    "%s (%d)", __FUNCTION__, mId,
186                    strerror(-res), res);
187            return res;
188        }
189        res = device->createReprocessStreamFromStream(mZslStreamId,
190                &mZslReprocessStreamId);
191        if (res != OK) {
192            ALOGE("%s: Camera %d: Can't create reprocess stream for ZSL: "
193                    "%s (%d)", __FUNCTION__, mId,
194                    strerror(-res), res);
195            return res;
196        }
197    }
198    client->registerFrameListener(Camera2Client::kPreviewRequestIdStart,
199            Camera2Client::kPreviewRequestIdEnd,
200            this);
201
202    return OK;
203}
204
205status_t ZslProcessor::deleteStream() {
206    ATRACE_CALL();
207    status_t res;
208
209    Mutex::Autolock l(mInputMutex);
210
211    if (mZslStreamId != NO_STREAM) {
212        sp<CameraDeviceBase> device = mDevice.promote();
213        if (device == 0) {
214            ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
215            return INVALID_OPERATION;
216        }
217
218        clearZslQueueLocked();
219
220        res = device->deleteReprocessStream(mZslReprocessStreamId);
221        if (res != OK) {
222            ALOGE("%s: Camera %d: Cannot delete ZSL reprocessing stream %d: "
223                    "%s (%d)", __FUNCTION__, mId,
224                    mZslReprocessStreamId, strerror(-res), res);
225            return res;
226        }
227
228        mZslReprocessStreamId = NO_STREAM;
229        res = device->deleteStream(mZslStreamId);
230        if (res != OK) {
231            ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: "
232                    "%s (%d)", __FUNCTION__, mId,
233                    mZslStreamId, strerror(-res), res);
234            return res;
235        }
236
237        mZslWindow.clear();
238        mZslConsumer.clear();
239
240        mZslStreamId = NO_STREAM;
241    }
242    return OK;
243}
244
245int ZslProcessor::getStreamId() const {
246    Mutex::Autolock l(mInputMutex);
247    return mZslStreamId;
248}
249
250status_t ZslProcessor::pushToReprocess(int32_t requestId) {
251    ALOGV("%s: Send in reprocess request with id %d",
252            __FUNCTION__, requestId);
253    Mutex::Autolock l(mInputMutex);
254    status_t res;
255    sp<Camera2Client> client = mClient.promote();
256
257    if (client == 0) {
258        ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
259        return INVALID_OPERATION;
260    }
261
262    IF_ALOGV() {
263        dumpZslQueue(-1);
264    }
265
266    if (mZslQueueTail != mZslQueueHead) {
267        CameraMetadata request;
268        size_t index = mZslQueueTail;
269        while (index != mZslQueueHead) {
270            if (!mZslQueue[index].frame.isEmpty()) {
271                request = mZslQueue[index].frame;
272                break;
273            }
274            index = (index + 1) % kZslBufferDepth;
275        }
276        if (index == mZslQueueHead) {
277            ALOGV("%s: ZSL queue has no valid frames to send yet.",
278                  __FUNCTION__);
279            return NOT_ENOUGH_DATA;
280        }
281        // Verify that the frame is reasonable for reprocessing
282
283        camera_metadata_entry_t entry;
284        entry = request.find(ANDROID_CONTROL_AE_STATE);
285        if (entry.count == 0) {
286            ALOGE("%s: ZSL queue frame has no AE state field!",
287                    __FUNCTION__);
288            return BAD_VALUE;
289        }
290        if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
291                entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
292            ALOGV("%s: ZSL queue frame AE state is %d, need full capture",
293                    __FUNCTION__, entry.data.u8[0]);
294            return NOT_ENOUGH_DATA;
295        }
296
297        buffer_handle_t *handle =
298            &(mZslQueue[index].buffer.mGraphicBuffer->handle);
299
300        uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS;
301        res = request.update(ANDROID_REQUEST_TYPE,
302                &requestType, 1);
303        int32_t inputStreams[1] =
304                { mZslReprocessStreamId };
305        if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS,
306                inputStreams, 1);
307        int32_t outputStreams[1] =
308                { client->getCaptureStreamId() };
309        if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS,
310                outputStreams, 1);
311        res = request.update(ANDROID_REQUEST_ID,
312                &requestId, 1);
313
314        if (res != OK ) {
315            ALOGE("%s: Unable to update frame to a reprocess request", __FUNCTION__);
316            return INVALID_OPERATION;
317        }
318
319        res = client->stopStream();
320        if (res != OK) {
321            ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: "
322                "%s (%d)",
323                __FUNCTION__, mId, strerror(-res), res);
324            return INVALID_OPERATION;
325        }
326        // TODO: have push-and-clear be atomic
327        res = client->getCameraDevice()->pushReprocessBuffer(mZslReprocessStreamId,
328                handle, this);
329        if (res != OK) {
330            ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)",
331                    __FUNCTION__, strerror(-res), res);
332            return res;
333        }
334
335        // Update JPEG settings
336        {
337            SharedParameters::Lock l(client->getParameters());
338            res = l.mParameters.updateRequestJpeg(&request);
339            if (res != OK) {
340                ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL "
341                        "capture request: %s (%d)", __FUNCTION__,
342                        mId,
343                        strerror(-res), res);
344                return res;
345            }
346        }
347
348        mLatestCapturedRequest = request;
349        res = client->getCameraDevice()->capture(request);
350        if (res != OK ) {
351            ALOGE("%s: Unable to send ZSL reprocess request to capture: %s (%d)",
352                    __FUNCTION__, strerror(-res), res);
353            return res;
354        }
355
356        mState = LOCKED;
357    } else {
358        ALOGV("%s: No ZSL buffers yet", __FUNCTION__);
359        return NOT_ENOUGH_DATA;
360    }
361    return OK;
362}
363
364status_t ZslProcessor::clearZslQueue() {
365    Mutex::Autolock l(mInputMutex);
366    // If in middle of capture, can't clear out queue
367    if (mState == LOCKED) return OK;
368
369    return clearZslQueueLocked();
370}
371
372status_t ZslProcessor::clearZslQueueLocked() {
373    for (size_t i = 0; i < mZslQueue.size(); i++) {
374        if (mZslQueue[i].buffer.mTimestamp != 0) {
375            mZslConsumer->releaseBuffer(mZslQueue[i].buffer);
376        }
377        mZslQueue.replaceAt(i);
378    }
379    mZslQueueHead = 0;
380    mZslQueueTail = 0;
381    return OK;
382}
383
384void ZslProcessor::dump(int fd, const Vector<String16>& /*args*/) const {
385    Mutex::Autolock l(mInputMutex);
386    if (!mLatestCapturedRequest.isEmpty()) {
387        String8 result("    Latest ZSL capture request:\n");
388        write(fd, result.string(), result.size());
389        mLatestCapturedRequest.dump(fd, 2, 6);
390    } else {
391        String8 result("    Latest ZSL capture request: none yet\n");
392        write(fd, result.string(), result.size());
393    }
394    dumpZslQueue(fd);
395}
396
397bool ZslProcessor::threadLoop() {
398    status_t res;
399
400    {
401        Mutex::Autolock l(mInputMutex);
402        while (!mZslBufferAvailable) {
403            res = mZslBufferAvailableSignal.waitRelative(mInputMutex,
404                    kWaitDuration);
405            if (res == TIMED_OUT) return true;
406        }
407        mZslBufferAvailable = false;
408    }
409
410    do {
411        res = processNewZslBuffer();
412    } while (res == OK);
413
414    return true;
415}
416
417status_t ZslProcessor::processNewZslBuffer() {
418    ATRACE_CALL();
419    status_t res;
420    sp<BufferItemConsumer> zslConsumer;
421    {
422        Mutex::Autolock l(mInputMutex);
423        if (mZslConsumer == 0) return OK;
424        zslConsumer = mZslConsumer;
425    }
426    ALOGVV("Trying to get next buffer");
427    BufferItemConsumer::BufferItem item;
428    res = zslConsumer->acquireBuffer(&item, 0);
429    if (res != OK) {
430        if (res != BufferItemConsumer::NO_BUFFER_AVAILABLE) {
431            ALOGE("%s: Camera %d: Error receiving ZSL image buffer: "
432                    "%s (%d)", __FUNCTION__,
433                    mId, strerror(-res), res);
434        } else {
435            ALOGVV("  No buffer");
436        }
437        return res;
438    }
439
440    Mutex::Autolock l(mInputMutex);
441
442    if (mState == LOCKED) {
443        ALOGVV("In capture, discarding new ZSL buffers");
444        zslConsumer->releaseBuffer(item);
445        return OK;
446    }
447
448    ALOGVV("Got ZSL buffer: head: %d, tail: %d", mZslQueueHead, mZslQueueTail);
449
450    if ( (mZslQueueHead + 1) % kZslBufferDepth == mZslQueueTail) {
451        ALOGVV("Releasing oldest buffer");
452        zslConsumer->releaseBuffer(mZslQueue[mZslQueueTail].buffer);
453        mZslQueue.replaceAt(mZslQueueTail);
454        mZslQueueTail = (mZslQueueTail + 1) % kZslBufferDepth;
455    }
456
457    ZslPair &queueHead = mZslQueue.editItemAt(mZslQueueHead);
458
459    queueHead.buffer = item;
460    queueHead.frame.release();
461
462    mZslQueueHead = (mZslQueueHead + 1) % kZslBufferDepth;
463
464    ALOGVV("  Acquired buffer, timestamp %lld", queueHead.buffer.mTimestamp);
465
466    findMatchesLocked();
467
468    return OK;
469}
470
471void ZslProcessor::findMatchesLocked() {
472    ALOGVV("Scanning");
473    for (size_t i = 0; i < mZslQueue.size(); i++) {
474        ZslPair &queueEntry = mZslQueue.editItemAt(i);
475        nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
476        IF_ALOGV() {
477            camera_metadata_entry_t entry;
478            nsecs_t frameTimestamp = 0;
479            if (!queueEntry.frame.isEmpty()) {
480                entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
481                frameTimestamp = entry.data.i64[0];
482            }
483            ALOGVV("   %d: b: %lld\tf: %lld", i,
484                    bufferTimestamp, frameTimestamp );
485        }
486        if (queueEntry.frame.isEmpty() && bufferTimestamp != 0) {
487            // Have buffer, no matching frame. Look for one
488            for (size_t j = 0; j < mFrameList.size(); j++) {
489                bool match = false;
490                CameraMetadata &frame = mFrameList.editItemAt(j);
491                if (!frame.isEmpty()) {
492                    camera_metadata_entry_t entry;
493                    entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
494                    if (entry.count == 0) {
495                        ALOGE("%s: Can't find timestamp in frame!",
496                                __FUNCTION__);
497                        continue;
498                    }
499                    nsecs_t frameTimestamp = entry.data.i64[0];
500                    if (bufferTimestamp == frameTimestamp) {
501                        ALOGVV("%s: Found match %lld", __FUNCTION__,
502                                frameTimestamp);
503                        match = true;
504                    } else {
505                        int64_t delta = abs(bufferTimestamp - frameTimestamp);
506                        if ( delta < 1000000) {
507                            ALOGVV("%s: Found close match %lld (delta %lld)",
508                                    __FUNCTION__, bufferTimestamp, delta);
509                            match = true;
510                        }
511                    }
512                }
513                if (match) {
514                    queueEntry.frame.acquire(frame);
515                    break;
516                }
517            }
518        }
519    }
520}
521
522void ZslProcessor::dumpZslQueue(int fd) const {
523    String8 header("ZSL queue contents:");
524    String8 indent("    ");
525    ALOGV("%s", header.string());
526    if (fd != -1) {
527        header = indent + header + "\n";
528        write(fd, header.string(), header.size());
529    }
530    for (size_t i = 0; i < mZslQueue.size(); i++) {
531        const ZslPair &queueEntry = mZslQueue[i];
532        nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
533        camera_metadata_ro_entry_t entry;
534        nsecs_t frameTimestamp = 0;
535        int frameAeState = -1;
536        if (!queueEntry.frame.isEmpty()) {
537            entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
538            if (entry.count > 0) frameTimestamp = entry.data.i64[0];
539            entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE);
540            if (entry.count > 0) frameAeState = entry.data.u8[0];
541        }
542        String8 result =
543                String8::format("   %d: b: %lld\tf: %lld, AE state: %d", i,
544                        bufferTimestamp, frameTimestamp, frameAeState);
545        ALOGV("%s", result.string());
546        if (fd != -1) {
547            result = indent + result + "\n";
548            write(fd, result.string(), result.size());
549        }
550
551    }
552}
553
554}; // namespace camera2
555}; // namespace android
556