ZslProcessor.cpp revision ecf17e82505fdb60d59e00b6dd59036df93de655 
1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "Camera2-ZslProcessor"
18#define ATRACE_TAG ATRACE_TAG_CAMERA
19//#define LOG_NDEBUG 0
20//#define LOG_NNDEBUG 0
21
22#ifdef LOG_NNDEBUG
23#define ALOGVV(...) ALOGV(__VA_ARGS__)
24#else
25#define ALOGVV(...) ((void)0)
26#endif
27
28#include <utils/Log.h>
29#include <utils/Trace.h>
30
31#include "ZslProcessor.h"
32#include <gui/SurfaceTextureClient.h>
33#include "../Camera2Device.h"
34#include "../Camera2Client.h"
35
36
37namespace android {
38namespace camera2 {
39
40ZslProcessor::ZslProcessor(
41    wp<Camera2Client> client,
42    wp<CaptureSequencer> sequencer):
43        Thread(false),
44        mState(RUNNING),
45        mClient(client),
46        mSequencer(sequencer),
47        mZslBufferAvailable(false),
48        mZslStreamId(NO_STREAM),
49        mZslReprocessStreamId(NO_STREAM),
50        mFrameListHead(0),
51        mZslQueueHead(0),
52        mZslQueueTail(0) {
53    mZslQueue.insertAt(0, kZslBufferDepth);
54    mFrameList.insertAt(0, kFrameListDepth);
55    sp<CaptureSequencer> captureSequencer = mSequencer.promote();
56    if (captureSequencer != 0) captureSequencer->setZslProcessor(this);
57}
58
59ZslProcessor::~ZslProcessor() {
60    ALOGV("%s: Exit", __FUNCTION__);
61    deleteStream();
62}
63
64void ZslProcessor::onFrameAvailable() {
65    Mutex::Autolock l(mInputMutex);
66    if (!mZslBufferAvailable) {
67        mZslBufferAvailable = true;
68        mZslBufferAvailableSignal.signal();
69    }
70}
71
72void ZslProcessor::onFrameAvailable(int32_t frameId, CameraMetadata &frame) {
73    Mutex::Autolock l(mInputMutex);
74    camera_metadata_entry_t entry;
75    entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
76    nsecs_t timestamp = entry.data.i64[0];
77    ALOGVV("Got preview frame for timestamp %lld", timestamp);
78
79    if (mState != RUNNING) return;
80
81    mFrameList.editItemAt(mFrameListHead).acquire(frame);
82    mFrameListHead = (mFrameListHead + 1) % kFrameListDepth;
83
84    findMatchesLocked();
85}
86
87void ZslProcessor::onBufferReleased(buffer_handle_t *handle) {
88    Mutex::Autolock l(mInputMutex);
89
90    // Verify that the buffer is in our queue
91    size_t i = 0;
92    for (; i < mZslQueue.size(); i++) {
93        if (&(mZslQueue[i].buffer.mGraphicBuffer->handle) == handle) break;
94    }
95    if (i == mZslQueue.size()) {
96        ALOGW("%s: Released buffer %p not found in queue",
97                __FUNCTION__, handle);
98    }
99
100    // Erase entire ZSL queue since we've now completed the capture and preview
101    // is stopped.
102    clearZslQueueLocked();
103
104    mState = RUNNING;
105}
106
107status_t ZslProcessor::updateStream(const Parameters &params) {
108    ATRACE_CALL();
109    ALOGV("%s: Configuring ZSL streams", __FUNCTION__);
110    status_t res;
111
112    Mutex::Autolock l(mInputMutex);
113
114    sp<Camera2Client> client = mClient.promote();
115    if (client == 0) return OK;
116    sp<Camera2Device> device = client->getCameraDevice();
117
118    if (mZslConsumer == 0) {
119        // Create CPU buffer queue endpoint
120        mZslConsumer = new BufferItemConsumer(
121            GRALLOC_USAGE_HW_CAMERA_ZSL,
122            kZslBufferDepth,
123            true);
124        mZslConsumer->setFrameAvailableListener(this);
125        mZslConsumer->setName(String8("Camera2Client::ZslConsumer"));
126        mZslWindow = new SurfaceTextureClient(
127            mZslConsumer->getProducerInterface());
128    }
129
130    if (mZslStreamId != NO_STREAM) {
131        // Check if stream parameters have to change
132        uint32_t currentWidth, currentHeight;
133        res = device->getStreamInfo(mZslStreamId,
134                &currentWidth, &currentHeight, 0);
135        if (res != OK) {
136            ALOGE("%s: Camera %d: Error querying capture output stream info: "
137                    "%s (%d)", __FUNCTION__,
138                    client->getCameraId(), strerror(-res), res);
139            return res;
140        }
141        if (currentWidth != (uint32_t)params.fastInfo.arrayWidth ||
142                currentHeight != (uint32_t)params.fastInfo.arrayHeight) {
143            res = device->deleteReprocessStream(mZslReprocessStreamId);
144            if (res != OK) {
145                ALOGE("%s: Camera %d: Unable to delete old reprocess stream "
146                        "for ZSL: %s (%d)", __FUNCTION__,
147                        client->getCameraId(), strerror(-res), res);
148                return res;
149            }
150            ALOGV("%s: Camera %d: Deleting stream %d since the buffer dimensions changed",
151                __FUNCTION__, client->getCameraId(), mZslStreamId);
152            res = device->deleteStream(mZslStreamId);
153            if (res != OK) {
154                ALOGE("%s: Camera %d: Unable to delete old output stream "
155                        "for ZSL: %s (%d)", __FUNCTION__,
156                        client->getCameraId(), strerror(-res), res);
157                return res;
158            }
159            mZslStreamId = NO_STREAM;
160        }
161    }
162
163    if (mZslStreamId == NO_STREAM) {
164        // Create stream for HAL production
165        // TODO: Sort out better way to select resolution for ZSL
166        int streamType = params.quirks.useZslFormat ?
167                (int)CAMERA2_HAL_PIXEL_FORMAT_ZSL :
168                (int)HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
169        res = device->createStream(mZslWindow,
170                params.fastInfo.arrayWidth, params.fastInfo.arrayHeight,
171                streamType, 0,
172                &mZslStreamId);
173        if (res != OK) {
174            ALOGE("%s: Camera %d: Can't create output stream for ZSL: "
175                    "%s (%d)", __FUNCTION__, client->getCameraId(),
176                    strerror(-res), res);
177            return res;
178        }
179        res = device->createReprocessStreamFromStream(mZslStreamId,
180                &mZslReprocessStreamId);
181        if (res != OK) {
182            ALOGE("%s: Camera %d: Can't create reprocess stream for ZSL: "
183                    "%s (%d)", __FUNCTION__, client->getCameraId(),
184                    strerror(-res), res);
185            return res;
186        }
187    }
188    client->registerFrameListener(Camera2Client::kPreviewRequestId, this);
189
190    return OK;
191}
192
193status_t ZslProcessor::deleteStream() {
194    ATRACE_CALL();
195    status_t res;
196
197    Mutex::Autolock l(mInputMutex);
198
199    if (mZslStreamId != NO_STREAM) {
200        sp<Camera2Client> client = mClient.promote();
201        if (client == 0) return OK;
202        sp<Camera2Device> device = client->getCameraDevice();
203
204        res = device->deleteReprocessStream(mZslReprocessStreamId);
205        if (res != OK) {
206            ALOGE("%s: Camera %d: Cannot delete ZSL reprocessing stream %d: "
207                    "%s (%d)", __FUNCTION__, client->getCameraId(),
208                    mZslReprocessStreamId, strerror(-res), res);
209            return res;
210        }
211
212        mZslReprocessStreamId = NO_STREAM;
213        res = device->deleteStream(mZslStreamId);
214        if (res != OK) {
215            ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: "
216                    "%s (%d)", __FUNCTION__, client->getCameraId(),
217                    mZslStreamId, strerror(-res), res);
218            return res;
219        }
220
221        mZslWindow.clear();
222        mZslConsumer.clear();
223
224        mZslStreamId = NO_STREAM;
225    }
226    return OK;
227}
228
229int ZslProcessor::getStreamId() const {
230    Mutex::Autolock l(mInputMutex);
231    return mZslStreamId;
232}
233
234int ZslProcessor::getReprocessStreamId() const {
235    Mutex::Autolock l(mInputMutex);
236    return mZslReprocessStreamId;
237}
238
239status_t ZslProcessor::pushToReprocess(int32_t requestId) {
240    ALOGV("%s: Send in reprocess request with id %d",
241            __FUNCTION__, requestId);
242    Mutex::Autolock l(mInputMutex);
243    status_t res;
244    sp<Camera2Client> client = mClient.promote();
245
246    if (client == 0) return INVALID_OPERATION;
247
248    IF_ALOGV() {
249        dumpZslQueue(-1);
250    }
251
252    if (mZslQueueTail != mZslQueueHead) {
253        CameraMetadata request;
254        size_t index = mZslQueueTail;
255        while (request.isEmpty() && index != mZslQueueHead) {
256            request = mZslQueue[index].frame;
257            index = (index + 1) % kZslBufferDepth;
258        }
259        if (request.isEmpty()) {
260            ALOGV("%s: ZSL queue has no valid frames to send yet.",
261                  __FUNCTION__);
262            return NOT_ENOUGH_DATA;
263        }
264        // Verify that the frame is reasonable for reprocessing
265
266        camera_metadata_entry_t entry;
267        entry = request.find(ANDROID_CONTROL_AE_STATE);
268        if (entry.count == 0) {
269            ALOGE("%s: ZSL queue frame has no AE state field!",
270                    __FUNCTION__);
271            return BAD_VALUE;
272        }
273        if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
274                entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
275            ALOGV("%s: ZSL queue frame AE state is %d, need full capture",
276                    __FUNCTION__, entry.data.u8[0]);
277            return NOT_ENOUGH_DATA;
278        }
279
280        buffer_handle_t *handle =
281            &(mZslQueue[index].buffer.mGraphicBuffer->handle);
282
283        uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS;
284        res = request.update(ANDROID_REQUEST_TYPE,
285                &requestType, 1);
286        uint8_t inputStreams[1] = { mZslReprocessStreamId };
287        if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS,
288                inputStreams, 1);
289        uint8_t outputStreams[1] = { client->getCaptureStreamId() };
290        if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS,
291                outputStreams, 1);
292        res = request.update(ANDROID_REQUEST_ID,
293                &requestId, 1);
294
295        if (res != OK ) {
296            ALOGE("%s: Unable to update frame to a reprocess request", __FUNCTION__);
297            return INVALID_OPERATION;
298        }
299
300        res = client->getCameraDevice()->pushReprocessBuffer(mZslReprocessStreamId,
301                handle, this);
302        if (res != OK) {
303            ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)",
304                    __FUNCTION__, strerror(-res), res);
305            return res;
306        }
307
308        res = client->getCameraDevice()->capture(request);
309        if (res != OK ) {
310            ALOGE("%s: Unable to send ZSL reprocess request to capture: %s (%d)",
311                    __FUNCTION__, strerror(-res), res);
312            return res;
313        }
314
315        mState = LOCKED;
316    } else {
317        ALOGV("%s: No ZSL buffers yet", __FUNCTION__);
318        return NOT_ENOUGH_DATA;
319    }
320    return OK;
321}
322
323status_t ZslProcessor::clearZslQueue() {
324    Mutex::Autolock l(mInputMutex);
325    // If in middle of capture, can't clear out queue
326    if (mState == LOCKED) return OK;
327
328    return clearZslQueueLocked();
329}
330
331status_t ZslProcessor::clearZslQueueLocked() {
332    for (size_t i = 0; i < mZslQueue.size(); i++) {
333        if (mZslQueue[i].buffer.mTimestamp != 0) {
334            mZslConsumer->releaseBuffer(mZslQueue[i].buffer);
335        }
336        mZslQueue.replaceAt(i);
337    }
338    mZslQueueHead = 0;
339    mZslQueueTail = 0;
340    return OK;
341}
342
343void ZslProcessor::dump(int fd, const Vector<String16>& args) const {
344    Mutex::Autolock l(mInputMutex);
345    dumpZslQueue(fd);
346}
347
348bool ZslProcessor::threadLoop() {
349    status_t res;
350
351    {
352        Mutex::Autolock l(mInputMutex);
353        while (!mZslBufferAvailable) {
354            res = mZslBufferAvailableSignal.waitRelative(mInputMutex,
355                    kWaitDuration);
356            if (res == TIMED_OUT) return true;
357        }
358        mZslBufferAvailable = false;
359    }
360
361    do {
362        sp<Camera2Client> client = mClient.promote();
363        if (client == 0) return false;
364        res = processNewZslBuffer(client);
365    } while (res == OK);
366
367    return true;
368}
369
370status_t ZslProcessor::processNewZslBuffer(sp<Camera2Client> &client) {
371    ATRACE_CALL();
372    status_t res;
373
374    ALOGVV("Trying to get next buffer");
375    BufferItemConsumer::BufferItem item;
376    res = mZslConsumer->acquireBuffer(&item);
377    if (res != OK) {
378        if (res != BufferItemConsumer::NO_BUFFER_AVAILABLE) {
379            ALOGE("%s: Camera %d: Error receiving ZSL image buffer: "
380                    "%s (%d)", __FUNCTION__,
381                    client->getCameraId(), strerror(-res), res);
382        } else {
383            ALOGVV("  No buffer");
384        }
385        return res;
386    }
387
388    Mutex::Autolock l(mInputMutex);
389
390    if (mState == LOCKED) {
391        ALOGVV("In capture, discarding new ZSL buffers");
392        mZslConsumer->releaseBuffer(item);
393        return OK;
394    }
395
396    ALOGVV("Got ZSL buffer: head: %d, tail: %d", mZslQueueHead, mZslQueueTail);
397
398    if ( (mZslQueueHead + 1) % kZslBufferDepth == mZslQueueTail) {
399        ALOGVV("Releasing oldest buffer");
400        mZslConsumer->releaseBuffer(mZslQueue[mZslQueueTail].buffer);
401        mZslQueue.replaceAt(mZslQueueTail);
402        mZslQueueTail = (mZslQueueTail + 1) % kZslBufferDepth;
403    }
404
405    ZslPair &queueHead = mZslQueue.editItemAt(mZslQueueHead);
406
407    queueHead.buffer = item;
408    queueHead.frame.release();
409
410    mZslQueueHead = (mZslQueueHead + 1) % kZslBufferDepth;
411
412    ALOGVV("  Acquired buffer, timestamp %lld", queueHead.buffer.mTimestamp);
413
414    findMatchesLocked();
415
416    return OK;
417}
418
419void ZslProcessor::findMatchesLocked() {
420    ALOGVV("Scanning");
421    for (size_t i = 0; i < mZslQueue.size(); i++) {
422        ZslPair &queueEntry = mZslQueue.editItemAt(i);
423        nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
424        IF_ALOGV() {
425            camera_metadata_entry_t entry;
426            nsecs_t frameTimestamp = 0;
427            if (!queueEntry.frame.isEmpty()) {
428                entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
429                frameTimestamp = entry.data.i64[0];
430            }
431            ALOGVV("   %d: b: %lld\tf: %lld", i,
432                    bufferTimestamp, frameTimestamp );
433        }
434        if (queueEntry.frame.isEmpty() && bufferTimestamp != 0) {
435            // Have buffer, no matching frame. Look for one
436            for (size_t j = 0; j < mFrameList.size(); j++) {
437                bool match = false;
438                CameraMetadata &frame = mFrameList.editItemAt(j);
439                if (!frame.isEmpty()) {
440                    camera_metadata_entry_t entry;
441                    entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
442                    if (entry.count == 0) {
443                        ALOGE("%s: Can't find timestamp in frame!",
444                                __FUNCTION__);
445                        continue;
446                    }
447                    nsecs_t frameTimestamp = entry.data.i64[0];
448                    if (bufferTimestamp == frameTimestamp) {
449                        ALOGVV("%s: Found match %lld", __FUNCTION__,
450                                frameTimestamp);
451                        match = true;
452                    } else {
453                        int64_t delta = abs(bufferTimestamp - frameTimestamp);
454                        if ( delta < 1000000) {
455                            ALOGVV("%s: Found close match %lld (delta %lld)",
456                                    __FUNCTION__, bufferTimestamp, delta);
457                            match = true;
458                        }
459                    }
460                }
461                if (match) {
462                    queueEntry.frame.acquire(frame);
463                    break;
464                }
465            }
466        }
467    }
468}
469
470void ZslProcessor::dumpZslQueue(int fd) const {
471    String8 header("ZSL queue contents:");
472    String8 indent("    ");
473    ALOGV("%s", header.string());
474    if (fd != -1) {
475        header = indent + header + "\n";
476        write(fd, header.string(), header.size());
477    }
478    for (size_t i = 0; i < mZslQueue.size(); i++) {
479        const ZslPair &queueEntry = mZslQueue[i];
480        nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
481        camera_metadata_ro_entry_t entry;
482        nsecs_t frameTimestamp = 0;
483        int frameAeState = -1;
484        if (!queueEntry.frame.isEmpty()) {
485            entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
486            if (entry.count > 0) frameTimestamp = entry.data.i64[0];
487            entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE);
488            if (entry.count > 0) frameAeState = entry.data.u8[0];
489        }
490        String8 result =
491                String8::format("   %d: b: %lld\tf: %lld, AE state: %d", i,
492                        bufferTimestamp, frameTimestamp, frameAeState);
493        ALOGV("%s", result.string());
494        if (fd != -1) {
495            result = indent + result + "\n";
496            write(fd, result.string(), result.size());
497        }
498
499    }
500}
501
502}; // namespace camera2
503}; // namespace android
504