ZslProcessor.cpp revision c20630569431234db23b6182dd17102023dee68e 
1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "Camera2-ZslProcessor"
18#define ATRACE_TAG ATRACE_TAG_CAMERA
19//#define LOG_NDEBUG 0
20//#define LOG_NNDEBUG 0
21
22#ifdef LOG_NNDEBUG
23#define ALOGVV(...) ALOGV(__VA_ARGS__)
24#else
25#define ALOGVV(...) ((void)0)
26#endif
27
28#include <utils/Log.h>
29#include <utils/Trace.h>
30
31#include "ZslProcessor.h"
32#include <gui/SurfaceTextureClient.h>
33#include "../Camera2Device.h"
34#include "../Camera2Client.h"
35
36
37namespace android {
38namespace camera2 {
39
40ZslProcessor::ZslProcessor(
41    wp<Camera2Client> client,
42    wp<CaptureSequencer> sequencer):
43        Thread(false),
44        mState(RUNNING),
45        mClient(client),
46        mSequencer(sequencer),
47        mZslBufferAvailable(false),
48        mZslStreamId(NO_STREAM),
49        mZslReprocessStreamId(NO_STREAM),
50        mFrameListHead(0),
51        mZslQueueHead(0),
52        mZslQueueTail(0) {
53    mZslQueue.insertAt(0, kZslBufferDepth);
54    mFrameList.insertAt(0, kFrameListDepth);
55    sp<CaptureSequencer> captureSequencer = mSequencer.promote();
56    if (captureSequencer != 0) captureSequencer->setZslProcessor(this);
57}
58
59ZslProcessor::~ZslProcessor() {
60    ALOGV("%s: Exit", __FUNCTION__);
61    deleteStream();
62}
63
64void ZslProcessor::onFrameAvailable() {
65    Mutex::Autolock l(mInputMutex);
66    if (!mZslBufferAvailable) {
67        mZslBufferAvailable = true;
68        mZslBufferAvailableSignal.signal();
69    }
70}
71
72void ZslProcessor::onFrameAvailable(int32_t frameId, CameraMetadata &frame) {
73    Mutex::Autolock l(mInputMutex);
74    camera_metadata_entry_t entry;
75    entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
76    nsecs_t timestamp = entry.data.i64[0];
77    ALOGVV("Got preview frame for timestamp %lld", timestamp);
78
79    if (mState != RUNNING) return;
80
81    mFrameList.editItemAt(mFrameListHead).acquire(frame);
82    mFrameListHead = (mFrameListHead + 1) % kFrameListDepth;
83
84    findMatchesLocked();
85}
86
87void ZslProcessor::onBufferReleased(buffer_handle_t *handle) {
88    Mutex::Autolock l(mInputMutex);
89
90    // Verify that the buffer is in our queue
91    size_t i = 0;
92    for (; i < mZslQueue.size(); i++) {
93        if (&(mZslQueue[i].buffer.mGraphicBuffer->handle) == handle) break;
94    }
95    if (i == mZslQueue.size()) {
96        ALOGW("%s: Released buffer %p not found in queue",
97                __FUNCTION__, handle);
98    }
99
100    // Erase entire ZSL queue since we've now completed the capture and preview
101    // is stopped.
102    clearZslQueueLocked();
103
104    mState = RUNNING;
105}
106
107status_t ZslProcessor::updateStream(const Parameters &params) {
108    ATRACE_CALL();
109    ALOGV("%s: Configuring ZSL streams", __FUNCTION__);
110    status_t res;
111
112    Mutex::Autolock l(mInputMutex);
113
114    sp<Camera2Client> client = mClient.promote();
115    if (client == 0) return OK;
116    sp<Camera2Device> device = client->getCameraDevice();
117
118    if (mZslConsumer == 0) {
119        // Create CPU buffer queue endpoint
120        mZslConsumer = new BufferItemConsumer(
121            GRALLOC_USAGE_HW_CAMERA_ZSL,
122            kZslBufferDepth,
123            true);
124        mZslConsumer->setFrameAvailableListener(this);
125        mZslConsumer->setName(String8("Camera2Client::ZslConsumer"));
126        mZslWindow = new SurfaceTextureClient(
127            mZslConsumer->getProducerInterface());
128    }
129
130    if (mZslStreamId != NO_STREAM) {
131        // Check if stream parameters have to change
132        uint32_t currentWidth, currentHeight;
133        res = device->getStreamInfo(mZslStreamId,
134                &currentWidth, &currentHeight, 0);
135        if (res != OK) {
136            ALOGE("%s: Camera %d: Error querying capture output stream info: "
137                    "%s (%d)", __FUNCTION__,
138                    client->getCameraId(), strerror(-res), res);
139            return res;
140        }
141        if (currentWidth != (uint32_t)params.fastInfo.arrayWidth ||
142                currentHeight != (uint32_t)params.fastInfo.arrayHeight) {
143            res = device->deleteReprocessStream(mZslReprocessStreamId);
144            if (res != OK) {
145                ALOGE("%s: Camera %d: Unable to delete old reprocess stream "
146                        "for ZSL: %s (%d)", __FUNCTION__,
147                        client->getCameraId(), strerror(-res), res);
148                return res;
149            }
150            ALOGV("%s: Camera %d: Deleting stream %d since the buffer dimensions changed",
151                __FUNCTION__, client->getCameraId(), mZslStreamId);
152            res = device->deleteStream(mZslStreamId);
153            if (res != OK) {
154                ALOGE("%s: Camera %d: Unable to delete old output stream "
155                        "for ZSL: %s (%d)", __FUNCTION__,
156                        client->getCameraId(), strerror(-res), res);
157                return res;
158            }
159            mZslStreamId = NO_STREAM;
160        }
161    }
162
163    if (mZslStreamId == NO_STREAM) {
164        // Create stream for HAL production
165        // TODO: Sort out better way to select resolution for ZSL
166        int streamType = params.quirks.useZslFormat ?
167                (int)CAMERA2_HAL_PIXEL_FORMAT_ZSL :
168                (int)HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
169        res = device->createStream(mZslWindow,
170                params.fastInfo.arrayWidth, params.fastInfo.arrayHeight,
171                streamType, 0,
172                &mZslStreamId);
173        if (res != OK) {
174            ALOGE("%s: Camera %d: Can't create output stream for ZSL: "
175                    "%s (%d)", __FUNCTION__, client->getCameraId(),
176                    strerror(-res), res);
177            return res;
178        }
179        res = device->createReprocessStreamFromStream(mZslStreamId,
180                &mZslReprocessStreamId);
181        if (res != OK) {
182            ALOGE("%s: Camera %d: Can't create reprocess stream for ZSL: "
183                    "%s (%d)", __FUNCTION__, client->getCameraId(),
184                    strerror(-res), res);
185            return res;
186        }
187    }
188    client->registerFrameListener(Camera2Client::kPreviewRequestId, this);
189
190    return OK;
191}
192
193status_t ZslProcessor::deleteStream() {
194    ATRACE_CALL();
195    status_t res;
196
197    Mutex::Autolock l(mInputMutex);
198
199    if (mZslStreamId != NO_STREAM) {
200        sp<Camera2Client> client = mClient.promote();
201        if (client == 0) return OK;
202        sp<Camera2Device> device = client->getCameraDevice();
203
204        res = device->deleteReprocessStream(mZslReprocessStreamId);
205        if (res != OK) {
206            ALOGE("%s: Camera %d: Cannot delete ZSL reprocessing stream %d: "
207                    "%s (%d)", __FUNCTION__, client->getCameraId(),
208                    mZslReprocessStreamId, strerror(-res), res);
209            return res;
210        }
211
212        mZslReprocessStreamId = NO_STREAM;
213        res = device->deleteStream(mZslStreamId);
214        if (res != OK) {
215            ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: "
216                    "%s (%d)", __FUNCTION__, client->getCameraId(),
217                    mZslStreamId, strerror(-res), res);
218            return res;
219        }
220
221        mZslWindow.clear();
222        mZslConsumer.clear();
223
224        mZslStreamId = NO_STREAM;
225    }
226    return OK;
227}
228
229int ZslProcessor::getStreamId() const {
230    Mutex::Autolock l(mInputMutex);
231    return mZslStreamId;
232}
233
234int ZslProcessor::getReprocessStreamId() const {
235    Mutex::Autolock l(mInputMutex);
236    return mZslReprocessStreamId;
237}
238
239status_t ZslProcessor::pushToReprocess(int32_t requestId) {
240    ALOGV("%s: Send in reprocess request with id %d",
241            __FUNCTION__, requestId);
242    Mutex::Autolock l(mInputMutex);
243    status_t res;
244    sp<Camera2Client> client = mClient.promote();
245
246    if (client == 0) return INVALID_OPERATION;
247
248    IF_ALOGV() {
249        dumpZslQueue(-1);
250    }
251
252    if (mZslQueueTail != mZslQueueHead) {
253        CameraMetadata request;
254        size_t index = mZslQueueTail;
255        while (request.isEmpty() && index != mZslQueueHead) {
256            request = mZslQueue[index].frame;
257            index = (index + 1) % kZslBufferDepth;
258        }
259        if (request.isEmpty()) {
260            ALOGV("%s: ZSL queue has no valid frames to send yet.",
261                  __FUNCTION__);
262            return NOT_ENOUGH_DATA;
263        }
264        // Verify that the frame is reasonable for reprocessing
265
266        camera_metadata_entry_t entry;
267        entry = request.find(ANDROID_CONTROL_AE_STATE);
268        if (entry.count == 0) {
269            ALOGE("%s: ZSL queue frame has no AE state field!",
270                    __FUNCTION__);
271            return BAD_VALUE;
272        }
273        if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
274                entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
275            ALOGV("%s: ZSL queue frame AE state is %d, need full capture",
276                    __FUNCTION__, entry.data.u8[0]);
277            return NOT_ENOUGH_DATA;
278        }
279
280        buffer_handle_t *handle =
281            &(mZslQueue[index].buffer.mGraphicBuffer->handle);
282
283        uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS;
284        res = request.update(ANDROID_REQUEST_TYPE,
285                &requestType, 1);
286        uint8_t inputStreams[1] = { mZslReprocessStreamId };
287        if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS,
288                inputStreams, 1);
289        uint8_t outputStreams[1] = { client->getCaptureStreamId() };
290        if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS,
291                outputStreams, 1);
292        res = request.update(ANDROID_REQUEST_ID,
293                &requestId, 1);
294
295        if (res != OK ) {
296            ALOGE("%s: Unable to update frame to a reprocess request", __FUNCTION__);
297            return INVALID_OPERATION;
298        }
299
300        res = client->getCameraDevice()->clearStreamingRequest();
301        if (res != OK) {
302            ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: "
303                "%s (%d)",
304                __FUNCTION__, client->getCameraId(), strerror(-res), res);
305            return INVALID_OPERATION;
306        }
307        // TODO: have push-and-clear be atomic
308        res = client->getCameraDevice()->pushReprocessBuffer(mZslReprocessStreamId,
309                handle, this);
310        if (res != OK) {
311            ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)",
312                    __FUNCTION__, strerror(-res), res);
313            return res;
314        }
315
316        res = client->getCameraDevice()->capture(request);
317        if (res != OK ) {
318            ALOGE("%s: Unable to send ZSL reprocess request to capture: %s (%d)",
319                    __FUNCTION__, strerror(-res), res);
320            return res;
321        }
322
323        mState = LOCKED;
324    } else {
325        ALOGV("%s: No ZSL buffers yet", __FUNCTION__);
326        return NOT_ENOUGH_DATA;
327    }
328    return OK;
329}
330
331status_t ZslProcessor::clearZslQueue() {
332    Mutex::Autolock l(mInputMutex);
333    // If in middle of capture, can't clear out queue
334    if (mState == LOCKED) return OK;
335
336    return clearZslQueueLocked();
337}
338
339status_t ZslProcessor::clearZslQueueLocked() {
340    for (size_t i = 0; i < mZslQueue.size(); i++) {
341        if (mZslQueue[i].buffer.mTimestamp != 0) {
342            mZslConsumer->releaseBuffer(mZslQueue[i].buffer);
343        }
344        mZslQueue.replaceAt(i);
345    }
346    mZslQueueHead = 0;
347    mZslQueueTail = 0;
348    return OK;
349}
350
351void ZslProcessor::dump(int fd, const Vector<String16>& args) const {
352    Mutex::Autolock l(mInputMutex);
353    dumpZslQueue(fd);
354}
355
356bool ZslProcessor::threadLoop() {
357    status_t res;
358
359    {
360        Mutex::Autolock l(mInputMutex);
361        while (!mZslBufferAvailable) {
362            res = mZslBufferAvailableSignal.waitRelative(mInputMutex,
363                    kWaitDuration);
364            if (res == TIMED_OUT) return true;
365        }
366        mZslBufferAvailable = false;
367    }
368
369    do {
370        sp<Camera2Client> client = mClient.promote();
371        if (client == 0) return false;
372        res = processNewZslBuffer(client);
373    } while (res == OK);
374
375    return true;
376}
377
378status_t ZslProcessor::processNewZslBuffer(sp<Camera2Client> &client) {
379    ATRACE_CALL();
380    status_t res;
381
382    ALOGVV("Trying to get next buffer");
383    BufferItemConsumer::BufferItem item;
384    res = mZslConsumer->acquireBuffer(&item);
385    if (res != OK) {
386        if (res != BufferItemConsumer::NO_BUFFER_AVAILABLE) {
387            ALOGE("%s: Camera %d: Error receiving ZSL image buffer: "
388                    "%s (%d)", __FUNCTION__,
389                    client->getCameraId(), strerror(-res), res);
390        } else {
391            ALOGVV("  No buffer");
392        }
393        return res;
394    }
395
396    Mutex::Autolock l(mInputMutex);
397
398    if (mState == LOCKED) {
399        ALOGVV("In capture, discarding new ZSL buffers");
400        mZslConsumer->releaseBuffer(item);
401        return OK;
402    }
403
404    ALOGVV("Got ZSL buffer: head: %d, tail: %d", mZslQueueHead, mZslQueueTail);
405
406    if ( (mZslQueueHead + 1) % kZslBufferDepth == mZslQueueTail) {
407        ALOGVV("Releasing oldest buffer");
408        mZslConsumer->releaseBuffer(mZslQueue[mZslQueueTail].buffer);
409        mZslQueue.replaceAt(mZslQueueTail);
410        mZslQueueTail = (mZslQueueTail + 1) % kZslBufferDepth;
411    }
412
413    ZslPair &queueHead = mZslQueue.editItemAt(mZslQueueHead);
414
415    queueHead.buffer = item;
416    queueHead.frame.release();
417
418    mZslQueueHead = (mZslQueueHead + 1) % kZslBufferDepth;
419
420    ALOGVV("  Acquired buffer, timestamp %lld", queueHead.buffer.mTimestamp);
421
422    findMatchesLocked();
423
424    return OK;
425}
426
427void ZslProcessor::findMatchesLocked() {
428    ALOGVV("Scanning");
429    for (size_t i = 0; i < mZslQueue.size(); i++) {
430        ZslPair &queueEntry = mZslQueue.editItemAt(i);
431        nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
432        IF_ALOGV() {
433            camera_metadata_entry_t entry;
434            nsecs_t frameTimestamp = 0;
435            if (!queueEntry.frame.isEmpty()) {
436                entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
437                frameTimestamp = entry.data.i64[0];
438            }
439            ALOGVV("   %d: b: %lld\tf: %lld", i,
440                    bufferTimestamp, frameTimestamp );
441        }
442        if (queueEntry.frame.isEmpty() && bufferTimestamp != 0) {
443            // Have buffer, no matching frame. Look for one
444            for (size_t j = 0; j < mFrameList.size(); j++) {
445                bool match = false;
446                CameraMetadata &frame = mFrameList.editItemAt(j);
447                if (!frame.isEmpty()) {
448                    camera_metadata_entry_t entry;
449                    entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
450                    if (entry.count == 0) {
451                        ALOGE("%s: Can't find timestamp in frame!",
452                                __FUNCTION__);
453                        continue;
454                    }
455                    nsecs_t frameTimestamp = entry.data.i64[0];
456                    if (bufferTimestamp == frameTimestamp) {
457                        ALOGVV("%s: Found match %lld", __FUNCTION__,
458                                frameTimestamp);
459                        match = true;
460                    } else {
461                        int64_t delta = abs(bufferTimestamp - frameTimestamp);
462                        if ( delta < 1000000) {
463                            ALOGVV("%s: Found close match %lld (delta %lld)",
464                                    __FUNCTION__, bufferTimestamp, delta);
465                            match = true;
466                        }
467                    }
468                }
469                if (match) {
470                    queueEntry.frame.acquire(frame);
471                    break;
472                }
473            }
474        }
475    }
476}
477
478void ZslProcessor::dumpZslQueue(int fd) const {
479    String8 header("ZSL queue contents:");
480    String8 indent("    ");
481    ALOGV("%s", header.string());
482    if (fd != -1) {
483        header = indent + header + "\n";
484        write(fd, header.string(), header.size());
485    }
486    for (size_t i = 0; i < mZslQueue.size(); i++) {
487        const ZslPair &queueEntry = mZslQueue[i];
488        nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
489        camera_metadata_ro_entry_t entry;
490        nsecs_t frameTimestamp = 0;
491        int frameAeState = -1;
492        if (!queueEntry.frame.isEmpty()) {
493            entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
494            if (entry.count > 0) frameTimestamp = entry.data.i64[0];
495            entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE);
496            if (entry.count > 0) frameAeState = entry.data.u8[0];
497        }
498        String8 result =
499                String8::format("   %d: b: %lld\tf: %lld, AE state: %d", i,
500                        bufferTimestamp, frameTimestamp, frameAeState);
501        ALOGV("%s", result.string());
502        if (fd != -1) {
503            result = indent + result + "\n";
504            write(fd, result.string(), result.size());
505        }
506
507    }
508}
509
510}; // namespace camera2
511}; // namespace android
512