1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "Camera2-ZslProcessor"
18#define ATRACE_TAG ATRACE_TAG_CAMERA
19//#define LOG_NDEBUG 0
20//#define LOG_NNDEBUG 0
21
22#ifdef LOG_NNDEBUG
23#define ALOGVV(...) ALOGV(__VA_ARGS__)
24#else
25#define ALOGVV(...) ((void)0)
26#endif
27
28#include <utils/Log.h>
29#include <utils/Trace.h>
30
31#include "ZslProcessor.h"
32#include <gui/SurfaceTextureClient.h>
33#include "../Camera2Device.h"
34#include "../Camera2Client.h"
35
36
37namespace android {
38namespace camera2 {
39
40ZslProcessor::ZslProcessor(
41    wp<Camera2Client> client,
42    wp<CaptureSequencer> sequencer):
43        Thread(false),
44        mState(RUNNING),
45        mClient(client),
46        mSequencer(sequencer),
47        mZslBufferAvailable(false),
48        mZslStreamId(NO_STREAM),
49        mZslReprocessStreamId(NO_STREAM),
50        mFrameListHead(0),
51        mZslQueueHead(0),
52        mZslQueueTail(0) {
53    mZslQueue.insertAt(0, kZslBufferDepth);
54    mFrameList.insertAt(0, kFrameListDepth);
55    sp<CaptureSequencer> captureSequencer = mSequencer.promote();
56    if (captureSequencer != 0) captureSequencer->setZslProcessor(this);
57}
58
59ZslProcessor::~ZslProcessor() {
60    ALOGV("%s: Exit", __FUNCTION__);
61    deleteStream();
62}
63
64void ZslProcessor::onFrameAvailable() {
65    Mutex::Autolock l(mInputMutex);
66    if (!mZslBufferAvailable) {
67        mZslBufferAvailable = true;
68        mZslBufferAvailableSignal.signal();
69    }
70}
71
72void ZslProcessor::onFrameAvailable(int32_t frameId, const CameraMetadata &frame) {
73    Mutex::Autolock l(mInputMutex);
74    camera_metadata_ro_entry_t entry;
75    entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
76    nsecs_t timestamp = entry.data.i64[0];
77    ALOGVV("Got preview frame for timestamp %lld", timestamp);
78
79    if (mState != RUNNING) return;
80
81    mFrameList.editItemAt(mFrameListHead) = frame;
82    mFrameListHead = (mFrameListHead + 1) % kFrameListDepth;
83
84    findMatchesLocked();
85}
86
87void ZslProcessor::onBufferReleased(buffer_handle_t *handle) {
88    Mutex::Autolock l(mInputMutex);
89
90    // Verify that the buffer is in our queue
91    size_t i = 0;
92    for (; i < mZslQueue.size(); i++) {
93        if (&(mZslQueue[i].buffer.mGraphicBuffer->handle) == handle) break;
94    }
95    if (i == mZslQueue.size()) {
96        ALOGW("%s: Released buffer %p not found in queue",
97                __FUNCTION__, handle);
98    }
99
100    // Erase entire ZSL queue since we've now completed the capture and preview
101    // is stopped.
102    clearZslQueueLocked();
103
104    mState = RUNNING;
105}
106
107status_t ZslProcessor::updateStream(const Parameters &params) {
108    ATRACE_CALL();
109    ALOGV("%s: Configuring ZSL streams", __FUNCTION__);
110    status_t res;
111
112    Mutex::Autolock l(mInputMutex);
113
114    sp<Camera2Client> client = mClient.promote();
115    if (client == 0) return OK;
116    sp<Camera2Device> device = client->getCameraDevice();
117
118    if (mZslConsumer == 0) {
119        // Create CPU buffer queue endpoint
120        mZslConsumer = new BufferItemConsumer(
121            GRALLOC_USAGE_HW_CAMERA_ZSL,
122            kZslBufferDepth,
123            true);
124        mZslConsumer->setFrameAvailableListener(this);
125        mZslConsumer->setName(String8("Camera2Client::ZslConsumer"));
126        mZslWindow = new SurfaceTextureClient(
127            mZslConsumer->getProducerInterface());
128    }
129
130    if (mZslStreamId != NO_STREAM) {
131        // Check if stream parameters have to change
132        uint32_t currentWidth, currentHeight;
133        res = device->getStreamInfo(mZslStreamId,
134                &currentWidth, &currentHeight, 0);
135        if (res != OK) {
136            ALOGE("%s: Camera %d: Error querying capture output stream info: "
137                    "%s (%d)", __FUNCTION__,
138                    client->getCameraId(), strerror(-res), res);
139            return res;
140        }
141        if (currentWidth != (uint32_t)params.fastInfo.arrayWidth ||
142                currentHeight != (uint32_t)params.fastInfo.arrayHeight) {
143            res = device->deleteReprocessStream(mZslReprocessStreamId);
144            if (res != OK) {
145                ALOGE("%s: Camera %d: Unable to delete old reprocess stream "
146                        "for ZSL: %s (%d)", __FUNCTION__,
147                        client->getCameraId(), strerror(-res), res);
148                return res;
149            }
150            ALOGV("%s: Camera %d: Deleting stream %d since the buffer dimensions changed",
151                __FUNCTION__, client->getCameraId(), mZslStreamId);
152            res = device->deleteStream(mZslStreamId);
153            if (res != OK) {
154                ALOGE("%s: Camera %d: Unable to delete old output stream "
155                        "for ZSL: %s (%d)", __FUNCTION__,
156                        client->getCameraId(), strerror(-res), res);
157                return res;
158            }
159            mZslStreamId = NO_STREAM;
160        }
161    }
162
163    if (mZslStreamId == NO_STREAM) {
164        // Create stream for HAL production
165        // TODO: Sort out better way to select resolution for ZSL
166        int streamType = params.quirks.useZslFormat ?
167                (int)CAMERA2_HAL_PIXEL_FORMAT_ZSL :
168                (int)HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
169        res = device->createStream(mZslWindow,
170                params.fastInfo.arrayWidth, params.fastInfo.arrayHeight,
171                streamType, 0,
172                &mZslStreamId);
173        if (res != OK) {
174            ALOGE("%s: Camera %d: Can't create output stream for ZSL: "
175                    "%s (%d)", __FUNCTION__, client->getCameraId(),
176                    strerror(-res), res);
177            return res;
178        }
179        res = device->createReprocessStreamFromStream(mZslStreamId,
180                &mZslReprocessStreamId);
181        if (res != OK) {
182            ALOGE("%s: Camera %d: Can't create reprocess stream for ZSL: "
183                    "%s (%d)", __FUNCTION__, client->getCameraId(),
184                    strerror(-res), res);
185            return res;
186        }
187    }
188    client->registerFrameListener(Camera2Client::kPreviewRequestIdStart,
189            Camera2Client::kPreviewRequestIdEnd,
190            this);
191
192    return OK;
193}
194
195status_t ZslProcessor::deleteStream() {
196    ATRACE_CALL();
197    status_t res;
198
199    Mutex::Autolock l(mInputMutex);
200
201    if (mZslStreamId != NO_STREAM) {
202        sp<Camera2Client> client = mClient.promote();
203        if (client == 0) return OK;
204        sp<Camera2Device> device = client->getCameraDevice();
205
206        res = device->deleteReprocessStream(mZslReprocessStreamId);
207        if (res != OK) {
208            ALOGE("%s: Camera %d: Cannot delete ZSL reprocessing stream %d: "
209                    "%s (%d)", __FUNCTION__, client->getCameraId(),
210                    mZslReprocessStreamId, strerror(-res), res);
211            return res;
212        }
213
214        mZslReprocessStreamId = NO_STREAM;
215        res = device->deleteStream(mZslStreamId);
216        if (res != OK) {
217            ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: "
218                    "%s (%d)", __FUNCTION__, client->getCameraId(),
219                    mZslStreamId, strerror(-res), res);
220            return res;
221        }
222
223        mZslWindow.clear();
224        mZslConsumer.clear();
225
226        mZslStreamId = NO_STREAM;
227    }
228    return OK;
229}
230
231int ZslProcessor::getStreamId() const {
232    Mutex::Autolock l(mInputMutex);
233    return mZslStreamId;
234}
235
236int ZslProcessor::getReprocessStreamId() const {
237    Mutex::Autolock l(mInputMutex);
238    return mZslReprocessStreamId;
239}
240
241status_t ZslProcessor::pushToReprocess(int32_t requestId) {
242    ALOGV("%s: Send in reprocess request with id %d",
243            __FUNCTION__, requestId);
244    Mutex::Autolock l(mInputMutex);
245    status_t res;
246    sp<Camera2Client> client = mClient.promote();
247
248    if (client == 0) return INVALID_OPERATION;
249
250    IF_ALOGV() {
251        dumpZslQueue(-1);
252    }
253
254    if (mZslQueueTail != mZslQueueHead) {
255        CameraMetadata request;
256        size_t index = mZslQueueTail;
257        while (index != mZslQueueHead) {
258            if (!mZslQueue[index].frame.isEmpty()) {
259                request = mZslQueue[index].frame;
260                break;
261            }
262            index = (index + 1) % kZslBufferDepth;
263        }
264        if (index == mZslQueueHead) {
265            ALOGV("%s: ZSL queue has no valid frames to send yet.",
266                  __FUNCTION__);
267            return NOT_ENOUGH_DATA;
268        }
269        // Verify that the frame is reasonable for reprocessing
270
271        camera_metadata_entry_t entry;
272        entry = request.find(ANDROID_CONTROL_AE_STATE);
273        if (entry.count == 0) {
274            ALOGE("%s: ZSL queue frame has no AE state field!",
275                    __FUNCTION__);
276            return BAD_VALUE;
277        }
278        if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
279                entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
280            ALOGV("%s: ZSL queue frame AE state is %d, need full capture",
281                    __FUNCTION__, entry.data.u8[0]);
282            return NOT_ENOUGH_DATA;
283        }
284
285        buffer_handle_t *handle =
286            &(mZslQueue[index].buffer.mGraphicBuffer->handle);
287
288        uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS;
289        res = request.update(ANDROID_REQUEST_TYPE,
290                &requestType, 1);
291        uint8_t inputStreams[1] = { mZslReprocessStreamId };
292        if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS,
293                inputStreams, 1);
294        uint8_t outputStreams[1] = { client->getCaptureStreamId() };
295        if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS,
296                outputStreams, 1);
297        res = request.update(ANDROID_REQUEST_ID,
298                &requestId, 1);
299
300        if (res != OK ) {
301            ALOGE("%s: Unable to update frame to a reprocess request", __FUNCTION__);
302            return INVALID_OPERATION;
303        }
304
305        res = client->stopStream();
306        if (res != OK) {
307            ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: "
308                "%s (%d)",
309                __FUNCTION__, client->getCameraId(), strerror(-res), res);
310            return INVALID_OPERATION;
311        }
312        // TODO: have push-and-clear be atomic
313        res = client->getCameraDevice()->pushReprocessBuffer(mZslReprocessStreamId,
314                handle, this);
315        if (res != OK) {
316            ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)",
317                    __FUNCTION__, strerror(-res), res);
318            return res;
319        }
320
321        // Update JPEG settings
322        {
323            SharedParameters::Lock l(client->getParameters());
324            res = l.mParameters.updateRequestJpeg(&request);
325            if (res != OK) {
326                ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL "
327                        "capture request: %s (%d)", __FUNCTION__,
328                        client->getCameraId(),
329                        strerror(-res), res);
330                return res;
331            }
332        }
333
334        mLatestCapturedRequest = request;
335        res = client->getCameraDevice()->capture(request);
336        if (res != OK ) {
337            ALOGE("%s: Unable to send ZSL reprocess request to capture: %s (%d)",
338                    __FUNCTION__, strerror(-res), res);
339            return res;
340        }
341
342        mState = LOCKED;
343    } else {
344        ALOGV("%s: No ZSL buffers yet", __FUNCTION__);
345        return NOT_ENOUGH_DATA;
346    }
347    return OK;
348}
349
350status_t ZslProcessor::clearZslQueue() {
351    Mutex::Autolock l(mInputMutex);
352    // If in middle of capture, can't clear out queue
353    if (mState == LOCKED) return OK;
354
355    return clearZslQueueLocked();
356}
357
358status_t ZslProcessor::clearZslQueueLocked() {
359    for (size_t i = 0; i < mZslQueue.size(); i++) {
360        if (mZslQueue[i].buffer.mTimestamp != 0) {
361            mZslConsumer->releaseBuffer(mZslQueue[i].buffer);
362        }
363        mZslQueue.replaceAt(i);
364    }
365    mZslQueueHead = 0;
366    mZslQueueTail = 0;
367    return OK;
368}
369
370void ZslProcessor::dump(int fd, const Vector<String16>& args) const {
371    Mutex::Autolock l(mInputMutex);
372    if (!mLatestCapturedRequest.isEmpty()) {
373        String8 result("    Latest ZSL capture request:\n");
374        write(fd, result.string(), result.size());
375        mLatestCapturedRequest.dump(fd, 2, 6);
376    } else {
377        String8 result("    Latest ZSL capture request: none yet\n");
378        write(fd, result.string(), result.size());
379    }
380    dumpZslQueue(fd);
381}
382
383bool ZslProcessor::threadLoop() {
384    status_t res;
385
386    {
387        Mutex::Autolock l(mInputMutex);
388        while (!mZslBufferAvailable) {
389            res = mZslBufferAvailableSignal.waitRelative(mInputMutex,
390                    kWaitDuration);
391            if (res == TIMED_OUT) return true;
392        }
393        mZslBufferAvailable = false;
394    }
395
396    do {
397        sp<Camera2Client> client = mClient.promote();
398        if (client == 0) return false;
399        res = processNewZslBuffer(client);
400    } while (res == OK);
401
402    return true;
403}
404
405status_t ZslProcessor::processNewZslBuffer(sp<Camera2Client> &client) {
406    ATRACE_CALL();
407    status_t res;
408
409    ALOGVV("Trying to get next buffer");
410    BufferItemConsumer::BufferItem item;
411    res = mZslConsumer->acquireBuffer(&item);
412    if (res != OK) {
413        if (res != BufferItemConsumer::NO_BUFFER_AVAILABLE) {
414            ALOGE("%s: Camera %d: Error receiving ZSL image buffer: "
415                    "%s (%d)", __FUNCTION__,
416                    client->getCameraId(), strerror(-res), res);
417        } else {
418            ALOGVV("  No buffer");
419        }
420        return res;
421    }
422
423    Mutex::Autolock l(mInputMutex);
424
425    if (mState == LOCKED) {
426        ALOGVV("In capture, discarding new ZSL buffers");
427        mZslConsumer->releaseBuffer(item);
428        return OK;
429    }
430
431    ALOGVV("Got ZSL buffer: head: %d, tail: %d", mZslQueueHead, mZslQueueTail);
432
433    if ( (mZslQueueHead + 1) % kZslBufferDepth == mZslQueueTail) {
434        ALOGVV("Releasing oldest buffer");
435        mZslConsumer->releaseBuffer(mZslQueue[mZslQueueTail].buffer);
436        mZslQueue.replaceAt(mZslQueueTail);
437        mZslQueueTail = (mZslQueueTail + 1) % kZslBufferDepth;
438    }
439
440    ZslPair &queueHead = mZslQueue.editItemAt(mZslQueueHead);
441
442    queueHead.buffer = item;
443    queueHead.frame.release();
444
445    mZslQueueHead = (mZslQueueHead + 1) % kZslBufferDepth;
446
447    ALOGVV("  Acquired buffer, timestamp %lld", queueHead.buffer.mTimestamp);
448
449    findMatchesLocked();
450
451    return OK;
452}
453
454void ZslProcessor::findMatchesLocked() {
455    ALOGVV("Scanning");
456    for (size_t i = 0; i < mZslQueue.size(); i++) {
457        ZslPair &queueEntry = mZslQueue.editItemAt(i);
458        nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
459        IF_ALOGV() {
460            camera_metadata_entry_t entry;
461            nsecs_t frameTimestamp = 0;
462            if (!queueEntry.frame.isEmpty()) {
463                entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
464                frameTimestamp = entry.data.i64[0];
465            }
466            ALOGVV("   %d: b: %lld\tf: %lld", i,
467                    bufferTimestamp, frameTimestamp );
468        }
469        if (queueEntry.frame.isEmpty() && bufferTimestamp != 0) {
470            // Have buffer, no matching frame. Look for one
471            for (size_t j = 0; j < mFrameList.size(); j++) {
472                bool match = false;
473                CameraMetadata &frame = mFrameList.editItemAt(j);
474                if (!frame.isEmpty()) {
475                    camera_metadata_entry_t entry;
476                    entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
477                    if (entry.count == 0) {
478                        ALOGE("%s: Can't find timestamp in frame!",
479                                __FUNCTION__);
480                        continue;
481                    }
482                    nsecs_t frameTimestamp = entry.data.i64[0];
483                    if (bufferTimestamp == frameTimestamp) {
484                        ALOGVV("%s: Found match %lld", __FUNCTION__,
485                                frameTimestamp);
486                        match = true;
487                    } else {
488                        int64_t delta = abs(bufferTimestamp - frameTimestamp);
489                        if ( delta < 1000000) {
490                            ALOGVV("%s: Found close match %lld (delta %lld)",
491                                    __FUNCTION__, bufferTimestamp, delta);
492                            match = true;
493                        }
494                    }
495                }
496                if (match) {
497                    queueEntry.frame.acquire(frame);
498                    break;
499                }
500            }
501        }
502    }
503}
504
505void ZslProcessor::dumpZslQueue(int fd) const {
506    String8 header("ZSL queue contents:");
507    String8 indent("    ");
508    ALOGV("%s", header.string());
509    if (fd != -1) {
510        header = indent + header + "\n";
511        write(fd, header.string(), header.size());
512    }
513    for (size_t i = 0; i < mZslQueue.size(); i++) {
514        const ZslPair &queueEntry = mZslQueue[i];
515        nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
516        camera_metadata_ro_entry_t entry;
517        nsecs_t frameTimestamp = 0;
518        int frameAeState = -1;
519        if (!queueEntry.frame.isEmpty()) {
520            entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
521            if (entry.count > 0) frameTimestamp = entry.data.i64[0];
522            entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE);
523            if (entry.count > 0) frameAeState = entry.data.u8[0];
524        }
525        String8 result =
526                String8::format("   %d: b: %lld\tf: %lld, AE state: %d", i,
527                        bufferTimestamp, frameTimestamp, frameAeState);
528        ALOGV("%s", result.string());
529        if (fd != -1) {
530            result = indent + result + "\n";
531            write(fd, result.string(), result.size());
532        }
533
534    }
535}
536
537}; // namespace camera2
538}; // namespace android
539