1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "Camera2-CallbackProcessor"
18#define ATRACE_TAG ATRACE_TAG_CAMERA
19//#define LOG_NDEBUG 0
20
21#include <utils/Log.h>
22#include <utils/Trace.h>
23#include <gui/Surface.h>
24
25#include "common/CameraDeviceBase.h"
26#include "api1/Camera2Client.h"
27#include "api1/client2/CallbackProcessor.h"
28
29#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
30
31namespace android {
32namespace camera2 {
33
34CallbackProcessor::CallbackProcessor(sp<Camera2Client> client):
35        Thread(false),
36        mClient(client),
37        mDevice(client->getCameraDevice()),
38        mId(client->getCameraId()),
39        mCallbackAvailable(false),
40        mCallbackToApp(false),
41        mCallbackStreamId(NO_STREAM) {
42}
43
44CallbackProcessor::~CallbackProcessor() {
45    ALOGV("%s: Exit", __FUNCTION__);
46    deleteStream();
47}
48
49void CallbackProcessor::onFrameAvailable(const BufferItem& /*item*/) {
50    Mutex::Autolock l(mInputMutex);
51    if (!mCallbackAvailable) {
52        mCallbackAvailable = true;
53        mCallbackAvailableSignal.signal();
54    }
55}
56
57status_t CallbackProcessor::setCallbackWindow(
58        const sp<Surface>& callbackWindow) {
59    ATRACE_CALL();
60    status_t res;
61
62    Mutex::Autolock l(mInputMutex);
63
64    sp<Camera2Client> client = mClient.promote();
65    if (client == 0) return OK;
66    sp<CameraDeviceBase> device = client->getCameraDevice();
67
68    // If the window is changing, clear out stream if it already exists
69    if (mCallbackWindow != callbackWindow && mCallbackStreamId != NO_STREAM) {
70        res = device->deleteStream(mCallbackStreamId);
71        if (res != OK) {
72            ALOGE("%s: Camera %d: Unable to delete old stream "
73                    "for callbacks: %s (%d)", __FUNCTION__,
74                    client->getCameraId(), strerror(-res), res);
75            return res;
76        }
77        mCallbackStreamId = NO_STREAM;
78        mCallbackConsumer.clear();
79    }
80    mCallbackWindow = callbackWindow;
81    mCallbackToApp = (mCallbackWindow != NULL);
82
83    return OK;
84}
85
86status_t CallbackProcessor::updateStream(const Parameters &params) {
87    ATRACE_CALL();
88    status_t res;
89
90    Mutex::Autolock l(mInputMutex);
91
92    sp<CameraDeviceBase> device = mDevice.promote();
93    if (device == 0) {
94        ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
95        return INVALID_OPERATION;
96    }
97
98    // If possible, use the flexible YUV format
99    int32_t callbackFormat = params.previewFormat;
100    if (mCallbackToApp) {
101        // TODO: etalvala: This should use the flexible YUV format as well, but
102        // need to reconcile HAL2/HAL3 requirements.
103        callbackFormat = HAL_PIXEL_FORMAT_YV12;
104    } else if(params.fastInfo.useFlexibleYuv &&
105            (params.previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
106             params.previewFormat == HAL_PIXEL_FORMAT_YV12) ) {
107        callbackFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;
108    }
109
110    if (!mCallbackToApp && mCallbackConsumer == 0) {
111        // Create CPU buffer queue endpoint, since app hasn't given us one
112        // Make it async to avoid disconnect deadlocks
113        sp<IGraphicBufferProducer> producer;
114        sp<IGraphicBufferConsumer> consumer;
115        BufferQueue::createBufferQueue(&producer, &consumer);
116        mCallbackConsumer = new CpuConsumer(consumer, kCallbackHeapCount);
117        mCallbackConsumer->setFrameAvailableListener(this);
118        mCallbackConsumer->setName(String8("Camera2-CallbackConsumer"));
119        mCallbackWindow = new Surface(producer);
120    }
121
122    if (mCallbackStreamId != NO_STREAM) {
123        // Check if stream parameters have to change
124        CameraDeviceBase::StreamInfo streamInfo;
125        res = device->getStreamInfo(mCallbackStreamId, &streamInfo);
126        if (res != OK) {
127            ALOGE("%s: Camera %d: Error querying callback output stream info: "
128                    "%s (%d)", __FUNCTION__, mId,
129                    strerror(-res), res);
130            return res;
131        }
132        if (streamInfo.width != (uint32_t)params.previewWidth ||
133                streamInfo.height != (uint32_t)params.previewHeight ||
134                !streamInfo.matchFormat((uint32_t)callbackFormat)) {
135            // Since size should only change while preview is not running,
136            // assuming that all existing use of old callback stream is
137            // completed.
138            ALOGV("%s: Camera %d: Deleting stream %d since the buffer "
139                    "parameters changed", __FUNCTION__, mId, mCallbackStreamId);
140            res = device->deleteStream(mCallbackStreamId);
141            if (res != OK) {
142                ALOGE("%s: Camera %d: Unable to delete old output stream "
143                        "for callbacks: %s (%d)", __FUNCTION__,
144                        mId, strerror(-res), res);
145                return res;
146            }
147            mCallbackStreamId = NO_STREAM;
148        }
149    }
150
151    if (mCallbackStreamId == NO_STREAM) {
152        ALOGV("Creating callback stream: %d x %d, format 0x%x, API format 0x%x",
153                params.previewWidth, params.previewHeight,
154                callbackFormat, params.previewFormat);
155        res = device->createStream(mCallbackWindow,
156                params.previewWidth, params.previewHeight, callbackFormat,
157                HAL_DATASPACE_V0_JFIF, CAMERA3_STREAM_ROTATION_0, &mCallbackStreamId);
158        if (res != OK) {
159            ALOGE("%s: Camera %d: Can't create output stream for callbacks: "
160                    "%s (%d)", __FUNCTION__, mId,
161                    strerror(-res), res);
162            return res;
163        }
164    }
165
166    return OK;
167}
168
169status_t CallbackProcessor::deleteStream() {
170    ATRACE_CALL();
171    sp<CameraDeviceBase> device;
172    status_t res;
173    {
174        Mutex::Autolock l(mInputMutex);
175
176        if (mCallbackStreamId == NO_STREAM) {
177            return OK;
178        }
179        device = mDevice.promote();
180        if (device == 0) {
181            ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
182            return INVALID_OPERATION;
183        }
184    }
185    res = device->waitUntilDrained();
186    if (res != OK) {
187        ALOGE("%s: Error waiting for HAL to drain: %s (%d)",
188                __FUNCTION__, strerror(-res), res);
189        return res;
190    }
191
192    res = device->deleteStream(mCallbackStreamId);
193    if (res != OK) {
194        ALOGE("%s: Unable to delete callback stream: %s (%d)",
195                __FUNCTION__, strerror(-res), res);
196        return res;
197    }
198
199    {
200        Mutex::Autolock l(mInputMutex);
201
202        mCallbackHeap.clear();
203        mCallbackWindow.clear();
204        mCallbackConsumer.clear();
205
206        mCallbackStreamId = NO_STREAM;
207    }
208    return OK;
209}
210
211int CallbackProcessor::getStreamId() const {
212    Mutex::Autolock l(mInputMutex);
213    return mCallbackStreamId;
214}
215
216void CallbackProcessor::dump(int /*fd*/, const Vector<String16>& /*args*/) const {
217}
218
219bool CallbackProcessor::threadLoop() {
220    status_t res;
221
222    {
223        Mutex::Autolock l(mInputMutex);
224        while (!mCallbackAvailable) {
225            res = mCallbackAvailableSignal.waitRelative(mInputMutex,
226                    kWaitDuration);
227            if (res == TIMED_OUT) return true;
228        }
229        mCallbackAvailable = false;
230    }
231
232    do {
233        sp<Camera2Client> client = mClient.promote();
234        if (client == 0) {
235            res = discardNewCallback();
236        } else {
237            res = processNewCallback(client);
238        }
239    } while (res == OK);
240
241    return true;
242}
243
244status_t CallbackProcessor::discardNewCallback() {
245    ATRACE_CALL();
246    status_t res;
247    CpuConsumer::LockedBuffer imgBuffer;
248    res = mCallbackConsumer->lockNextBuffer(&imgBuffer);
249    if (res != OK) {
250        if (res != BAD_VALUE) {
251            ALOGE("%s: Camera %d: Error receiving next callback buffer: "
252                    "%s (%d)", __FUNCTION__, mId, strerror(-res), res);
253        }
254        return res;
255    }
256    mCallbackConsumer->unlockBuffer(imgBuffer);
257    return OK;
258}
259
260status_t CallbackProcessor::processNewCallback(sp<Camera2Client> &client) {
261    ATRACE_CALL();
262    status_t res;
263
264    sp<Camera2Heap> callbackHeap;
265    bool useFlexibleYuv = false;
266    int32_t previewFormat = 0;
267    size_t heapIdx;
268
269    {
270        /* acquire SharedParameters before mMutex so we don't dead lock
271            with Camera2Client code calling into StreamingProcessor */
272        SharedParameters::Lock l(client->getParameters());
273        Mutex::Autolock m(mInputMutex);
274        CpuConsumer::LockedBuffer imgBuffer;
275        if (mCallbackStreamId == NO_STREAM) {
276            ALOGV("%s: Camera %d:No stream is available"
277                    , __FUNCTION__, mId);
278            return INVALID_OPERATION;
279        }
280
281        ALOGV("%s: Getting buffer", __FUNCTION__);
282        res = mCallbackConsumer->lockNextBuffer(&imgBuffer);
283        if (res != OK) {
284            if (res != BAD_VALUE) {
285                ALOGE("%s: Camera %d: Error receiving next callback buffer: "
286                        "%s (%d)", __FUNCTION__, mId, strerror(-res), res);
287            }
288            return res;
289        }
290        ALOGV("%s: Camera %d: Preview callback available", __FUNCTION__,
291                mId);
292
293        if ( l.mParameters.state != Parameters::PREVIEW
294                && l.mParameters.state != Parameters::RECORD
295                && l.mParameters.state != Parameters::VIDEO_SNAPSHOT) {
296            ALOGV("%s: Camera %d: No longer streaming",
297                    __FUNCTION__, mId);
298            mCallbackConsumer->unlockBuffer(imgBuffer);
299            return OK;
300        }
301
302        if (! (l.mParameters.previewCallbackFlags &
303                CAMERA_FRAME_CALLBACK_FLAG_ENABLE_MASK) ) {
304            ALOGV("%s: No longer enabled, dropping", __FUNCTION__);
305            mCallbackConsumer->unlockBuffer(imgBuffer);
306            return OK;
307        }
308        if ((l.mParameters.previewCallbackFlags &
309                        CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) &&
310                !l.mParameters.previewCallbackOneShot) {
311            ALOGV("%s: One shot mode, already sent, dropping", __FUNCTION__);
312            mCallbackConsumer->unlockBuffer(imgBuffer);
313            return OK;
314        }
315
316        if (imgBuffer.width != static_cast<uint32_t>(l.mParameters.previewWidth) ||
317                imgBuffer.height != static_cast<uint32_t>(l.mParameters.previewHeight)) {
318            ALOGW("%s: The preview size has changed to %d x %d from %d x %d, this buffer is"
319                    " no longer valid, dropping",__FUNCTION__,
320                    l.mParameters.previewWidth, l.mParameters.previewHeight,
321                    imgBuffer.width, imgBuffer.height);
322            mCallbackConsumer->unlockBuffer(imgBuffer);
323            return OK;
324        }
325
326        previewFormat = l.mParameters.previewFormat;
327        useFlexibleYuv = l.mParameters.fastInfo.useFlexibleYuv &&
328                (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
329                 previewFormat == HAL_PIXEL_FORMAT_YV12);
330
331        int32_t expectedFormat = useFlexibleYuv ?
332                HAL_PIXEL_FORMAT_YCbCr_420_888 : previewFormat;
333
334        if (imgBuffer.format != expectedFormat) {
335            ALOGE("%s: Camera %d: Unexpected format for callback: "
336                    "0x%x, expected 0x%x", __FUNCTION__, mId,
337                    imgBuffer.format, expectedFormat);
338            mCallbackConsumer->unlockBuffer(imgBuffer);
339            return INVALID_OPERATION;
340        }
341
342        // In one-shot mode, stop sending callbacks after the first one
343        if (l.mParameters.previewCallbackFlags &
344                CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) {
345            ALOGV("%s: clearing oneshot", __FUNCTION__);
346            l.mParameters.previewCallbackOneShot = false;
347        }
348
349        uint32_t destYStride = 0;
350        uint32_t destCStride = 0;
351        if (useFlexibleYuv) {
352            if (previewFormat == HAL_PIXEL_FORMAT_YV12) {
353                // Strides must align to 16 for YV12
354                destYStride = ALIGN(imgBuffer.width, 16);
355                destCStride = ALIGN(destYStride / 2, 16);
356            } else {
357                // No padding for NV21
358                ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP,
359                        "Unexpected preview format 0x%x", previewFormat);
360                destYStride = imgBuffer.width;
361                destCStride = destYStride / 2;
362            }
363        } else {
364            destYStride = imgBuffer.stride;
365            // don't care about cStride
366        }
367
368        size_t bufferSize = Camera2Client::calculateBufferSize(
369                imgBuffer.width, imgBuffer.height,
370                previewFormat, destYStride);
371        size_t currentBufferSize = (mCallbackHeap == 0) ?
372                0 : (mCallbackHeap->mHeap->getSize() / kCallbackHeapCount);
373        if (bufferSize != currentBufferSize) {
374            mCallbackHeap.clear();
375            mCallbackHeap = new Camera2Heap(bufferSize, kCallbackHeapCount,
376                    "Camera2Client::CallbackHeap");
377            if (mCallbackHeap->mHeap->getSize() == 0) {
378                ALOGE("%s: Camera %d: Unable to allocate memory for callbacks",
379                        __FUNCTION__, mId);
380                mCallbackConsumer->unlockBuffer(imgBuffer);
381                return INVALID_OPERATION;
382            }
383
384            mCallbackHeapHead = 0;
385            mCallbackHeapFree = kCallbackHeapCount;
386        }
387
388        if (mCallbackHeapFree == 0) {
389            ALOGE("%s: Camera %d: No free callback buffers, dropping frame",
390                    __FUNCTION__, mId);
391            mCallbackConsumer->unlockBuffer(imgBuffer);
392            return OK;
393        }
394
395        heapIdx = mCallbackHeapHead;
396
397        mCallbackHeapHead = (mCallbackHeapHead + 1) % kCallbackHeapCount;
398        mCallbackHeapFree--;
399
400        // TODO: Get rid of this copy by passing the gralloc queue all the way
401        // to app
402
403        ssize_t offset;
404        size_t size;
405        sp<IMemoryHeap> heap =
406                mCallbackHeap->mBuffers[heapIdx]->getMemory(&offset,
407                        &size);
408        uint8_t *data = (uint8_t*)heap->getBase() + offset;
409
410        if (!useFlexibleYuv) {
411            // Can just memcpy when HAL format matches API format
412            memcpy(data, imgBuffer.data, bufferSize);
413        } else {
414            res = convertFromFlexibleYuv(previewFormat, data, imgBuffer,
415                    destYStride, destCStride);
416            if (res != OK) {
417                ALOGE("%s: Camera %d: Can't convert between 0x%x and 0x%x formats!",
418                        __FUNCTION__, mId, imgBuffer.format, previewFormat);
419                mCallbackConsumer->unlockBuffer(imgBuffer);
420                return BAD_VALUE;
421            }
422        }
423
424        ALOGV("%s: Freeing buffer", __FUNCTION__);
425        mCallbackConsumer->unlockBuffer(imgBuffer);
426
427        // mCallbackHeap may get freed up once input mutex is released
428        callbackHeap = mCallbackHeap;
429    }
430
431    // Call outside parameter lock to allow re-entrancy from notification
432    {
433        Camera2Client::SharedCameraCallbacks::Lock
434            l(client->mSharedCameraCallbacks);
435        if (l.mRemoteCallback != 0) {
436            ALOGV("%s: Camera %d: Invoking client data callback",
437                    __FUNCTION__, mId);
438            l.mRemoteCallback->dataCallback(CAMERA_MSG_PREVIEW_FRAME,
439                    callbackHeap->mBuffers[heapIdx], NULL);
440        }
441    }
442
443    // Only increment free if we're still using the same heap
444    mCallbackHeapFree++;
445
446    ALOGV("%s: exit", __FUNCTION__);
447
448    return OK;
449}
450
451status_t CallbackProcessor::convertFromFlexibleYuv(int32_t previewFormat,
452        uint8_t *dst,
453        const CpuConsumer::LockedBuffer &src,
454        uint32_t dstYStride,
455        uint32_t dstCStride) const {
456
457    if (previewFormat != HAL_PIXEL_FORMAT_YCrCb_420_SP &&
458            previewFormat != HAL_PIXEL_FORMAT_YV12) {
459        ALOGE("%s: Camera %d: Unexpected preview format when using "
460                "flexible YUV: 0x%x", __FUNCTION__, mId, previewFormat);
461        return INVALID_OPERATION;
462    }
463
464    // Copy Y plane, adjusting for stride
465    const uint8_t *ySrc = src.data;
466    uint8_t *yDst = dst;
467    for (size_t row = 0; row < src.height; row++) {
468        memcpy(yDst, ySrc, src.width);
469        ySrc += src.stride;
470        yDst += dstYStride;
471    }
472
473    // Copy/swizzle chroma planes, 4:2:0 subsampling
474    const uint8_t *cbSrc = src.dataCb;
475    const uint8_t *crSrc = src.dataCr;
476    size_t chromaHeight = src.height / 2;
477    size_t chromaWidth = src.width / 2;
478    ssize_t chromaGap = src.chromaStride -
479            (chromaWidth * src.chromaStep);
480    size_t dstChromaGap = dstCStride - chromaWidth;
481
482    if (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
483        // Flexible YUV chroma to NV21 chroma
484        uint8_t *crcbDst = yDst;
485        // Check for shortcuts
486        if (cbSrc == crSrc + 1 && src.chromaStep == 2) {
487            ALOGV("%s: Fast NV21->NV21", __FUNCTION__);
488            // Source has semiplanar CrCb chroma layout, can copy by rows
489            for (size_t row = 0; row < chromaHeight; row++) {
490                memcpy(crcbDst, crSrc, src.width);
491                crcbDst += src.width;
492                crSrc += src.chromaStride;
493            }
494        } else {
495            ALOGV("%s: Generic->NV21", __FUNCTION__);
496            // Generic copy, always works but not very efficient
497            for (size_t row = 0; row < chromaHeight; row++) {
498                for (size_t col = 0; col < chromaWidth; col++) {
499                    *(crcbDst++) = *crSrc;
500                    *(crcbDst++) = *cbSrc;
501                    crSrc += src.chromaStep;
502                    cbSrc += src.chromaStep;
503                }
504                crSrc += chromaGap;
505                cbSrc += chromaGap;
506            }
507        }
508    } else {
509        // flexible YUV chroma to YV12 chroma
510        ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YV12,
511                "Unexpected preview format 0x%x", previewFormat);
512        uint8_t *crDst = yDst;
513        uint8_t *cbDst = yDst + chromaHeight * dstCStride;
514        if (src.chromaStep == 1) {
515            ALOGV("%s: Fast YV12->YV12", __FUNCTION__);
516            // Source has planar chroma layout, can copy by row
517            for (size_t row = 0; row < chromaHeight; row++) {
518                memcpy(crDst, crSrc, chromaWidth);
519                crDst += dstCStride;
520                crSrc += src.chromaStride;
521            }
522            for (size_t row = 0; row < chromaHeight; row++) {
523                memcpy(cbDst, cbSrc, chromaWidth);
524                cbDst += dstCStride;
525                cbSrc += src.chromaStride;
526            }
527        } else {
528            ALOGV("%s: Generic->YV12", __FUNCTION__);
529            // Generic copy, always works but not very efficient
530            for (size_t row = 0; row < chromaHeight; row++) {
531                for (size_t col = 0; col < chromaWidth; col++) {
532                    *(crDst++) = *crSrc;
533                    *(cbDst++) = *cbSrc;
534                    crSrc += src.chromaStep;
535                    cbSrc += src.chromaStep;
536                }
537                crSrc += chromaGap;
538                cbSrc += chromaGap;
539                crDst += dstChromaGap;
540                cbDst += dstChromaGap;
541            }
542        }
543    }
544
545    return OK;
546}
547
548}; // namespace camera2
549}; // namespace android
550