1/*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include <inttypes.h>
18
19//#define LOG_NDEBUG 0
20#define LOG_TAG "NdkImageReader"
21
22#include "NdkImagePriv.h"
23#include "NdkImageReaderPriv.h"
24
25#include <utils/Log.h>
26#include <android_runtime/android_view_Surface.h>
27
28using namespace android;
29
30namespace {
31    // Get an ID that's unique within this process.
32    static int32_t createProcessUniqueId() {
33        static volatile int32_t globalCounter = 0;
34        return android_atomic_inc(&globalCounter);
35    }
36}
37
38const char* AImageReader::kCallbackFpKey = "Callback";
39const char* AImageReader::kContextKey    = "Context";
40
41bool
42AImageReader::isSupportedFormat(int32_t format) {
43    switch (format) {
44        case AIMAGE_FORMAT_YUV_420_888:
45        case AIMAGE_FORMAT_JPEG:
46        case AIMAGE_FORMAT_RAW16:
47        case AIMAGE_FORMAT_RAW_PRIVATE:
48        case AIMAGE_FORMAT_RAW10:
49        case AIMAGE_FORMAT_RAW12:
50        case AIMAGE_FORMAT_DEPTH16:
51        case AIMAGE_FORMAT_DEPTH_POINT_CLOUD:
52            return true;
53        default:
54            return false;
55    }
56}
57
58int
59AImageReader::getNumPlanesForFormat(int32_t format) {
60    switch (format) {
61        case AIMAGE_FORMAT_YUV_420_888:
62            return 3;
63        case AIMAGE_FORMAT_JPEG:
64        case AIMAGE_FORMAT_RAW16:
65        case AIMAGE_FORMAT_RAW_PRIVATE:
66        case AIMAGE_FORMAT_RAW10:
67        case AIMAGE_FORMAT_RAW12:
68        case AIMAGE_FORMAT_DEPTH16:
69        case AIMAGE_FORMAT_DEPTH_POINT_CLOUD:
70            return 1;
71        default:
72            return -1;
73    }
74}
75
76void
77AImageReader::FrameListener::onFrameAvailable(const BufferItem& /*item*/) {
78    Mutex::Autolock _l(mLock);
79    sp<AImageReader> reader = mReader.promote();
80    if (reader == nullptr) {
81        ALOGW("A frame is available after AImageReader closed!");
82        return; // reader has been closed
83    }
84    if (mListener.onImageAvailable == nullptr) {
85        return; // No callback registered
86    }
87
88    sp<AMessage> msg = new AMessage(AImageReader::kWhatImageAvailable, reader->mHandler);
89    msg->setPointer(AImageReader::kCallbackFpKey, (void *) mListener.onImageAvailable);
90    msg->setPointer(AImageReader::kContextKey, mListener.context);
91    msg->post();
92}
93
94media_status_t
95AImageReader::FrameListener::setImageListener(AImageReader_ImageListener* listener) {
96    Mutex::Autolock _l(mLock);
97    if (listener == nullptr) {
98        mListener.context = nullptr;
99        mListener.onImageAvailable = nullptr;
100    } else {
101        mListener = *listener;
102    }
103    return AMEDIA_OK;
104}
105
106media_status_t
107AImageReader::setImageListenerLocked(AImageReader_ImageListener* listener) {
108    return mFrameListener->setImageListener(listener);
109}
110
111media_status_t
112AImageReader::setImageListener(AImageReader_ImageListener* listener) {
113    Mutex::Autolock _l(mLock);
114    return setImageListenerLocked(listener);
115}
116
117void AImageReader::CallbackHandler::onMessageReceived(
118        const sp<AMessage> &msg) {
119    switch (msg->what()) {
120        case kWhatImageAvailable:
121        {
122            AImageReader_ImageCallback onImageAvailable;
123            void* context;
124            bool found = msg->findPointer(kCallbackFpKey, (void**) &onImageAvailable);
125            if (!found || onImageAvailable == nullptr) {
126                ALOGE("%s: Cannot find onImageAvailable callback fp!", __FUNCTION__);
127                return;
128            }
129            found = msg->findPointer(kContextKey, &context);
130            if (!found) {
131                ALOGE("%s: Cannot find callback context!", __FUNCTION__);
132                return;
133            }
134            (*onImageAvailable)(context, mReader);
135            break;
136        }
137        default:
138            ALOGE("%s: unknown message type %d", __FUNCTION__, msg->what());
139            break;
140    }
141}
142
143AImageReader::AImageReader(int32_t width, int32_t height, int32_t format, int32_t maxImages) :
144        mWidth(width), mHeight(height), mFormat(format), mMaxImages(maxImages),
145        mNumPlanes(getNumPlanesForFormat(format)),
146        mFrameListener(new FrameListener(this)) {}
147
148media_status_t
149AImageReader::init() {
150    PublicFormat publicFormat = static_cast<PublicFormat>(mFormat);
151    mHalFormat = android_view_Surface_mapPublicFormatToHalFormat(publicFormat);
152    mHalDataSpace = android_view_Surface_mapPublicFormatToHalDataspace(publicFormat);
153
154    sp<IGraphicBufferProducer> gbProducer;
155    sp<IGraphicBufferConsumer> gbConsumer;
156    BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
157
158    sp<CpuConsumer> cpuConsumer;
159    String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
160            mWidth, mHeight, mFormat, mMaxImages, getpid(),
161            createProcessUniqueId());
162
163    cpuConsumer = new CpuConsumer(gbConsumer, mMaxImages, /*controlledByApp*/true);
164    if (cpuConsumer == nullptr) {
165        ALOGE("Failed to allocate CpuConsumer");
166        return AMEDIA_ERROR_UNKNOWN;
167    }
168
169    mCpuConsumer = cpuConsumer;
170    mCpuConsumer->setName(consumerName);
171    mProducer = gbProducer;
172
173    sp<ConsumerBase> consumer = cpuConsumer;
174    consumer->setFrameAvailableListener(mFrameListener);
175
176    status_t res;
177    res = cpuConsumer->setDefaultBufferSize(mWidth, mHeight);
178    if (res != OK) {
179        ALOGE("Failed to set CpuConsumer buffer size");
180        return AMEDIA_ERROR_UNKNOWN;
181    }
182    res = cpuConsumer->setDefaultBufferFormat(mHalFormat);
183    if (res != OK) {
184        ALOGE("Failed to set CpuConsumer buffer format");
185        return AMEDIA_ERROR_UNKNOWN;
186    }
187    res = cpuConsumer->setDefaultBufferDataSpace(mHalDataSpace);
188    if (res != OK) {
189        ALOGE("Failed to set CpuConsumer buffer dataSpace");
190        return AMEDIA_ERROR_UNKNOWN;
191    }
192
193    mSurface = new Surface(mProducer, /*controlledByApp*/true);
194    if (mSurface == nullptr) {
195        ALOGE("Failed to create surface");
196        return AMEDIA_ERROR_UNKNOWN;
197    }
198    mWindow = static_cast<ANativeWindow*>(mSurface.get());
199
200    for (int i = 0; i < mMaxImages; i++) {
201        CpuConsumer::LockedBuffer* buffer = new CpuConsumer::LockedBuffer;
202        mBuffers.push_back(buffer);
203    }
204
205    mCbLooper = new ALooper;
206    mCbLooper->setName(consumerName.string());
207    res = mCbLooper->start(
208            /*runOnCallingThread*/false,
209            /*canCallJava*/       true,
210            PRIORITY_DEFAULT);
211    if (res != OK) {
212        ALOGE("Failed to start the looper");
213        return AMEDIA_ERROR_UNKNOWN;
214    }
215    mHandler = new CallbackHandler(this);
216    mCbLooper->registerHandler(mHandler);
217
218    return AMEDIA_OK;
219}
220
221AImageReader::~AImageReader() {
222    Mutex::Autolock _l(mLock);
223    AImageReader_ImageListener nullListener = {nullptr, nullptr};
224    setImageListenerLocked(&nullListener);
225
226    if (mCbLooper != nullptr) {
227        mCbLooper->unregisterHandler(mHandler->id());
228        mCbLooper->stop();
229    }
230    mCbLooper.clear();
231    mHandler.clear();
232
233    // Close all previously acquired images
234    for (auto it = mAcquiredImages.begin();
235              it != mAcquiredImages.end(); it++) {
236        AImage* image = *it;
237        image->close();
238    }
239
240    // Delete LockedBuffers
241    for (auto it = mBuffers.begin();
242              it != mBuffers.end(); it++) {
243        delete *it;
244    }
245
246    if (mCpuConsumer != nullptr) {
247        mCpuConsumer->abandon();
248        mCpuConsumer->setFrameAvailableListener(nullptr);
249    }
250}
251
252media_status_t
253AImageReader::acquireCpuConsumerImageLocked(/*out*/AImage** image) {
254    *image = nullptr;
255    CpuConsumer::LockedBuffer* buffer = getLockedBufferLocked();
256    if (buffer == nullptr) {
257        ALOGW("Unable to acquire a lockedBuffer, very likely client tries to lock more than"
258            " maxImages buffers");
259        return AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED;
260    }
261
262    status_t res = mCpuConsumer->lockNextBuffer(buffer);
263    if (res != NO_ERROR) {
264        returnLockedBufferLocked(buffer);
265        if (res != BAD_VALUE /*no buffers*/) {
266            if (res == NOT_ENOUGH_DATA) {
267                return AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED;
268            } else {
269                ALOGE("%s Fail to lockNextBuffer with error: %d ",
270                      __FUNCTION__, res);
271                return AMEDIA_ERROR_UNKNOWN;
272            }
273        }
274        return AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE;
275    }
276
277    if (buffer->flexFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
278        ALOGE("NV21 format is not supported by AImageReader");
279        return AMEDIA_ERROR_UNSUPPORTED;
280    }
281
282    // Check if the left-top corner of the crop rect is origin, we currently assume this point is
283    // zero, will revist this once this assumption turns out problematic.
284    Point lt = buffer->crop.leftTop();
285    if (lt.x != 0 || lt.y != 0) {
286        ALOGE("crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
287        return AMEDIA_ERROR_UNKNOWN;
288    }
289
290    // Check if the producer buffer configurations match what ImageReader configured.
291    int outputWidth = getBufferWidth(buffer);
292    int outputHeight = getBufferHeight(buffer);
293
294    int readerFmt = mHalFormat;
295    int readerWidth = mWidth;
296    int readerHeight = mHeight;
297
298    if ((buffer->format != HAL_PIXEL_FORMAT_BLOB) && (readerFmt != HAL_PIXEL_FORMAT_BLOB) &&
299            (readerWidth != outputWidth || readerHeight != outputHeight)) {
300        ALOGW("%s: Producer buffer size: %dx%d, doesn't match AImageReader configured size: %dx%d",
301                __FUNCTION__, outputWidth, outputHeight, readerWidth, readerHeight);
302    }
303
304    int bufFmt = buffer->format;
305    if (readerFmt == HAL_PIXEL_FORMAT_YCbCr_420_888) {
306        bufFmt = buffer->flexFormat;
307    }
308
309    if (readerFmt != bufFmt) {
310        if (readerFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && (bufFmt ==
311                HAL_PIXEL_FORMAT_YCrCb_420_SP || bufFmt == HAL_PIXEL_FORMAT_YV12)) {
312            // Special casing for when producer switches to a format compatible with flexible YUV
313            // (HAL_PIXEL_FORMAT_YCbCr_420_888).
314            mHalFormat = bufFmt;
315            ALOGD("%s: Overriding buffer format YUV_420_888 to %x.", __FUNCTION__, bufFmt);
316        } else {
317            // Return the buffer to the queue.
318            mCpuConsumer->unlockBuffer(*buffer);
319            returnLockedBufferLocked(buffer);
320
321            ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
322                    buffer->format, readerFmt);
323
324            return AMEDIA_ERROR_UNKNOWN;
325        }
326    }
327
328    if (mHalFormat == HAL_PIXEL_FORMAT_BLOB) {
329        *image = new AImage(this, mFormat, buffer, buffer->timestamp,
330                            readerWidth, readerHeight, mNumPlanes);
331    } else {
332        *image = new AImage(this, mFormat, buffer, buffer->timestamp,
333                            outputWidth, outputHeight, mNumPlanes);
334    }
335    mAcquiredImages.push_back(*image);
336    return AMEDIA_OK;
337}
338
339CpuConsumer::LockedBuffer*
340AImageReader::getLockedBufferLocked() {
341    if (mBuffers.empty()) {
342        return nullptr;
343    }
344    // Return a LockedBuffer pointer and remove it from the list
345    auto it = mBuffers.begin();
346    CpuConsumer::LockedBuffer* buffer = *it;
347    mBuffers.erase(it);
348    return buffer;
349}
350
351void
352AImageReader::returnLockedBufferLocked(CpuConsumer::LockedBuffer* buffer) {
353    mBuffers.push_back(buffer);
354}
355
356void
357AImageReader::releaseImageLocked(AImage* image) {
358    CpuConsumer::LockedBuffer* buffer = image->mBuffer;
359    if (buffer == nullptr) {
360        // This should not happen, but is not fatal
361        ALOGW("AImage %p has no buffer!", image);
362        return;
363    }
364
365    mCpuConsumer->unlockBuffer(*buffer);
366    returnLockedBufferLocked(buffer);
367    image->mBuffer = nullptr;
368
369    bool found = false;
370    // cleanup acquired image list
371    for (auto it = mAcquiredImages.begin();
372              it != mAcquiredImages.end(); it++) {
373        AImage* readerCopy = *it;
374        if (readerCopy == image) {
375            found = true;
376            mAcquiredImages.erase(it);
377            break;
378        }
379    }
380    if (!found) {
381        ALOGE("Error: AImage %p is not generated by AImageReader %p",
382                image, this);
383    }
384}
385
386int
387AImageReader::getBufferWidth(CpuConsumer::LockedBuffer* buffer) {
388    if (buffer == nullptr) return -1;
389
390    if (!buffer->crop.isEmpty()) {
391        return buffer->crop.getWidth();
392    }
393    return buffer->width;
394}
395
396int
397AImageReader::getBufferHeight(CpuConsumer::LockedBuffer* buffer) {
398    if (buffer == nullptr) return -1;
399
400    if (!buffer->crop.isEmpty()) {
401        return buffer->crop.getHeight();
402    }
403    return buffer->height;
404}
405
406media_status_t
407AImageReader::acquireNextImage(/*out*/AImage** image) {
408    Mutex::Autolock _l(mLock);
409    return acquireCpuConsumerImageLocked(image);
410}
411
412media_status_t
413AImageReader::acquireLatestImage(/*out*/AImage** image) {
414    if (image == nullptr) {
415        return AMEDIA_ERROR_INVALID_PARAMETER;
416    }
417    Mutex::Autolock _l(mLock);
418    *image = nullptr;
419    AImage* prevImage = nullptr;
420    AImage* nextImage = nullptr;
421    media_status_t ret = acquireCpuConsumerImageLocked(&prevImage);
422    if (prevImage == nullptr) {
423        return ret;
424    }
425    for (;;) {
426        ret = acquireCpuConsumerImageLocked(&nextImage);
427        if (nextImage == nullptr) {
428            *image = prevImage;
429            return AMEDIA_OK;
430        }
431        prevImage->close();
432        prevImage->free();
433        prevImage = nextImage;
434        nextImage = nullptr;
435    }
436}
437
438EXPORT
439media_status_t AImageReader_new(
440        int32_t width, int32_t height, int32_t format, int32_t maxImages,
441        /*out*/AImageReader** reader) {
442    ALOGV("%s", __FUNCTION__);
443
444    if (width < 1 || height < 1) {
445        ALOGE("%s: image dimension must be positive: w:%d h:%d",
446                __FUNCTION__, width, height);
447        return AMEDIA_ERROR_INVALID_PARAMETER;
448    }
449
450    if (maxImages < 1) {
451        ALOGE("%s: max outstanding image count must be at least 1 (%d)",
452                __FUNCTION__, maxImages);
453        return AMEDIA_ERROR_INVALID_PARAMETER;
454    }
455
456    if (!AImageReader::isSupportedFormat(format)) {
457        ALOGE("%s: format %d is not supported by AImageReader",
458                __FUNCTION__, format);
459        return AMEDIA_ERROR_INVALID_PARAMETER;
460    }
461
462    if (reader == nullptr) {
463        ALOGE("%s: reader argument is null", __FUNCTION__);
464        return AMEDIA_ERROR_INVALID_PARAMETER;
465    }
466
467    //*reader = new AImageReader(width, height, format, maxImages);
468    AImageReader* tmpReader = new AImageReader(width, height, format, maxImages);
469    if (tmpReader == nullptr) {
470        ALOGE("%s: AImageReader allocation failed", __FUNCTION__);
471        return AMEDIA_ERROR_UNKNOWN;
472    }
473    media_status_t ret = tmpReader->init();
474    if (ret != AMEDIA_OK) {
475        ALOGE("%s: AImageReader initialization failed!", __FUNCTION__);
476        delete tmpReader;
477        return ret;
478    }
479    *reader = tmpReader;
480    (*reader)->incStrong((void*) AImageReader_new);
481    return AMEDIA_OK;
482}
483
484EXPORT
485void AImageReader_delete(AImageReader* reader) {
486    ALOGV("%s", __FUNCTION__);
487    if (reader != nullptr) {
488        reader->decStrong((void*) AImageReader_delete);
489    }
490    return;
491}
492
493EXPORT
494media_status_t AImageReader_getWindow(AImageReader* reader, /*out*/ANativeWindow** window) {
495    ALOGE("%s", __FUNCTION__);
496    if (reader == nullptr || window == nullptr) {
497        ALOGE("%s: invalid argument. reader %p, window %p",
498                __FUNCTION__, reader, window);
499        return AMEDIA_ERROR_INVALID_PARAMETER;
500    }
501    *window = reader->getWindow();
502    return AMEDIA_OK;
503}
504
505EXPORT
506media_status_t AImageReader_getWidth(const AImageReader* reader, /*out*/int32_t* width) {
507    ALOGV("%s", __FUNCTION__);
508    if (reader == nullptr || width == nullptr) {
509        ALOGE("%s: invalid argument. reader %p, width %p",
510                __FUNCTION__, reader, width);
511        return AMEDIA_ERROR_INVALID_PARAMETER;
512    }
513    *width = reader->getWidth();
514    return AMEDIA_OK;
515}
516
517EXPORT
518media_status_t AImageReader_getHeight(const AImageReader* reader, /*out*/int32_t* height) {
519    ALOGV("%s", __FUNCTION__);
520    if (reader == nullptr || height == nullptr) {
521        ALOGE("%s: invalid argument. reader %p, height %p",
522                __FUNCTION__, reader, height);
523        return AMEDIA_ERROR_INVALID_PARAMETER;
524    }
525    *height = reader->getHeight();
526    return AMEDIA_OK;
527}
528
529EXPORT
530media_status_t AImageReader_getFormat(const AImageReader* reader, /*out*/int32_t* format) {
531    ALOGV("%s", __FUNCTION__);
532    if (reader == nullptr || format == nullptr) {
533        ALOGE("%s: invalid argument. reader %p, format %p",
534                __FUNCTION__, reader, format);
535        return AMEDIA_ERROR_INVALID_PARAMETER;
536    }
537    *format = reader->getFormat();
538    return AMEDIA_OK;
539}
540
541EXPORT
542media_status_t AImageReader_getMaxImages(const AImageReader* reader, /*out*/int32_t* maxImages) {
543    ALOGV("%s", __FUNCTION__);
544    if (reader == nullptr || maxImages == nullptr) {
545        ALOGE("%s: invalid argument. reader %p, maxImages %p",
546                __FUNCTION__, reader, maxImages);
547        return AMEDIA_ERROR_INVALID_PARAMETER;
548    }
549    *maxImages = reader->getMaxImages();
550    return AMEDIA_OK;
551}
552
553EXPORT
554media_status_t AImageReader_acquireNextImage(AImageReader* reader, /*out*/AImage** image) {
555    ALOGV("%s", __FUNCTION__);
556    if (reader == nullptr || image == nullptr) {
557        ALOGE("%s: invalid argument. reader %p, maxImages %p",
558                __FUNCTION__, reader, image);
559        return AMEDIA_ERROR_INVALID_PARAMETER;
560    }
561    return reader->acquireNextImage(image);
562}
563
564EXPORT
565media_status_t AImageReader_acquireLatestImage(AImageReader* reader, /*out*/AImage** image) {
566    ALOGV("%s", __FUNCTION__);
567    if (reader == nullptr || image == nullptr) {
568        ALOGE("%s: invalid argument. reader %p, maxImages %p",
569                __FUNCTION__, reader, image);
570        return AMEDIA_ERROR_INVALID_PARAMETER;
571    }
572    return reader->acquireLatestImage(image);
573}
574
575EXPORT
576media_status_t AImageReader_setImageListener(
577        AImageReader* reader, AImageReader_ImageListener* listener) {
578    ALOGV("%s", __FUNCTION__);
579    if (reader == nullptr) {
580        ALOGE("%s: invalid argument! reader %p", __FUNCTION__, reader);
581        return AMEDIA_ERROR_INVALID_PARAMETER;
582    }
583
584    reader->setImageListener(listener);
585    return AMEDIA_OK;
586}
587