NdkImageReader.cpp revision 9cd007e2aaf7004387e8df005bc4d688b78bcc50 
1/*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include <inttypes.h>
18
19//#define LOG_NDEBUG 0
20#define LOG_TAG "NdkImageReader"
21
22#include "NdkImagePriv.h"
23#include "NdkImageReaderPriv.h"
24
25#include <utils/Log.h>
26#include <android_runtime/android_view_Surface.h>
27
28using namespace android;
29
30namespace {
31    // Get an ID that's unique within this process.
32    static int32_t createProcessUniqueId() {
33        static volatile int32_t globalCounter = 0;
34        return android_atomic_inc(&globalCounter);
35    }
36}
37
38const char* AImageReader::kCallbackFpKey = "Callback";
39const char* AImageReader::kContextKey    = "Context";
40
41bool
42AImageReader::isSupportedFormat(int32_t format) {
43    switch (format) {
44        case AIMAGE_FORMAT_RGBA_8888:
45        case AIMAGE_FORMAT_RGBX_8888:
46        case AIMAGE_FORMAT_RGB_888:
47        case AIMAGE_FORMAT_RGB_565:
48        case AIMAGE_FORMAT_RGBA_FP16:
49        case AIMAGE_FORMAT_YUV_420_888:
50        case AIMAGE_FORMAT_JPEG:
51        case AIMAGE_FORMAT_RAW16:
52        case AIMAGE_FORMAT_RAW_PRIVATE:
53        case AIMAGE_FORMAT_RAW10:
54        case AIMAGE_FORMAT_RAW12:
55        case AIMAGE_FORMAT_DEPTH16:
56        case AIMAGE_FORMAT_DEPTH_POINT_CLOUD:
57            return true;
58        default:
59            return false;
60    }
61}
62
63int
64AImageReader::getNumPlanesForFormat(int32_t format) {
65    switch (format) {
66        case AIMAGE_FORMAT_YUV_420_888:
67            return 3;
68        case AIMAGE_FORMAT_RGBA_8888:
69        case AIMAGE_FORMAT_RGBX_8888:
70        case AIMAGE_FORMAT_RGB_888:
71        case AIMAGE_FORMAT_RGB_565:
72        case AIMAGE_FORMAT_RGBA_FP16:
73        case AIMAGE_FORMAT_JPEG:
74        case AIMAGE_FORMAT_RAW16:
75        case AIMAGE_FORMAT_RAW_PRIVATE:
76        case AIMAGE_FORMAT_RAW10:
77        case AIMAGE_FORMAT_RAW12:
78        case AIMAGE_FORMAT_DEPTH16:
79        case AIMAGE_FORMAT_DEPTH_POINT_CLOUD:
80            return 1;
81        default:
82            return -1;
83    }
84}
85
86void
87AImageReader::FrameListener::onFrameAvailable(const BufferItem& /*item*/) {
88    Mutex::Autolock _l(mLock);
89    sp<AImageReader> reader = mReader.promote();
90    if (reader == nullptr) {
91        ALOGW("A frame is available after AImageReader closed!");
92        return; // reader has been closed
93    }
94    if (mListener.onImageAvailable == nullptr) {
95        return; // No callback registered
96    }
97
98    sp<AMessage> msg = new AMessage(AImageReader::kWhatImageAvailable, reader->mHandler);
99    msg->setPointer(AImageReader::kCallbackFpKey, (void *) mListener.onImageAvailable);
100    msg->setPointer(AImageReader::kContextKey, mListener.context);
101    msg->post();
102}
103
104media_status_t
105AImageReader::FrameListener::setImageListener(AImageReader_ImageListener* listener) {
106    Mutex::Autolock _l(mLock);
107    if (listener == nullptr) {
108        mListener.context = nullptr;
109        mListener.onImageAvailable = nullptr;
110    } else {
111        mListener = *listener;
112    }
113    return AMEDIA_OK;
114}
115
116media_status_t
117AImageReader::setImageListenerLocked(AImageReader_ImageListener* listener) {
118    return mFrameListener->setImageListener(listener);
119}
120
121media_status_t
122AImageReader::setImageListener(AImageReader_ImageListener* listener) {
123    Mutex::Autolock _l(mLock);
124    return setImageListenerLocked(listener);
125}
126
127void AImageReader::CallbackHandler::onMessageReceived(
128        const sp<AMessage> &msg) {
129    switch (msg->what()) {
130        case kWhatImageAvailable:
131        {
132            AImageReader_ImageCallback onImageAvailable;
133            void* context;
134            bool found = msg->findPointer(kCallbackFpKey, (void**) &onImageAvailable);
135            if (!found || onImageAvailable == nullptr) {
136                ALOGE("%s: Cannot find onImageAvailable callback fp!", __FUNCTION__);
137                return;
138            }
139            found = msg->findPointer(kContextKey, &context);
140            if (!found) {
141                ALOGE("%s: Cannot find callback context!", __FUNCTION__);
142                return;
143            }
144            (*onImageAvailable)(context, mReader);
145            break;
146        }
147        default:
148            ALOGE("%s: unknown message type %d", __FUNCTION__, msg->what());
149            break;
150    }
151}
152
153AImageReader::AImageReader(int32_t width, int32_t height, int32_t format, int32_t maxImages) :
154        mWidth(width), mHeight(height), mFormat(format), mMaxImages(maxImages),
155        mNumPlanes(getNumPlanesForFormat(format)),
156        mFrameListener(new FrameListener(this)) {}
157
158media_status_t
159AImageReader::init() {
160    PublicFormat publicFormat = static_cast<PublicFormat>(mFormat);
161    mHalFormat = android_view_Surface_mapPublicFormatToHalFormat(publicFormat);
162    mHalDataSpace = android_view_Surface_mapPublicFormatToHalDataspace(publicFormat);
163
164    sp<IGraphicBufferProducer> gbProducer;
165    sp<IGraphicBufferConsumer> gbConsumer;
166    BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
167
168    sp<CpuConsumer> cpuConsumer;
169    String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
170            mWidth, mHeight, mFormat, mMaxImages, getpid(),
171            createProcessUniqueId());
172
173    cpuConsumer = new CpuConsumer(gbConsumer, mMaxImages, /*controlledByApp*/true);
174    if (cpuConsumer == nullptr) {
175        ALOGE("Failed to allocate CpuConsumer");
176        return AMEDIA_ERROR_UNKNOWN;
177    }
178
179    mCpuConsumer = cpuConsumer;
180    mCpuConsumer->setName(consumerName);
181    mProducer = gbProducer;
182
183    sp<ConsumerBase> consumer = cpuConsumer;
184    consumer->setFrameAvailableListener(mFrameListener);
185
186    status_t res;
187    res = cpuConsumer->setDefaultBufferSize(mWidth, mHeight);
188    if (res != OK) {
189        ALOGE("Failed to set CpuConsumer buffer size");
190        return AMEDIA_ERROR_UNKNOWN;
191    }
192    res = cpuConsumer->setDefaultBufferFormat(mHalFormat);
193    if (res != OK) {
194        ALOGE("Failed to set CpuConsumer buffer format");
195        return AMEDIA_ERROR_UNKNOWN;
196    }
197    res = cpuConsumer->setDefaultBufferDataSpace(mHalDataSpace);
198    if (res != OK) {
199        ALOGE("Failed to set CpuConsumer buffer dataSpace");
200        return AMEDIA_ERROR_UNKNOWN;
201    }
202
203    mSurface = new Surface(mProducer, /*controlledByApp*/true);
204    if (mSurface == nullptr) {
205        ALOGE("Failed to create surface");
206        return AMEDIA_ERROR_UNKNOWN;
207    }
208    mWindow = static_cast<ANativeWindow*>(mSurface.get());
209
210    for (int i = 0; i < mMaxImages; i++) {
211        CpuConsumer::LockedBuffer* buffer = new CpuConsumer::LockedBuffer;
212        mBuffers.push_back(buffer);
213    }
214
215    mCbLooper = new ALooper;
216    mCbLooper->setName(consumerName.string());
217    res = mCbLooper->start(
218            /*runOnCallingThread*/false,
219            /*canCallJava*/       true,
220            PRIORITY_DEFAULT);
221    if (res != OK) {
222        ALOGE("Failed to start the looper");
223        return AMEDIA_ERROR_UNKNOWN;
224    }
225    mHandler = new CallbackHandler(this);
226    mCbLooper->registerHandler(mHandler);
227
228    return AMEDIA_OK;
229}
230
231AImageReader::~AImageReader() {
232    Mutex::Autolock _l(mLock);
233    AImageReader_ImageListener nullListener = {nullptr, nullptr};
234    setImageListenerLocked(&nullListener);
235
236    if (mCbLooper != nullptr) {
237        mCbLooper->unregisterHandler(mHandler->id());
238        mCbLooper->stop();
239    }
240    mCbLooper.clear();
241    mHandler.clear();
242
243    // Close all previously acquired images
244    for (auto it = mAcquiredImages.begin();
245              it != mAcquiredImages.end(); it++) {
246        AImage* image = *it;
247        image->close();
248    }
249
250    // Delete LockedBuffers
251    for (auto it = mBuffers.begin();
252              it != mBuffers.end(); it++) {
253        delete *it;
254    }
255
256    if (mCpuConsumer != nullptr) {
257        mCpuConsumer->abandon();
258        mCpuConsumer->setFrameAvailableListener(nullptr);
259    }
260}
261
262media_status_t
263AImageReader::acquireCpuConsumerImageLocked(/*out*/AImage** image) {
264    *image = nullptr;
265    CpuConsumer::LockedBuffer* buffer = getLockedBufferLocked();
266    if (buffer == nullptr) {
267        ALOGW("Unable to acquire a lockedBuffer, very likely client tries to lock more than"
268            " maxImages buffers");
269        return AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED;
270    }
271
272    status_t res = mCpuConsumer->lockNextBuffer(buffer);
273    if (res != NO_ERROR) {
274        returnLockedBufferLocked(buffer);
275        if (res != BAD_VALUE /*no buffers*/) {
276            if (res == NOT_ENOUGH_DATA) {
277                return AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED;
278            } else {
279                ALOGE("%s Fail to lockNextBuffer with error: %d ",
280                      __FUNCTION__, res);
281                return AMEDIA_ERROR_UNKNOWN;
282            }
283        }
284        return AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE;
285    }
286
287    if (buffer->flexFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
288        ALOGE("NV21 format is not supported by AImageReader");
289        return AMEDIA_ERROR_UNSUPPORTED;
290    }
291
292    // Check if the left-top corner of the crop rect is origin, we currently assume this point is
293    // zero, will revist this once this assumption turns out problematic.
294    Point lt = buffer->crop.leftTop();
295    if (lt.x != 0 || lt.y != 0) {
296        ALOGE("crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
297        return AMEDIA_ERROR_UNKNOWN;
298    }
299
300    // Check if the producer buffer configurations match what ImageReader configured.
301    int outputWidth = getBufferWidth(buffer);
302    int outputHeight = getBufferHeight(buffer);
303
304    int readerFmt = mHalFormat;
305    int readerWidth = mWidth;
306    int readerHeight = mHeight;
307
308    if ((buffer->format != HAL_PIXEL_FORMAT_BLOB) && (readerFmt != HAL_PIXEL_FORMAT_BLOB) &&
309            (readerWidth != outputWidth || readerHeight != outputHeight)) {
310        ALOGW("%s: Producer buffer size: %dx%d, doesn't match AImageReader configured size: %dx%d",
311                __FUNCTION__, outputWidth, outputHeight, readerWidth, readerHeight);
312    }
313
314    int bufFmt = buffer->format;
315    if (readerFmt == HAL_PIXEL_FORMAT_YCbCr_420_888) {
316        bufFmt = buffer->flexFormat;
317    }
318
319    if (readerFmt != bufFmt) {
320        if (readerFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && (bufFmt ==
321                HAL_PIXEL_FORMAT_YCrCb_420_SP || bufFmt == HAL_PIXEL_FORMAT_YV12)) {
322            // Special casing for when producer switches to a format compatible with flexible YUV
323            // (HAL_PIXEL_FORMAT_YCbCr_420_888).
324            mHalFormat = bufFmt;
325            ALOGD("%s: Overriding buffer format YUV_420_888 to %x.", __FUNCTION__, bufFmt);
326        } else {
327            // Return the buffer to the queue.
328            mCpuConsumer->unlockBuffer(*buffer);
329            returnLockedBufferLocked(buffer);
330
331            ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
332                    buffer->format, readerFmt);
333
334            return AMEDIA_ERROR_UNKNOWN;
335        }
336    }
337
338    if (mHalFormat == HAL_PIXEL_FORMAT_BLOB) {
339        *image = new AImage(this, mFormat, buffer, buffer->timestamp,
340                            readerWidth, readerHeight, mNumPlanes);
341    } else {
342        *image = new AImage(this, mFormat, buffer, buffer->timestamp,
343                            outputWidth, outputHeight, mNumPlanes);
344    }
345    mAcquiredImages.push_back(*image);
346    return AMEDIA_OK;
347}
348
349CpuConsumer::LockedBuffer*
350AImageReader::getLockedBufferLocked() {
351    if (mBuffers.empty()) {
352        return nullptr;
353    }
354    // Return a LockedBuffer pointer and remove it from the list
355    auto it = mBuffers.begin();
356    CpuConsumer::LockedBuffer* buffer = *it;
357    mBuffers.erase(it);
358    return buffer;
359}
360
361void
362AImageReader::returnLockedBufferLocked(CpuConsumer::LockedBuffer* buffer) {
363    mBuffers.push_back(buffer);
364}
365
366void
367AImageReader::releaseImageLocked(AImage* image) {
368    CpuConsumer::LockedBuffer* buffer = image->mBuffer;
369    if (buffer == nullptr) {
370        // This should not happen, but is not fatal
371        ALOGW("AImage %p has no buffer!", image);
372        return;
373    }
374
375    mCpuConsumer->unlockBuffer(*buffer);
376    returnLockedBufferLocked(buffer);
377    image->mBuffer = nullptr;
378
379    bool found = false;
380    // cleanup acquired image list
381    for (auto it = mAcquiredImages.begin();
382              it != mAcquiredImages.end(); it++) {
383        AImage* readerCopy = *it;
384        if (readerCopy == image) {
385            found = true;
386            mAcquiredImages.erase(it);
387            break;
388        }
389    }
390    if (!found) {
391        ALOGE("Error: AImage %p is not generated by AImageReader %p",
392                image, this);
393    }
394}
395
396int
397AImageReader::getBufferWidth(CpuConsumer::LockedBuffer* buffer) {
398    if (buffer == nullptr) return -1;
399
400    if (!buffer->crop.isEmpty()) {
401        return buffer->crop.getWidth();
402    }
403    return buffer->width;
404}
405
406int
407AImageReader::getBufferHeight(CpuConsumer::LockedBuffer* buffer) {
408    if (buffer == nullptr) return -1;
409
410    if (!buffer->crop.isEmpty()) {
411        return buffer->crop.getHeight();
412    }
413    return buffer->height;
414}
415
416media_status_t
417AImageReader::acquireNextImage(/*out*/AImage** image) {
418    Mutex::Autolock _l(mLock);
419    return acquireCpuConsumerImageLocked(image);
420}
421
422media_status_t
423AImageReader::acquireLatestImage(/*out*/AImage** image) {
424    if (image == nullptr) {
425        return AMEDIA_ERROR_INVALID_PARAMETER;
426    }
427    Mutex::Autolock _l(mLock);
428    *image = nullptr;
429    AImage* prevImage = nullptr;
430    AImage* nextImage = nullptr;
431    media_status_t ret = acquireCpuConsumerImageLocked(&prevImage);
432    if (prevImage == nullptr) {
433        return ret;
434    }
435    for (;;) {
436        ret = acquireCpuConsumerImageLocked(&nextImage);
437        if (nextImage == nullptr) {
438            *image = prevImage;
439            return AMEDIA_OK;
440        }
441        prevImage->close();
442        prevImage->free();
443        prevImage = nextImage;
444        nextImage = nullptr;
445    }
446}
447
448EXPORT
449media_status_t AImageReader_new(
450        int32_t width, int32_t height, int32_t format, int32_t maxImages,
451        /*out*/AImageReader** reader) {
452    ALOGV("%s", __FUNCTION__);
453
454    if (width < 1 || height < 1) {
455        ALOGE("%s: image dimension must be positive: w:%d h:%d",
456                __FUNCTION__, width, height);
457        return AMEDIA_ERROR_INVALID_PARAMETER;
458    }
459
460    if (maxImages < 1) {
461        ALOGE("%s: max outstanding image count must be at least 1 (%d)",
462                __FUNCTION__, maxImages);
463        return AMEDIA_ERROR_INVALID_PARAMETER;
464    }
465
466    if (!AImageReader::isSupportedFormat(format)) {
467        ALOGE("%s: format %d is not supported by AImageReader",
468                __FUNCTION__, format);
469        return AMEDIA_ERROR_INVALID_PARAMETER;
470    }
471
472    if (reader == nullptr) {
473        ALOGE("%s: reader argument is null", __FUNCTION__);
474        return AMEDIA_ERROR_INVALID_PARAMETER;
475    }
476
477    //*reader = new AImageReader(width, height, format, maxImages);
478    AImageReader* tmpReader = new AImageReader(width, height, format, maxImages);
479    if (tmpReader == nullptr) {
480        ALOGE("%s: AImageReader allocation failed", __FUNCTION__);
481        return AMEDIA_ERROR_UNKNOWN;
482    }
483    media_status_t ret = tmpReader->init();
484    if (ret != AMEDIA_OK) {
485        ALOGE("%s: AImageReader initialization failed!", __FUNCTION__);
486        delete tmpReader;
487        return ret;
488    }
489    *reader = tmpReader;
490    (*reader)->incStrong((void*) AImageReader_new);
491    return AMEDIA_OK;
492}
493
494EXPORT
495void AImageReader_delete(AImageReader* reader) {
496    ALOGV("%s", __FUNCTION__);
497    if (reader != nullptr) {
498        reader->decStrong((void*) AImageReader_delete);
499    }
500    return;
501}
502
503EXPORT
504media_status_t AImageReader_getWindow(AImageReader* reader, /*out*/ANativeWindow** window) {
505    ALOGE("%s", __FUNCTION__);
506    if (reader == nullptr || window == nullptr) {
507        ALOGE("%s: invalid argument. reader %p, window %p",
508                __FUNCTION__, reader, window);
509        return AMEDIA_ERROR_INVALID_PARAMETER;
510    }
511    *window = reader->getWindow();
512    return AMEDIA_OK;
513}
514
515EXPORT
516media_status_t AImageReader_getWidth(const AImageReader* reader, /*out*/int32_t* width) {
517    ALOGV("%s", __FUNCTION__);
518    if (reader == nullptr || width == nullptr) {
519        ALOGE("%s: invalid argument. reader %p, width %p",
520                __FUNCTION__, reader, width);
521        return AMEDIA_ERROR_INVALID_PARAMETER;
522    }
523    *width = reader->getWidth();
524    return AMEDIA_OK;
525}
526
527EXPORT
528media_status_t AImageReader_getHeight(const AImageReader* reader, /*out*/int32_t* height) {
529    ALOGV("%s", __FUNCTION__);
530    if (reader == nullptr || height == nullptr) {
531        ALOGE("%s: invalid argument. reader %p, height %p",
532                __FUNCTION__, reader, height);
533        return AMEDIA_ERROR_INVALID_PARAMETER;
534    }
535    *height = reader->getHeight();
536    return AMEDIA_OK;
537}
538
539EXPORT
540media_status_t AImageReader_getFormat(const AImageReader* reader, /*out*/int32_t* format) {
541    ALOGV("%s", __FUNCTION__);
542    if (reader == nullptr || format == nullptr) {
543        ALOGE("%s: invalid argument. reader %p, format %p",
544                __FUNCTION__, reader, format);
545        return AMEDIA_ERROR_INVALID_PARAMETER;
546    }
547    *format = reader->getFormat();
548    return AMEDIA_OK;
549}
550
551EXPORT
552media_status_t AImageReader_getMaxImages(const AImageReader* reader, /*out*/int32_t* maxImages) {
553    ALOGV("%s", __FUNCTION__);
554    if (reader == nullptr || maxImages == nullptr) {
555        ALOGE("%s: invalid argument. reader %p, maxImages %p",
556                __FUNCTION__, reader, maxImages);
557        return AMEDIA_ERROR_INVALID_PARAMETER;
558    }
559    *maxImages = reader->getMaxImages();
560    return AMEDIA_OK;
561}
562
563EXPORT
564media_status_t AImageReader_acquireNextImage(AImageReader* reader, /*out*/AImage** image) {
565    ALOGV("%s", __FUNCTION__);
566    if (reader == nullptr || image == nullptr) {
567        ALOGE("%s: invalid argument. reader %p, maxImages %p",
568                __FUNCTION__, reader, image);
569        return AMEDIA_ERROR_INVALID_PARAMETER;
570    }
571    return reader->acquireNextImage(image);
572}
573
574EXPORT
575media_status_t AImageReader_acquireLatestImage(AImageReader* reader, /*out*/AImage** image) {
576    ALOGV("%s", __FUNCTION__);
577    if (reader == nullptr || image == nullptr) {
578        ALOGE("%s: invalid argument. reader %p, maxImages %p",
579                __FUNCTION__, reader, image);
580        return AMEDIA_ERROR_INVALID_PARAMETER;
581    }
582    return reader->acquireLatestImage(image);
583}
584
585EXPORT
586media_status_t AImageReader_setImageListener(
587        AImageReader* reader, AImageReader_ImageListener* listener) {
588    ALOGV("%s", __FUNCTION__);
589    if (reader == nullptr) {
590        ALOGE("%s: invalid argument! reader %p", __FUNCTION__, reader);
591        return AMEDIA_ERROR_INVALID_PARAMETER;
592    }
593
594    reader->setImageListener(listener);
595    return AMEDIA_OK;
596}
597