android_media_ImageReader.cpp revision 5dacbef0448c20718725376d6d3d30d6a1983d3f 
1/*
2 * Copyright 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "ImageReader_JNI"
19#include "android_media_Utils.h"
20#include <cutils/atomic.h>
21#include <utils/Log.h>
22#include <utils/misc.h>
23#include <utils/List.h>
24#include <utils/String8.h>
25
26#include <cstdio>
27
28#include <gui/BufferItemConsumer.h>
29#include <gui/Surface.h>
30#include <camera3.h>
31
32#include <android_runtime/AndroidRuntime.h>
33#include <android_runtime/android_view_Surface.h>
34#include <android_runtime/android_hardware_HardwareBuffer.h>
35#include <grallocusage/GrallocUsageConversion.h>
36
37#include <jni.h>
38#include <JNIHelp.h>
39
40#include <stdint.h>
41#include <inttypes.h>
42
43#define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID       "mNativeContext"
44#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID   "mNativeBuffer"
45#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID       "mTimestamp"
46
47#define CONSUMER_BUFFER_USAGE_UNKNOWN              0;
48// ----------------------------------------------------------------------------
49
50using namespace android;
51
52
53enum {
54    ACQUIRE_SUCCESS = 0,
55    ACQUIRE_NO_BUFFERS = 1,
56    ACQUIRE_MAX_IMAGES = 2,
57};
58
59static struct {
60    jfieldID mNativeContext;
61    jmethodID postEventFromNative;
62} gImageReaderClassInfo;
63
64static struct {
65    jfieldID mNativeBuffer;
66    jfieldID mTimestamp;
67    jfieldID mPlanes;
68} gSurfaceImageClassInfo;
69
70static struct {
71    jclass clazz;
72    jmethodID ctor;
73} gSurfacePlaneClassInfo;
74
75// Get an ID that's unique within this process.
76static int32_t createProcessUniqueId() {
77    static volatile int32_t globalCounter = 0;
78    return android_atomic_inc(&globalCounter);
79}
80
81// ----------------------------------------------------------------------------
82
83class JNIImageReaderContext : public ConsumerBase::FrameAvailableListener
84{
85public:
86    JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages);
87
88    virtual ~JNIImageReaderContext();
89
90    virtual void onFrameAvailable(const BufferItem& item);
91
92    BufferItem* getBufferItem();
93    void returnBufferItem(BufferItem* buffer);
94
95
96    void setBufferConsumer(const sp<BufferItemConsumer>& consumer) { mConsumer = consumer; }
97    BufferItemConsumer* getBufferConsumer() { return mConsumer.get(); }
98
99    void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; }
100    IGraphicBufferProducer* getProducer() { return mProducer.get(); }
101
102    void setBufferFormat(int format) { mFormat = format; }
103    int getBufferFormat() { return mFormat; }
104
105    void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; }
106    android_dataspace getBufferDataspace() { return mDataSpace; }
107
108    void setBufferWidth(int width) { mWidth = width; }
109    int getBufferWidth() { return mWidth; }
110
111    void setBufferHeight(int height) { mHeight = height; }
112    int getBufferHeight() { return mHeight; }
113
114private:
115    static JNIEnv* getJNIEnv(bool* needsDetach);
116    static void detachJNI();
117
118    List<BufferItem*> mBuffers;
119    sp<BufferItemConsumer> mConsumer;
120    sp<IGraphicBufferProducer> mProducer;
121    jobject mWeakThiz;
122    jclass mClazz;
123    int mFormat;
124    android_dataspace mDataSpace;
125    int mWidth;
126    int mHeight;
127};
128
129JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env,
130        jobject weakThiz, jclass clazz, int maxImages) :
131    mWeakThiz(env->NewGlobalRef(weakThiz)),
132    mClazz((jclass)env->NewGlobalRef(clazz)),
133    mFormat(0),
134    mDataSpace(HAL_DATASPACE_UNKNOWN),
135    mWidth(-1),
136    mHeight(-1) {
137    for (int i = 0; i < maxImages; i++) {
138        BufferItem* buffer = new BufferItem;
139        mBuffers.push_back(buffer);
140    }
141}
142
143JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) {
144    LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
145    *needsDetach = false;
146    JNIEnv* env = AndroidRuntime::getJNIEnv();
147    if (env == NULL) {
148        JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
149        JavaVM* vm = AndroidRuntime::getJavaVM();
150        int result = vm->AttachCurrentThread(&env, (void*) &args);
151        if (result != JNI_OK) {
152            ALOGE("thread attach failed: %#x", result);
153            return NULL;
154        }
155        *needsDetach = true;
156    }
157    return env;
158}
159
160void JNIImageReaderContext::detachJNI() {
161    JavaVM* vm = AndroidRuntime::getJavaVM();
162    int result = vm->DetachCurrentThread();
163    if (result != JNI_OK) {
164        ALOGE("thread detach failed: %#x", result);
165    }
166}
167
168BufferItem* JNIImageReaderContext::getBufferItem() {
169    if (mBuffers.empty()) {
170        return NULL;
171    }
172    // Return a BufferItem pointer and remove it from the list
173    List<BufferItem*>::iterator it = mBuffers.begin();
174    BufferItem* buffer = *it;
175    mBuffers.erase(it);
176    return buffer;
177}
178
179void JNIImageReaderContext::returnBufferItem(BufferItem* buffer) {
180    buffer->mGraphicBuffer = nullptr;
181    mBuffers.push_back(buffer);
182}
183
184JNIImageReaderContext::~JNIImageReaderContext() {
185    bool needsDetach = false;
186    JNIEnv* env = getJNIEnv(&needsDetach);
187    if (env != NULL) {
188        env->DeleteGlobalRef(mWeakThiz);
189        env->DeleteGlobalRef(mClazz);
190    } else {
191        ALOGW("leaking JNI object references");
192    }
193    if (needsDetach) {
194        detachJNI();
195    }
196
197    // Delete buffer items.
198    for (List<BufferItem *>::iterator it = mBuffers.begin();
199            it != mBuffers.end(); it++) {
200        delete *it;
201    }
202
203    if (mConsumer != 0) {
204        mConsumer.clear();
205    }
206}
207
208void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
209{
210    ALOGV("%s: frame available", __FUNCTION__);
211    bool needsDetach = false;
212    JNIEnv* env = getJNIEnv(&needsDetach);
213    if (env != NULL) {
214        env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
215    } else {
216        ALOGW("onFrameAvailable event will not posted");
217    }
218    if (needsDetach) {
219        detachJNI();
220    }
221}
222
223// ----------------------------------------------------------------------------
224
225extern "C" {
226
227static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz)
228{
229    JNIImageReaderContext *ctx;
230    ctx = reinterpret_cast<JNIImageReaderContext *>
231              (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext));
232    return ctx;
233}
234
235static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz)
236{
237    ALOGV("%s:", __FUNCTION__);
238    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
239    if (ctx == NULL) {
240        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
241        return NULL;
242    }
243
244    return ctx->getProducer();
245}
246
247static void ImageReader_setNativeContext(JNIEnv* env,
248        jobject thiz, sp<JNIImageReaderContext> ctx)
249{
250    ALOGV("%s:", __FUNCTION__);
251    JNIImageReaderContext* const p = ImageReader_getContext(env, thiz);
252    if (ctx != 0) {
253        ctx->incStrong((void*)ImageReader_setNativeContext);
254    }
255    if (p) {
256        p->decStrong((void*)ImageReader_setNativeContext);
257    }
258    env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext,
259            reinterpret_cast<jlong>(ctx.get()));
260}
261
262static BufferItemConsumer* ImageReader_getBufferConsumer(JNIEnv* env, jobject thiz)
263{
264    ALOGV("%s:", __FUNCTION__);
265    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
266    if (ctx == NULL) {
267        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
268        return NULL;
269    }
270
271    return ctx->getBufferConsumer();
272}
273
274static void Image_setBufferItem(JNIEnv* env, jobject thiz,
275        const BufferItem* buffer)
276{
277    env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, reinterpret_cast<jlong>(buffer));
278}
279
280static BufferItem* Image_getBufferItem(JNIEnv* env, jobject image)
281{
282    return reinterpret_cast<BufferItem*>(
283            env->GetLongField(image, gSurfaceImageClassInfo.mNativeBuffer));
284}
285
286
287// ----------------------------------------------------------------------------
288
289static void ImageReader_classInit(JNIEnv* env, jclass clazz)
290{
291    ALOGV("%s:", __FUNCTION__);
292
293    jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage");
294    LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
295                        "can't find android/graphics/ImageReader$SurfaceImage");
296    gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID(
297            imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J");
298    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL,
299                        "can't find android/graphics/ImageReader.%s",
300                        ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID);
301
302    gSurfaceImageClassInfo.mTimestamp = env->GetFieldID(
303            imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J");
304    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL,
305                        "can't find android/graphics/ImageReader.%s",
306                        ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID);
307
308    gSurfaceImageClassInfo.mPlanes = env->GetFieldID(
309            imageClazz, "mPlanes", "[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;");
310    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL,
311            "can't find android/media/ImageReader$ReaderSurfaceImage.mPlanes");
312
313    gImageReaderClassInfo.mNativeContext = env->GetFieldID(
314            clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J");
315    LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL,
316                        "can't find android/graphics/ImageReader.%s",
317                          ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID);
318
319    gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(
320            clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
321    LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL,
322                        "can't find android/graphics/ImageReader.postEventFromNative");
323
324    jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane");
325    LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
326    // FindClass only gives a local reference of jclass object.
327    gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
328    gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
329            "(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V");
330    LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
331            "Can not find SurfacePlane constructor");
332}
333
334static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height,
335                             jint format, jint maxImages, jlong ndkUsage)
336{
337    status_t res;
338    int nativeFormat;
339    android_dataspace nativeDataspace;
340
341    ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
342          __FUNCTION__, width, height, format, maxImages);
343
344    PublicFormat publicFormat = static_cast<PublicFormat>(format);
345    nativeFormat = android_view_Surface_mapPublicFormatToHalFormat(
346        publicFormat);
347    nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace(
348        publicFormat);
349
350    jclass clazz = env->GetObjectClass(thiz);
351    if (clazz == NULL) {
352        jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader");
353        return;
354    }
355    sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
356
357    sp<IGraphicBufferProducer> gbProducer;
358    sp<IGraphicBufferConsumer> gbConsumer;
359    BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
360    sp<BufferItemConsumer> bufferConsumer;
361    String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
362            width, height, format, maxImages, getpid(),
363            createProcessUniqueId());
364    uint32_t consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
365    bool needUsageOverride = ndkUsage != CONSUMER_BUFFER_USAGE_UNKNOWN;
366    uint64_t outProducerUsage = 0;
367    uint64_t outConsumerUsage = 0;
368    android_hardware_HardwareBuffer_convertToGrallocUsageBits(&outProducerUsage, &outConsumerUsage,
369            ndkUsage, 0);
370
371    if (isFormatOpaque(nativeFormat)) {
372        // Use the SW_READ_NEVER usage to tell producer that this format is not for preview or video
373        // encoding. The only possibility will be ZSL output.
374        consumerUsage = GRALLOC_USAGE_SW_READ_NEVER;
375        if (needUsageOverride) {
376            consumerUsage = android_convertGralloc1To0Usage(0, outConsumerUsage);
377        }
378    } else if (needUsageOverride) {
379        ALOGW("Consumer usage override for non-opaque format is not implemented yet, "
380                "ignore the provided usage from the application");
381    }
382    bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages,
383            /*controlledByApp*/true);
384    if (bufferConsumer == nullptr) {
385        jniThrowExceptionFmt(env, "java/lang/RuntimeException",
386                "Failed to allocate native buffer consumer for format 0x%x and usage 0x%x",
387                nativeFormat, consumerUsage);
388        return;
389    }
390    ctx->setBufferConsumer(bufferConsumer);
391    bufferConsumer->setName(consumerName);
392
393    ctx->setProducer(gbProducer);
394    bufferConsumer->setFrameAvailableListener(ctx);
395    ImageReader_setNativeContext(env, thiz, ctx);
396    ctx->setBufferFormat(nativeFormat);
397    ctx->setBufferDataspace(nativeDataspace);
398    ctx->setBufferWidth(width);
399    ctx->setBufferHeight(height);
400
401    // Set the width/height/format/dataspace to the bufferConsumer.
402    res = bufferConsumer->setDefaultBufferSize(width, height);
403    if (res != OK) {
404        jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
405                          "Failed to set buffer consumer default size (%dx%d) for format 0x%x",
406                          width, height, nativeFormat);
407        return;
408    }
409    res = bufferConsumer->setDefaultBufferFormat(nativeFormat);
410    if (res != OK) {
411        jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
412                          "Failed to set buffer consumer default format 0x%x", nativeFormat);
413    }
414    res = bufferConsumer->setDefaultBufferDataSpace(nativeDataspace);
415    if (res != OK) {
416        jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
417                          "Failed to set buffer consumer default dataSpace 0x%x", nativeDataspace);
418    }
419}
420
421static void ImageReader_close(JNIEnv* env, jobject thiz)
422{
423    ALOGV("%s:", __FUNCTION__);
424
425    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
426    if (ctx == NULL) {
427        // ImageReader is already closed.
428        return;
429    }
430
431    BufferItemConsumer* consumer = NULL;
432    consumer = ImageReader_getBufferConsumer(env, thiz);
433
434    if (consumer != NULL) {
435        consumer->abandon();
436        consumer->setFrameAvailableListener(NULL);
437    }
438    ImageReader_setNativeContext(env, thiz, NULL);
439}
440
441static sp<Fence> Image_unlockIfLocked(JNIEnv* env, jobject image) {
442    ALOGV("%s", __FUNCTION__);
443    BufferItem* buffer = Image_getBufferItem(env, image);
444    if (buffer == NULL) {
445        jniThrowException(env, "java/lang/IllegalStateException",
446                "Image is not initialized");
447        return Fence::NO_FENCE;
448    }
449
450    // Is locked?
451    bool wasBufferLocked = false;
452    jobject planes = NULL;
453    if (!isFormatOpaque(buffer->mGraphicBuffer->getPixelFormat())) {
454        planes = env->GetObjectField(image, gSurfaceImageClassInfo.mPlanes);
455    }
456    wasBufferLocked = (planes != NULL);
457    if (wasBufferLocked) {
458        status_t res = OK;
459        int fenceFd = -1;
460        if (wasBufferLocked) {
461            res = buffer->mGraphicBuffer->unlockAsync(&fenceFd);
462            if (res != OK) {
463                jniThrowRuntimeException(env, "unlock buffer failed");
464                return Fence::NO_FENCE;
465            }
466        }
467        sp<Fence> releaseFence = new Fence(fenceFd);
468        return releaseFence;
469        ALOGV("Successfully unlocked the image");
470    }
471    return Fence::NO_FENCE;
472}
473
474static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image)
475{
476    ALOGV("%s:", __FUNCTION__);
477    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
478    if (ctx == NULL) {
479        ALOGW("ImageReader#close called before Image#close, consider calling Image#close first");
480        return;
481    }
482
483    BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
484    BufferItem* buffer = Image_getBufferItem(env, image);
485    if (buffer == nullptr) {
486        // Release an already closed image is harmless.
487        return;
488    }
489
490    sp<Fence> releaseFence = Image_unlockIfLocked(env, image);
491    bufferConsumer->releaseBuffer(*buffer, releaseFence);
492    Image_setBufferItem(env, image, NULL);
493    ctx->returnBufferItem(buffer);
494    ALOGV("%s: Image (format: 0x%x) has been released", __FUNCTION__, ctx->getBufferFormat());
495}
496
497static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image) {
498    ALOGV("%s:", __FUNCTION__);
499    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
500    if (ctx == NULL) {
501        jniThrowException(env, "java/lang/IllegalStateException",
502                "ImageReader is not initialized or was already closed");
503        return -1;
504    }
505
506    BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
507    BufferItem* buffer = ctx->getBufferItem();
508    if (buffer == NULL) {
509        ALOGW("Unable to acquire a buffer item, very likely client tried to acquire more than"
510            " maxImages buffers");
511        return ACQUIRE_MAX_IMAGES;
512    }
513
514    status_t res = bufferConsumer->acquireBuffer(buffer, 0);
515    if (res != OK) {
516        ctx->returnBufferItem(buffer);
517        if (res != BufferQueue::NO_BUFFER_AVAILABLE) {
518            if (res == INVALID_OPERATION) {
519                // Max number of images were already acquired.
520                ALOGE("%s: Max number of buffers allowed are already acquired : %s (%d)",
521                        __FUNCTION__, strerror(-res), res);
522                return ACQUIRE_MAX_IMAGES;
523            } else {
524                ALOGE("%s: Acquire image failed with some unknown error: %s (%d)",
525                        __FUNCTION__, strerror(-res), res);
526                jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
527                        "Unknown error (%d) when we tried to acquire an image.",
528                                          res);
529                return ACQUIRE_NO_BUFFERS;
530            }
531        }
532        // This isn't really an error case, as the application may acquire buffer at any time.
533        return ACQUIRE_NO_BUFFERS;
534    }
535
536    // Add some extra checks for non-opaque formats.
537    if (!isFormatOpaque(ctx->getBufferFormat())) {
538        // Check if the left-top corner of the crop rect is origin, we currently assume this point is
539        // zero, will revisit this once this assumption turns out problematic.
540        Point lt = buffer->mCrop.leftTop();
541        if (lt.x != 0 || lt.y != 0) {
542            jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
543                    "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
544            return -1;
545        }
546
547        // Check if the producer buffer configurations match what ImageReader configured.
548        int outputWidth = getBufferWidth(buffer);
549        int outputHeight = getBufferHeight(buffer);
550
551        int imgReaderFmt = ctx->getBufferFormat();
552        int imageReaderWidth = ctx->getBufferWidth();
553        int imageReaderHeight = ctx->getBufferHeight();
554        int bufferFormat = buffer->mGraphicBuffer->getPixelFormat();
555        if ((bufferFormat != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) &&
556                (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) {
557            ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d",
558                    __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight);
559        }
560        if (imgReaderFmt != bufferFormat) {
561            if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 &&
562                    isPossiblyYUV(bufferFormat)) {
563                // Treat formats that are compatible with flexible YUV
564                // (HAL_PIXEL_FORMAT_YCbCr_420_888) as HAL_PIXEL_FORMAT_YCbCr_420_888.
565                ALOGV("%s: Treat buffer format to 0x%x as HAL_PIXEL_FORMAT_YCbCr_420_888",
566                        __FUNCTION__, bufferFormat);
567            } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB &&
568                    bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888) {
569                // Using HAL_PIXEL_FORMAT_RGBA_8888 Gralloc buffers containing JPEGs to get around
570                // SW write limitations for (b/17379185).
571                ALOGV("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__);
572            } else {
573                // Return the buffer to the queue. No need to provide fence, as this buffer wasn't
574                // used anywhere yet.
575                bufferConsumer->releaseBuffer(*buffer);
576                ctx->returnBufferItem(buffer);
577
578                // Throw exception
579                ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
580                        bufferFormat, ctx->getBufferFormat());
581                String8 msg;
582                msg.appendFormat("The producer output buffer format 0x%x doesn't "
583                        "match the ImageReader's configured buffer format 0x%x.",
584                        bufferFormat, ctx->getBufferFormat());
585                jniThrowException(env, "java/lang/UnsupportedOperationException",
586                        msg.string());
587                return -1;
588            }
589        }
590
591    }
592
593    // Set SurfaceImage instance member variables
594    Image_setBufferItem(env, image, buffer);
595    env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp,
596            static_cast<jlong>(buffer->mTimestamp));
597
598    return ACQUIRE_SUCCESS;
599}
600
601static jint ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) {
602    ALOGV("%s:", __FUNCTION__);
603    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
604    if (ctx == NULL) {
605        jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
606        return -1;
607    }
608
609    BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
610    BufferItem* buffer = Image_getBufferItem(env, image);
611    if (!buffer) {
612        ALOGE(
613                "Image already released and can not be detached from ImageReader!!!");
614        jniThrowException(env, "java/lang/IllegalStateException",
615                "Image detach from ImageReader failed: buffer was already released");
616        return -1;
617    }
618
619    status_t res = OK;
620    Image_unlockIfLocked(env, image);
621    res = bufferConsumer->detachBuffer(buffer->mSlot);
622    if (res != OK) {
623        ALOGE("Image detach failed: %s (%d)!!!", strerror(-res), res);
624        jniThrowRuntimeException(env,
625                "nativeDetachImage failed for image!!!");
626        return res;
627    }
628    return OK;
629}
630
631static void ImageReader_discardFreeBuffers(JNIEnv* env, jobject thiz) {
632    ALOGV("%s:", __FUNCTION__);
633    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
634    if (ctx == NULL) {
635        jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
636        return;
637    }
638
639    BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
640    status_t res = bufferConsumer->discardFreeBuffers();
641    if (res != OK) {
642        ALOGE("Buffer discard failed: %s (%d)", strerror(-res), res);
643        jniThrowRuntimeException(env,
644                "nativeDicardFreebuffers failed");
645    }
646}
647
648static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
649{
650    ALOGV("%s: ", __FUNCTION__);
651
652    IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz);
653    if (gbp == NULL) {
654        jniThrowRuntimeException(env, "Buffer consumer is uninitialized");
655        return NULL;
656    }
657
658    // Wrap the IGBP in a Java-language Surface.
659    return android_view_Surface_createFromIGraphicBufferProducer(env, gbp);
660}
661
662static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) {
663    ALOGV("%s", __FUNCTION__);
664    BufferItem* buffer = Image_getBufferItem(env, thiz);
665    if (buffer == NULL) {
666        jniThrowException(env, "java/lang/IllegalStateException",
667                "Image is not initialized");
668        return;
669    }
670
671    status_t res = lockImageFromBuffer(buffer,
672            GRALLOC_USAGE_SW_READ_OFTEN, buffer->mFence->dup(), image);
673    if (res != OK) {
674        jniThrowExceptionFmt(env, "java/lang/RuntimeException",
675                "lock buffer failed for format 0x%x",
676                buffer->mGraphicBuffer->getPixelFormat());
677        return;
678    }
679
680    // Carry over some fields from BufferItem.
681    image->crop        = buffer->mCrop;
682    image->transform   = buffer->mTransform;
683    image->scalingMode = buffer->mScalingMode;
684    image->timestamp   = buffer->mTimestamp;
685    image->dataSpace   = buffer->mDataSpace;
686    image->frameNumber = buffer->mFrameNumber;
687
688    ALOGV("%s: Successfully locked the image", __FUNCTION__);
689    // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer,
690    // and we don't set them here.
691}
692
693static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
694        int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {
695    ALOGV("%s", __FUNCTION__);
696
697    status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size,
698            pixelStride, rowStride);
699    if (res != OK) {
700        jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
701                             "Pixel format: 0x%x is unsupported", buffer->flexFormat);
702    }
703}
704
705static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
706        int numPlanes, int readerFormat)
707{
708    ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes);
709    int rowStride = 0;
710    int pixelStride = 0;
711    uint8_t *pData = NULL;
712    uint32_t dataSize = 0;
713    jobject byteBuffer = NULL;
714
715    PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat);
716    int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat(
717        publicReaderFormat);
718
719    if (isFormatOpaque(halReaderFormat) && numPlanes > 0) {
720        String8 msg;
721        msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
722                " must be 0", halReaderFormat, numPlanes);
723        jniThrowException(env, "java/lang/IllegalArgumentException", msg.string());
724        return NULL;
725    }
726
727    jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz,
728            /*initial_element*/NULL);
729    if (surfacePlanes == NULL) {
730        jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays,"
731                " probably out of memory");
732        return NULL;
733    }
734    if (isFormatOpaque(halReaderFormat)) {
735        // Return 0 element surface array.
736        return surfacePlanes;
737    }
738
739    LockedImage lockedImg = LockedImage();
740    Image_getLockedImage(env, thiz, &lockedImg);
741    if (env->ExceptionCheck()) {
742        return NULL;
743    }
744    // Create all SurfacePlanes
745    for (int i = 0; i < numPlanes; i++) {
746        Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat,
747                &pData, &dataSize, &pixelStride, &rowStride);
748        byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
749        if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
750            jniThrowException(env, "java/lang/IllegalStateException",
751                    "Failed to allocate ByteBuffer");
752            return NULL;
753        }
754
755        // Finally, create this SurfacePlane.
756        jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
757                    gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
758        env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
759    }
760
761    return surfacePlanes;
762}
763
764static jint Image_getWidth(JNIEnv* env, jobject thiz)
765{
766    BufferItem* buffer = Image_getBufferItem(env, thiz);
767    return getBufferWidth(buffer);
768}
769
770static jint Image_getHeight(JNIEnv* env, jobject thiz)
771{
772    BufferItem* buffer = Image_getBufferItem(env, thiz);
773    return getBufferHeight(buffer);
774}
775
776static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat)
777{
778    if (isFormatOpaque(readerFormat)) {
779        // Assuming opaque reader produce opaque images.
780        return static_cast<jint>(PublicFormat::PRIVATE);
781    } else {
782        BufferItem* buffer = Image_getBufferItem(env, thiz);
783        int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat(
784                static_cast<PublicFormat>(readerFormat));
785        int32_t fmt = applyFormatOverrides(
786                buffer->mGraphicBuffer->getPixelFormat(), readerHalFormat);
787        // Override the image format to HAL_PIXEL_FORMAT_YCbCr_420_888 if the actual format is
788        // NV21 or YV12. This could only happen when the Gralloc HAL version is v0.1 thus doesn't
789        // support lockycbcr(), the CpuConsumer need to use the lock() method in the
790        // lockNextBuffer() call. For Gralloc HAL v0.2 or newer, this format should already be
791        // overridden to HAL_PIXEL_FORMAT_YCbCr_420_888 for the flexible YUV compatible formats.
792        if (isPossiblyYUV(fmt)) {
793            fmt = HAL_PIXEL_FORMAT_YCbCr_420_888;
794        }
795        PublicFormat publicFmt = android_view_Surface_mapHalFormatDataspaceToPublicFormat(
796                fmt, buffer->mDataSpace);
797        return static_cast<jint>(publicFmt);
798    }
799}
800
801} // extern "C"
802
803// ----------------------------------------------------------------------------
804
805static const JNINativeMethod gImageReaderMethods[] = {
806    {"nativeClassInit",        "()V",                        (void*)ImageReader_classInit },
807    {"nativeInit",             "(Ljava/lang/Object;IIIIJ)V",  (void*)ImageReader_init },
808    {"nativeClose",            "()V",                        (void*)ImageReader_close },
809    {"nativeReleaseImage",     "(Landroid/media/Image;)V",   (void*)ImageReader_imageRelease },
810    {"nativeImageSetup",       "(Landroid/media/Image;)I",   (void*)ImageReader_imageSetup },
811    {"nativeGetSurface",       "()Landroid/view/Surface;",   (void*)ImageReader_getSurface },
812    {"nativeDetachImage",      "(Landroid/media/Image;)I",   (void*)ImageReader_detachImage },
813    {"nativeDiscardFreeBuffers", "()V",                      (void*)ImageReader_discardFreeBuffers }
814};
815
816static const JNINativeMethod gImageMethods[] = {
817    {"nativeCreatePlanes",      "(II)[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",
818                                                              (void*)Image_createSurfacePlanes },
819    {"nativeGetWidth",         "()I",                        (void*)Image_getWidth },
820    {"nativeGetHeight",        "()I",                        (void*)Image_getHeight },
821    {"nativeGetFormat",        "(I)I",                        (void*)Image_getFormat },
822};
823
824int register_android_media_ImageReader(JNIEnv *env) {
825
826    int ret1 = AndroidRuntime::registerNativeMethods(env,
827                   "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods));
828
829    int ret2 = AndroidRuntime::registerNativeMethods(env,
830                   "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods));
831
832    return (ret1 || ret2);
833}
834