1/*
2 * Copyright 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "ImageReader_JNI"
19#include "android_media_Utils.h"
20#include <utils/Log.h>
21#include <utils/misc.h>
22#include <utils/List.h>
23#include <utils/String8.h>
24
25#include <cstdio>
26
27#include <gui/BufferItemConsumer.h>
28#include <gui/Surface.h>
29#include <camera3.h>
30
31#include <android_runtime/AndroidRuntime.h>
32#include <android_runtime/android_view_Surface.h>
33
34#include <jni.h>
35#include <JNIHelp.h>
36
37#include <stdint.h>
38#include <inttypes.h>
39
40#define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID       "mNativeContext"
41#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID   "mNativeBuffer"
42#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID       "mTimestamp"
43
44// ----------------------------------------------------------------------------
45
46using namespace android;
47
48
49enum {
50    ACQUIRE_SUCCESS = 0,
51    ACQUIRE_NO_BUFFERS = 1,
52    ACQUIRE_MAX_IMAGES = 2,
53};
54
55static struct {
56    jfieldID mNativeContext;
57    jmethodID postEventFromNative;
58} gImageReaderClassInfo;
59
60static struct {
61    jfieldID mNativeBuffer;
62    jfieldID mTimestamp;
63    jfieldID mPlanes;
64} gSurfaceImageClassInfo;
65
66static struct {
67    jclass clazz;
68    jmethodID ctor;
69} gSurfacePlaneClassInfo;
70
71// Get an ID that's unique within this process.
72static int32_t createProcessUniqueId() {
73    static volatile int32_t globalCounter = 0;
74    return android_atomic_inc(&globalCounter);
75}
76
77// ----------------------------------------------------------------------------
78
79class JNIImageReaderContext : public ConsumerBase::FrameAvailableListener
80{
81public:
82    JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages);
83
84    virtual ~JNIImageReaderContext();
85
86    virtual void onFrameAvailable(const BufferItem& item);
87
88    BufferItem* getBufferItem();
89    void returnBufferItem(BufferItem* buffer);
90
91
92    void setBufferConsumer(const sp<BufferItemConsumer>& consumer) { mConsumer = consumer; }
93    BufferItemConsumer* getBufferConsumer() { return mConsumer.get(); }
94
95    void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; }
96    IGraphicBufferProducer* getProducer() { return mProducer.get(); }
97
98    void setBufferFormat(int format) { mFormat = format; }
99    int getBufferFormat() { return mFormat; }
100
101    void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; }
102    android_dataspace getBufferDataspace() { return mDataSpace; }
103
104    void setBufferWidth(int width) { mWidth = width; }
105    int getBufferWidth() { return mWidth; }
106
107    void setBufferHeight(int height) { mHeight = height; }
108    int getBufferHeight() { return mHeight; }
109
110private:
111    static JNIEnv* getJNIEnv(bool* needsDetach);
112    static void detachJNI();
113
114    List<BufferItem*> mBuffers;
115    sp<BufferItemConsumer> mConsumer;
116    sp<IGraphicBufferProducer> mProducer;
117    jobject mWeakThiz;
118    jclass mClazz;
119    int mFormat;
120    android_dataspace mDataSpace;
121    int mWidth;
122    int mHeight;
123};
124
125JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env,
126        jobject weakThiz, jclass clazz, int maxImages) :
127    mWeakThiz(env->NewGlobalRef(weakThiz)),
128    mClazz((jclass)env->NewGlobalRef(clazz)),
129    mFormat(0),
130    mDataSpace(HAL_DATASPACE_UNKNOWN),
131    mWidth(-1),
132    mHeight(-1) {
133    for (int i = 0; i < maxImages; i++) {
134        BufferItem* buffer = new BufferItem;
135        mBuffers.push_back(buffer);
136    }
137}
138
139JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) {
140    LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
141    *needsDetach = false;
142    JNIEnv* env = AndroidRuntime::getJNIEnv();
143    if (env == NULL) {
144        JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
145        JavaVM* vm = AndroidRuntime::getJavaVM();
146        int result = vm->AttachCurrentThread(&env, (void*) &args);
147        if (result != JNI_OK) {
148            ALOGE("thread attach failed: %#x", result);
149            return NULL;
150        }
151        *needsDetach = true;
152    }
153    return env;
154}
155
156void JNIImageReaderContext::detachJNI() {
157    JavaVM* vm = AndroidRuntime::getJavaVM();
158    int result = vm->DetachCurrentThread();
159    if (result != JNI_OK) {
160        ALOGE("thread detach failed: %#x", result);
161    }
162}
163
164BufferItem* JNIImageReaderContext::getBufferItem() {
165    if (mBuffers.empty()) {
166        return NULL;
167    }
168    // Return a BufferItem pointer and remove it from the list
169    List<BufferItem*>::iterator it = mBuffers.begin();
170    BufferItem* buffer = *it;
171    mBuffers.erase(it);
172    return buffer;
173}
174
175void JNIImageReaderContext::returnBufferItem(BufferItem* buffer) {
176    mBuffers.push_back(buffer);
177}
178
179JNIImageReaderContext::~JNIImageReaderContext() {
180    bool needsDetach = false;
181    JNIEnv* env = getJNIEnv(&needsDetach);
182    if (env != NULL) {
183        env->DeleteGlobalRef(mWeakThiz);
184        env->DeleteGlobalRef(mClazz);
185    } else {
186        ALOGW("leaking JNI object references");
187    }
188    if (needsDetach) {
189        detachJNI();
190    }
191
192    // Delete buffer items.
193    for (List<BufferItem *>::iterator it = mBuffers.begin();
194            it != mBuffers.end(); it++) {
195        delete *it;
196    }
197
198    if (mConsumer != 0) {
199        mConsumer.clear();
200    }
201}
202
203void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
204{
205    ALOGV("%s: frame available", __FUNCTION__);
206    bool needsDetach = false;
207    JNIEnv* env = getJNIEnv(&needsDetach);
208    if (env != NULL) {
209        env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
210    } else {
211        ALOGW("onFrameAvailable event will not posted");
212    }
213    if (needsDetach) {
214        detachJNI();
215    }
216}
217
218// ----------------------------------------------------------------------------
219
220extern "C" {
221
222static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz)
223{
224    JNIImageReaderContext *ctx;
225    ctx = reinterpret_cast<JNIImageReaderContext *>
226              (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext));
227    return ctx;
228}
229
230static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz)
231{
232    ALOGV("%s:", __FUNCTION__);
233    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
234    if (ctx == NULL) {
235        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
236        return NULL;
237    }
238
239    return ctx->getProducer();
240}
241
242static void ImageReader_setNativeContext(JNIEnv* env,
243        jobject thiz, sp<JNIImageReaderContext> ctx)
244{
245    ALOGV("%s:", __FUNCTION__);
246    JNIImageReaderContext* const p = ImageReader_getContext(env, thiz);
247    if (ctx != 0) {
248        ctx->incStrong((void*)ImageReader_setNativeContext);
249    }
250    if (p) {
251        p->decStrong((void*)ImageReader_setNativeContext);
252    }
253    env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext,
254            reinterpret_cast<jlong>(ctx.get()));
255}
256
257static BufferItemConsumer* ImageReader_getBufferConsumer(JNIEnv* env, jobject thiz)
258{
259    ALOGV("%s:", __FUNCTION__);
260    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
261    if (ctx == NULL) {
262        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
263        return NULL;
264    }
265
266    return ctx->getBufferConsumer();
267}
268
269static void Image_setBufferItem(JNIEnv* env, jobject thiz,
270        const BufferItem* buffer)
271{
272    env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, reinterpret_cast<jlong>(buffer));
273}
274
275static BufferItem* Image_getBufferItem(JNIEnv* env, jobject image)
276{
277    return reinterpret_cast<BufferItem*>(
278            env->GetLongField(image, gSurfaceImageClassInfo.mNativeBuffer));
279}
280
281
282// ----------------------------------------------------------------------------
283
284static void ImageReader_classInit(JNIEnv* env, jclass clazz)
285{
286    ALOGV("%s:", __FUNCTION__);
287
288    jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage");
289    LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
290                        "can't find android/graphics/ImageReader$SurfaceImage");
291    gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID(
292            imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J");
293    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL,
294                        "can't find android/graphics/ImageReader.%s",
295                        ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID);
296
297    gSurfaceImageClassInfo.mTimestamp = env->GetFieldID(
298            imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J");
299    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL,
300                        "can't find android/graphics/ImageReader.%s",
301                        ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID);
302
303    gSurfaceImageClassInfo.mPlanes = env->GetFieldID(
304            imageClazz, "mPlanes", "[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;");
305    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL,
306            "can't find android/media/ImageReader$ReaderSurfaceImage.mPlanes");
307
308    gImageReaderClassInfo.mNativeContext = env->GetFieldID(
309            clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J");
310    LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL,
311                        "can't find android/graphics/ImageReader.%s",
312                          ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID);
313
314    gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(
315            clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
316    LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL,
317                        "can't find android/graphics/ImageReader.postEventFromNative");
318
319    jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane");
320    LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
321    // FindClass only gives a local reference of jclass object.
322    gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
323    gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
324            "(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V");
325    LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
326            "Can not find SurfacePlane constructor");
327}
328
329static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
330                             jint width, jint height, jint format, jint maxImages)
331{
332    status_t res;
333    int nativeFormat;
334    android_dataspace nativeDataspace;
335
336    ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
337          __FUNCTION__, width, height, format, maxImages);
338
339    PublicFormat publicFormat = static_cast<PublicFormat>(format);
340    nativeFormat = android_view_Surface_mapPublicFormatToHalFormat(
341        publicFormat);
342    nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace(
343        publicFormat);
344
345    jclass clazz = env->GetObjectClass(thiz);
346    if (clazz == NULL) {
347        jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader");
348        return;
349    }
350    sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
351
352    sp<IGraphicBufferProducer> gbProducer;
353    sp<IGraphicBufferConsumer> gbConsumer;
354    BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
355    sp<BufferItemConsumer> bufferConsumer;
356    String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
357            width, height, format, maxImages, getpid(),
358            createProcessUniqueId());
359    uint32_t consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
360
361    if (isFormatOpaque(nativeFormat)) {
362        // Use the SW_READ_NEVER usage to tell producer that this format is not for preview or video
363        // encoding. The only possibility will be ZSL output.
364        consumerUsage = GRALLOC_USAGE_SW_READ_NEVER;
365    }
366    bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages,
367            /*controlledByApp*/true);
368    if (bufferConsumer == nullptr) {
369        jniThrowExceptionFmt(env, "java/lang/RuntimeException",
370                "Failed to allocate native buffer consumer for format 0x%x", nativeFormat);
371        return;
372    }
373    ctx->setBufferConsumer(bufferConsumer);
374    bufferConsumer->setName(consumerName);
375
376    ctx->setProducer(gbProducer);
377    bufferConsumer->setFrameAvailableListener(ctx);
378    ImageReader_setNativeContext(env, thiz, ctx);
379    ctx->setBufferFormat(nativeFormat);
380    ctx->setBufferDataspace(nativeDataspace);
381    ctx->setBufferWidth(width);
382    ctx->setBufferHeight(height);
383
384    // Set the width/height/format/dataspace to the bufferConsumer.
385    res = bufferConsumer->setDefaultBufferSize(width, height);
386    if (res != OK) {
387        jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
388                          "Failed to set buffer consumer default size (%dx%d) for format 0x%x",
389                          width, height, nativeFormat);
390        return;
391    }
392    res = bufferConsumer->setDefaultBufferFormat(nativeFormat);
393    if (res != OK) {
394        jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
395                          "Failed to set buffer consumer default format 0x%x", nativeFormat);
396    }
397    res = bufferConsumer->setDefaultBufferDataSpace(nativeDataspace);
398    if (res != OK) {
399        jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
400                          "Failed to set buffer consumer default dataSpace 0x%x", nativeDataspace);
401    }
402}
403
404static void ImageReader_close(JNIEnv* env, jobject thiz)
405{
406    ALOGV("%s:", __FUNCTION__);
407
408    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
409    if (ctx == NULL) {
410        // ImageReader is already closed.
411        return;
412    }
413
414    BufferItemConsumer* consumer = NULL;
415    consumer = ImageReader_getBufferConsumer(env, thiz);
416
417    if (consumer != NULL) {
418        consumer->abandon();
419        consumer->setFrameAvailableListener(NULL);
420    }
421    ImageReader_setNativeContext(env, thiz, NULL);
422}
423
424static sp<Fence> Image_unlockIfLocked(JNIEnv* env, jobject image) {
425    ALOGV("%s", __FUNCTION__);
426    BufferItem* buffer = Image_getBufferItem(env, image);
427    if (buffer == NULL) {
428        jniThrowException(env, "java/lang/IllegalStateException",
429                "Image is not initialized");
430        return Fence::NO_FENCE;
431    }
432
433    // Is locked?
434    bool wasBufferLocked = false;
435    jobject planes = NULL;
436    if (!isFormatOpaque(buffer->mGraphicBuffer->getPixelFormat())) {
437        planes = env->GetObjectField(image, gSurfaceImageClassInfo.mPlanes);
438    }
439    wasBufferLocked = (planes != NULL);
440    if (wasBufferLocked) {
441        status_t res = OK;
442        int fenceFd = -1;
443        if (wasBufferLocked) {
444            res = buffer->mGraphicBuffer->unlockAsync(&fenceFd);
445            if (res != OK) {
446                jniThrowRuntimeException(env, "unlock buffer failed");
447                return Fence::NO_FENCE;
448            }
449        }
450        sp<Fence> releaseFence = new Fence(fenceFd);
451        return releaseFence;
452        ALOGV("Successfully unlocked the image");
453    }
454    return Fence::NO_FENCE;
455}
456
457static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image)
458{
459    ALOGV("%s:", __FUNCTION__);
460    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
461    if (ctx == NULL) {
462        ALOGW("ImageReader#close called before Image#close, consider calling Image#close first");
463        return;
464    }
465
466    BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
467    BufferItem* buffer = Image_getBufferItem(env, image);
468    if (buffer == nullptr) {
469        // Release an already closed image is harmless.
470        return;
471    }
472
473    sp<Fence> releaseFence = Image_unlockIfLocked(env, image);
474    bufferConsumer->releaseBuffer(*buffer, releaseFence);
475    Image_setBufferItem(env, image, NULL);
476    ctx->returnBufferItem(buffer);
477    ALOGV("%s: Image (format: 0x%x) has been released", __FUNCTION__, ctx->getBufferFormat());
478}
479
480static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image) {
481    ALOGV("%s:", __FUNCTION__);
482    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
483    if (ctx == NULL) {
484        jniThrowException(env, "java/lang/IllegalStateException",
485                "ImageReader is not initialized or was already closed");
486        return -1;
487    }
488
489    BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
490    BufferItem* buffer = ctx->getBufferItem();
491    if (buffer == NULL) {
492        ALOGW("Unable to acquire a buffer item, very likely client tried to acquire more than"
493            " maxImages buffers");
494        return ACQUIRE_MAX_IMAGES;
495    }
496
497    status_t res = bufferConsumer->acquireBuffer(buffer, 0);
498    if (res != OK) {
499        ctx->returnBufferItem(buffer);
500        if (res != BufferQueue::NO_BUFFER_AVAILABLE) {
501            if (res == INVALID_OPERATION) {
502                // Max number of images were already acquired.
503                ALOGE("%s: Max number of buffers allowed are already acquired : %s (%d)",
504                        __FUNCTION__, strerror(-res), res);
505                return ACQUIRE_MAX_IMAGES;
506            } else {
507                ALOGE("%s: Acquire image failed with some unknown error: %s (%d)",
508                        __FUNCTION__, strerror(-res), res);
509                jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
510                        "Unknown error (%d) when we tried to acquire an image.",
511                                          res);
512                return ACQUIRE_NO_BUFFERS;
513            }
514        }
515        // This isn't really an error case, as the application may acquire buffer at any time.
516        return ACQUIRE_NO_BUFFERS;
517    }
518
519    // Add some extra checks for non-opaque formats.
520    if (!isFormatOpaque(ctx->getBufferFormat())) {
521        // Check if the left-top corner of the crop rect is origin, we currently assume this point is
522        // zero, will revisit this once this assumption turns out problematic.
523        Point lt = buffer->mCrop.leftTop();
524        if (lt.x != 0 || lt.y != 0) {
525            jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
526                    "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
527            return -1;
528        }
529
530        // Check if the producer buffer configurations match what ImageReader configured.
531        int outputWidth = getBufferWidth(buffer);
532        int outputHeight = getBufferHeight(buffer);
533
534        int imgReaderFmt = ctx->getBufferFormat();
535        int imageReaderWidth = ctx->getBufferWidth();
536        int imageReaderHeight = ctx->getBufferHeight();
537        int bufferFormat = buffer->mGraphicBuffer->getPixelFormat();
538        if ((bufferFormat != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) &&
539                (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) {
540            ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d",
541                    __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight);
542        }
543        if (imgReaderFmt != bufferFormat) {
544            if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 &&
545                    isPossiblyYUV(bufferFormat)) {
546                // Treat formats that are compatible with flexible YUV
547                // (HAL_PIXEL_FORMAT_YCbCr_420_888) as HAL_PIXEL_FORMAT_YCbCr_420_888.
548                ALOGV("%s: Treat buffer format to 0x%x as HAL_PIXEL_FORMAT_YCbCr_420_888",
549                        __FUNCTION__, bufferFormat);
550            } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB &&
551                    bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888) {
552                // Using HAL_PIXEL_FORMAT_RGBA_8888 Gralloc buffers containing JPEGs to get around
553                // SW write limitations for (b/17379185).
554                ALOGV("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__);
555            } else {
556                // Return the buffer to the queue. No need to provide fence, as this buffer wasn't
557                // used anywhere yet.
558                bufferConsumer->releaseBuffer(*buffer);
559                ctx->returnBufferItem(buffer);
560
561                // Throw exception
562                ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
563                        bufferFormat, ctx->getBufferFormat());
564                String8 msg;
565                msg.appendFormat("The producer output buffer format 0x%x doesn't "
566                        "match the ImageReader's configured buffer format 0x%x.",
567                        bufferFormat, ctx->getBufferFormat());
568                jniThrowException(env, "java/lang/UnsupportedOperationException",
569                        msg.string());
570                return -1;
571            }
572        }
573
574    }
575
576    // Set SurfaceImage instance member variables
577    Image_setBufferItem(env, image, buffer);
578    env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp,
579            static_cast<jlong>(buffer->mTimestamp));
580
581    return ACQUIRE_SUCCESS;
582}
583
584static jint ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) {
585    ALOGV("%s:", __FUNCTION__);
586    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
587    if (ctx == NULL) {
588        jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
589        return -1;
590    }
591
592    BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
593    BufferItem* buffer = Image_getBufferItem(env, image);
594    if (!buffer) {
595        ALOGE(
596                "Image already released and can not be detached from ImageReader!!!");
597        jniThrowException(env, "java/lang/IllegalStateException",
598                "Image detach from ImageReader failed: buffer was already released");
599        return -1;
600    }
601
602    status_t res = OK;
603    Image_unlockIfLocked(env, image);
604    res = bufferConsumer->detachBuffer(buffer->mSlot);
605    if (res != OK) {
606        ALOGE("Image detach failed: %s (%d)!!!", strerror(-res), res);
607        jniThrowRuntimeException(env,
608                "nativeDetachImage failed for image!!!");
609        return res;
610    }
611    return OK;
612}
613
614static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
615{
616    ALOGV("%s: ", __FUNCTION__);
617
618    IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz);
619    if (gbp == NULL) {
620        jniThrowRuntimeException(env, "Buffer consumer is uninitialized");
621        return NULL;
622    }
623
624    // Wrap the IGBP in a Java-language Surface.
625    return android_view_Surface_createFromIGraphicBufferProducer(env, gbp);
626}
627
628static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) {
629    ALOGV("%s", __FUNCTION__);
630    BufferItem* buffer = Image_getBufferItem(env, thiz);
631    if (buffer == NULL) {
632        jniThrowException(env, "java/lang/IllegalStateException",
633                "Image is not initialized");
634        return;
635    }
636
637    status_t res = lockImageFromBuffer(buffer,
638            GRALLOC_USAGE_SW_READ_OFTEN, buffer->mFence->dup(), image);
639    if (res != OK) {
640        jniThrowExceptionFmt(env, "java/lang/RuntimeException",
641                "lock buffer failed for format 0x%x",
642                buffer->mGraphicBuffer->getPixelFormat());
643        return;
644    }
645
646    // Carry over some fields from BufferItem.
647    image->crop        = buffer->mCrop;
648    image->transform   = buffer->mTransform;
649    image->scalingMode = buffer->mScalingMode;
650    image->timestamp   = buffer->mTimestamp;
651    image->dataSpace   = buffer->mDataSpace;
652    image->frameNumber = buffer->mFrameNumber;
653
654    ALOGV("%s: Successfully locked the image", __FUNCTION__);
655    // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer,
656    // and we don't set them here.
657}
658
659static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
660        int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {
661    ALOGV("%s", __FUNCTION__);
662
663    status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size,
664            pixelStride, rowStride);
665    if (res != OK) {
666        jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
667                             "Pixel format: 0x%x is unsupported", buffer->flexFormat);
668    }
669}
670
671static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
672        int numPlanes, int readerFormat)
673{
674    ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes);
675    int rowStride = 0;
676    int pixelStride = 0;
677    uint8_t *pData = NULL;
678    uint32_t dataSize = 0;
679    jobject byteBuffer = NULL;
680
681    PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat);
682    int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat(
683        publicReaderFormat);
684
685    if (isFormatOpaque(halReaderFormat) && numPlanes > 0) {
686        String8 msg;
687        msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
688                " must be 0", halReaderFormat, numPlanes);
689        jniThrowException(env, "java/lang/IllegalArgumentException", msg.string());
690        return NULL;
691    }
692
693    jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz,
694            /*initial_element*/NULL);
695    if (surfacePlanes == NULL) {
696        jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays,"
697                " probably out of memory");
698        return NULL;
699    }
700    if (isFormatOpaque(halReaderFormat)) {
701        // Return 0 element surface array.
702        return surfacePlanes;
703    }
704
705    LockedImage lockedImg = LockedImage();
706    Image_getLockedImage(env, thiz, &lockedImg);
707    // Create all SurfacePlanes
708    for (int i = 0; i < numPlanes; i++) {
709        Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat,
710                &pData, &dataSize, &pixelStride, &rowStride);
711        byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
712        if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
713            jniThrowException(env, "java/lang/IllegalStateException",
714                    "Failed to allocate ByteBuffer");
715            return NULL;
716        }
717
718        // Finally, create this SurfacePlane.
719        jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
720                    gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
721        env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
722    }
723
724    return surfacePlanes;
725}
726
727static jint Image_getWidth(JNIEnv* env, jobject thiz)
728{
729    BufferItem* buffer = Image_getBufferItem(env, thiz);
730    return getBufferWidth(buffer);
731}
732
733static jint Image_getHeight(JNIEnv* env, jobject thiz)
734{
735    BufferItem* buffer = Image_getBufferItem(env, thiz);
736    return getBufferHeight(buffer);
737}
738
739static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat)
740{
741    if (isFormatOpaque(readerFormat)) {
742        // Assuming opaque reader produce opaque images.
743        return static_cast<jint>(PublicFormat::PRIVATE);
744    } else {
745        BufferItem* buffer = Image_getBufferItem(env, thiz);
746        int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat(
747                static_cast<PublicFormat>(readerFormat));
748        int32_t fmt = applyFormatOverrides(
749                buffer->mGraphicBuffer->getPixelFormat(), readerHalFormat);
750        // Override the image format to HAL_PIXEL_FORMAT_YCbCr_420_888 if the actual format is
751        // NV21 or YV12. This could only happen when the Gralloc HAL version is v0.1 thus doesn't
752        // support lockycbcr(), the CpuConsumer need to use the lock() method in the
753        // lockNextBuffer() call. For Gralloc HAL v0.2 or newer, this format should already be
754        // overridden to HAL_PIXEL_FORMAT_YCbCr_420_888 for the flexible YUV compatible formats.
755        if (isPossiblyYUV(fmt)) {
756            fmt = HAL_PIXEL_FORMAT_YCbCr_420_888;
757        }
758        PublicFormat publicFmt = android_view_Surface_mapHalFormatDataspaceToPublicFormat(
759                fmt, buffer->mDataSpace);
760        return static_cast<jint>(publicFmt);
761    }
762}
763
764} // extern "C"
765
766// ----------------------------------------------------------------------------
767
768static const JNINativeMethod gImageReaderMethods[] = {
769    {"nativeClassInit",        "()V",                        (void*)ImageReader_classInit },
770    {"nativeInit",             "(Ljava/lang/Object;IIII)V",  (void*)ImageReader_init },
771    {"nativeClose",            "()V",                        (void*)ImageReader_close },
772    {"nativeReleaseImage",     "(Landroid/media/Image;)V",   (void*)ImageReader_imageRelease },
773    {"nativeImageSetup",       "(Landroid/media/Image;)I",   (void*)ImageReader_imageSetup },
774    {"nativeGetSurface",       "()Landroid/view/Surface;",   (void*)ImageReader_getSurface },
775    {"nativeDetachImage",      "(Landroid/media/Image;)I",   (void*)ImageReader_detachImage },
776};
777
778static const JNINativeMethod gImageMethods[] = {
779    {"nativeCreatePlanes",      "(II)[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",
780                                                              (void*)Image_createSurfacePlanes },
781    {"nativeGetWidth",         "()I",                        (void*)Image_getWidth },
782    {"nativeGetHeight",        "()I",                        (void*)Image_getHeight },
783    {"nativeGetFormat",        "(I)I",                        (void*)Image_getFormat },
784};
785
786int register_android_media_ImageReader(JNIEnv *env) {
787
788    int ret1 = AndroidRuntime::registerNativeMethods(env,
789                   "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods));
790
791    int ret2 = AndroidRuntime::registerNativeMethods(env,
792                   "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods));
793
794    return (ret1 || ret2);
795}
796