android_media_ImageReader.cpp revision 91b9aabc9fa0c058ecc4a8b3f486540c28fe1cc0 
1/*
2 * Copyright 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "ImageReader_JNI"
19#include <utils/Log.h>
20#include <utils/misc.h>
21#include <utils/List.h>
22#include <utils/String8.h>
23
24#include <cstdio>
25
26#include <gui/CpuConsumer.h>
27#include <gui/Surface.h>
28#include <camera3.h>
29
30#include <android_runtime/AndroidRuntime.h>
31#include <android_runtime/android_view_Surface.h>
32
33#include <jni.h>
34#include <JNIHelp.h>
35
36#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
37
38#define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID       "mNativeContext"
39#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID   "mLockedBuffer"
40#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID       "mTimestamp"
41
42// ----------------------------------------------------------------------------
43
44using namespace android;
45
46enum {
47    IMAGE_READER_MAX_NUM_PLANES = 3,
48};
49
50enum {
51    ACQUIRE_SUCCESS = 0,
52    ACQUIRE_NO_BUFFERS = 1,
53    ACQUIRE_MAX_IMAGES = 2,
54};
55
56static struct {
57    jfieldID mNativeContext;
58    jmethodID postEventFromNative;
59} gImageReaderClassInfo;
60
61static struct {
62    jfieldID mLockedBuffer;
63    jfieldID mTimestamp;
64} gSurfaceImageClassInfo;
65
66static struct {
67    jclass clazz;
68    jmethodID ctor;
69} gSurfacePlaneClassInfo;
70
71// ----------------------------------------------------------------------------
72
73class JNIImageReaderContext : public CpuConsumer::FrameAvailableListener
74{
75public:
76    JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages);
77
78    virtual ~JNIImageReaderContext();
79
80    virtual void onFrameAvailable();
81
82    CpuConsumer::LockedBuffer* getLockedBuffer();
83
84    void returnLockedBuffer(CpuConsumer::LockedBuffer* buffer);
85
86    void setCpuConsumer(const sp<CpuConsumer>& consumer) { mConsumer = consumer; }
87    CpuConsumer* getCpuConsumer() { return mConsumer.get(); }
88
89    void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; }
90    IGraphicBufferProducer* getProducer() { return mProducer.get(); }
91
92    void setBufferFormat(int format) { mFormat = format; }
93    int getBufferFormat() { return mFormat; }
94
95    void setBufferWidth(int width) { mWidth = width; }
96    int getBufferWidth() { return mWidth; }
97
98    void setBufferHeight(int height) { mHeight = height; }
99    int getBufferHeight() { return mHeight; }
100
101private:
102    static JNIEnv* getJNIEnv(bool* needsDetach);
103    static void detachJNI();
104
105    List<CpuConsumer::LockedBuffer*> mBuffers;
106    sp<CpuConsumer> mConsumer;
107    sp<IGraphicBufferProducer> mProducer;
108    jobject mWeakThiz;
109    jclass mClazz;
110    int mFormat;
111    int mWidth;
112    int mHeight;
113};
114
115JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env,
116        jobject weakThiz, jclass clazz, int maxImages) :
117    mWeakThiz(env->NewGlobalRef(weakThiz)),
118    mClazz((jclass)env->NewGlobalRef(clazz)) {
119    for (int i = 0; i < maxImages; i++) {
120        CpuConsumer::LockedBuffer *buffer = new CpuConsumer::LockedBuffer;
121        mBuffers.push_back(buffer);
122    }
123}
124
125JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) {
126    LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
127    *needsDetach = false;
128    JNIEnv* env = AndroidRuntime::getJNIEnv();
129    if (env == NULL) {
130        JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
131        JavaVM* vm = AndroidRuntime::getJavaVM();
132        int result = vm->AttachCurrentThread(&env, (void*) &args);
133        if (result != JNI_OK) {
134            ALOGE("thread attach failed: %#x", result);
135            return NULL;
136        }
137        *needsDetach = true;
138    }
139    return env;
140}
141
142void JNIImageReaderContext::detachJNI() {
143    JavaVM* vm = AndroidRuntime::getJavaVM();
144    int result = vm->DetachCurrentThread();
145    if (result != JNI_OK) {
146        ALOGE("thread detach failed: %#x", result);
147    }
148}
149
150CpuConsumer::LockedBuffer* JNIImageReaderContext::getLockedBuffer() {
151    if (mBuffers.empty()) {
152        return NULL;
153    }
154    // Return a LockedBuffer pointer and remove it from the list
155    List<CpuConsumer::LockedBuffer*>::iterator it = mBuffers.begin();
156    CpuConsumer::LockedBuffer* buffer = *it;
157    mBuffers.erase(it);
158    return buffer;
159}
160
161void JNIImageReaderContext::returnLockedBuffer(CpuConsumer::LockedBuffer* buffer) {
162    mBuffers.push_back(buffer);
163}
164
165JNIImageReaderContext::~JNIImageReaderContext() {
166    bool needsDetach = false;
167    JNIEnv* env = getJNIEnv(&needsDetach);
168    if (env != NULL) {
169        env->DeleteGlobalRef(mWeakThiz);
170        env->DeleteGlobalRef(mClazz);
171    } else {
172        ALOGW("leaking JNI object references");
173    }
174    if (needsDetach) {
175        detachJNI();
176    }
177
178    // Delete LockedBuffers
179    for (List<CpuConsumer::LockedBuffer *>::iterator it = mBuffers.begin();
180            it != mBuffers.end(); it++) {
181        delete *it;
182    }
183    mBuffers.clear();
184    mConsumer.clear();
185}
186
187void JNIImageReaderContext::onFrameAvailable()
188{
189    ALOGV("%s: frame available", __FUNCTION__);
190    bool needsDetach = false;
191    JNIEnv* env = getJNIEnv(&needsDetach);
192    if (env != NULL) {
193        env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
194    } else {
195        ALOGW("onFrameAvailable event will not posted");
196    }
197    if (needsDetach) {
198        detachJNI();
199    }
200}
201
202// ----------------------------------------------------------------------------
203
204extern "C" {
205
206static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz)
207{
208    JNIImageReaderContext *ctx;
209    ctx = reinterpret_cast<JNIImageReaderContext *>
210              (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext));
211    return ctx;
212}
213
214static CpuConsumer* ImageReader_getCpuConsumer(JNIEnv* env, jobject thiz)
215{
216    ALOGV("%s:", __FUNCTION__);
217    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
218    if (ctx == NULL) {
219        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
220        return NULL;
221    }
222    return ctx->getCpuConsumer();
223}
224
225static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz)
226{
227    ALOGV("%s:", __FUNCTION__);
228    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
229    if (ctx == NULL) {
230        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
231        return NULL;
232    }
233    return ctx->getProducer();
234}
235
236static void ImageReader_setNativeContext(JNIEnv* env,
237        jobject thiz, sp<JNIImageReaderContext> ctx)
238{
239    ALOGV("%s:", __FUNCTION__);
240    JNIImageReaderContext* const p = ImageReader_getContext(env, thiz);
241    if (ctx != 0) {
242        ctx->incStrong((void*)ImageReader_setNativeContext);
243    }
244    if (p) {
245        p->decStrong((void*)ImageReader_setNativeContext);
246    }
247    env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext,
248            reinterpret_cast<jlong>(ctx.get()));
249}
250
251static CpuConsumer::LockedBuffer* Image_getLockedBuffer(JNIEnv* env, jobject image)
252{
253    return reinterpret_cast<CpuConsumer::LockedBuffer*>(
254            env->GetLongField(image, gSurfaceImageClassInfo.mLockedBuffer));
255}
256
257static void Image_setBuffer(JNIEnv* env, jobject thiz,
258        const CpuConsumer::LockedBuffer* buffer)
259{
260    env->SetLongField(thiz, gSurfaceImageClassInfo.mLockedBuffer, reinterpret_cast<jlong>(buffer));
261}
262
263// Some formats like JPEG defined with different values between android.graphics.ImageFormat and
264// graphics.h, need convert to the one defined in graphics.h here.
265static int Image_getPixelFormat(JNIEnv* env, int format)
266{
267    int jpegFormat, rawSensorFormat;
268    jfieldID fid;
269
270    ALOGV("%s: format = 0x%x", __FUNCTION__, format);
271
272    jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat");
273    ALOG_ASSERT(imageFormatClazz != NULL);
274
275    fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I");
276    jpegFormat = env->GetStaticIntField(imageFormatClazz, fid);
277
278    // Translate the JPEG to BLOB for camera purpose.
279    if (format == jpegFormat) {
280        format = HAL_PIXEL_FORMAT_BLOB;
281    }
282
283    return format;
284}
285
286static uint32_t Image_getJpegSize(CpuConsumer::LockedBuffer* buffer)
287{
288    ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
289    uint32_t size = 0;
290    uint32_t width = buffer->width;
291    uint8_t* jpegBuffer = buffer->data;
292
293    // First check for JPEG transport header at the end of the buffer
294    uint8_t* header = jpegBuffer + (width - sizeof(struct camera3_jpeg_blob));
295    struct camera3_jpeg_blob *blob = (struct camera3_jpeg_blob*)(header);
296    if (blob->jpeg_blob_id == CAMERA3_JPEG_BLOB_ID) {
297        size = blob->jpeg_size;
298        ALOGV("%s: Jpeg size = %d", __FUNCTION__, size);
299    }
300
301    // failed to find size, default to whole buffer
302    if (size == 0) {
303        size = width;
304    }
305
306    return size;
307}
308
309static void Image_getLockedBufferInfo(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx,
310                                uint8_t **base, uint32_t *size)
311{
312    ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
313    ALOG_ASSERT(base != NULL, "base is NULL!!!");
314    ALOG_ASSERT(size != NULL, "size is NULL!!!");
315    ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0));
316
317    ALOGV("%s: buffer: %p", __FUNCTION__, buffer);
318
319    uint32_t dataSize, ySize, cSize, cStride;
320    uint8_t *cb, *cr;
321    uint8_t *pData = NULL;
322    int bytesPerPixel = 0;
323
324    dataSize = ySize = cSize = cStride = 0;
325    int32_t fmt = buffer->format;
326    switch (fmt) {
327        case HAL_PIXEL_FORMAT_YCbCr_420_888:
328            pData =
329                (idx == 0) ?
330                    buffer->data :
331                (idx == 1) ?
332                    buffer->dataCb :
333                buffer->dataCr;
334            if (idx == 0) {
335                dataSize = buffer->stride * buffer->height;
336            } else {
337                dataSize = buffer->chromaStride * buffer->height / 2;
338            }
339            break;
340        // NV21
341        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
342            cr = buffer->data + (buffer->stride * buffer->height);
343            cb = cr + 1;
344            ySize = buffer->width * buffer->height;
345            cSize = buffer->width * buffer->height / 2;
346
347            pData =
348                (idx == 0) ?
349                    buffer->data :
350                (idx == 1) ?
351                    cb:
352                cr;
353
354            dataSize = (idx == 0) ? ySize : cSize;
355            break;
356        case HAL_PIXEL_FORMAT_YV12:
357            // Y and C stride need to be 16 pixel aligned.
358            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
359                                "Stride is not 16 pixel aligned %d", buffer->stride);
360
361            ySize = buffer->stride * buffer->height;
362            cStride = ALIGN(buffer->stride / 2, 16);
363            cr = buffer->data + ySize;
364            cSize = cStride * buffer->height / 2;
365            cb = cr + cSize;
366
367            pData =
368                (idx == 0) ?
369                    buffer->data :
370                (idx == 1) ?
371                    cb :
372                cr;
373            dataSize = (idx == 0) ? ySize : cSize;
374            break;
375        case HAL_PIXEL_FORMAT_Y8:
376            // Single plane, 8bpp.
377            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
378
379            pData = buffer->data;
380            dataSize = buffer->stride * buffer->height;
381            break;
382        case HAL_PIXEL_FORMAT_Y16:
383            bytesPerPixel = 2;
384            // Single plane, 16bpp, strides are specified in pixels, not in bytes
385            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
386
387            pData = buffer->data;
388            dataSize = buffer->stride * buffer->height * bytesPerPixel;
389            break;
390        case HAL_PIXEL_FORMAT_BLOB:
391            // Used for JPEG data, height must be 1, width == size, single plane.
392            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
393            ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height);
394
395            pData = buffer->data;
396            dataSize = Image_getJpegSize(buffer);
397            break;
398        case HAL_PIXEL_FORMAT_RAW_SENSOR:
399            // Single plane 16bpp bayer data.
400            bytesPerPixel = 2;
401            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
402            pData = buffer->data;
403            dataSize = buffer->stride * buffer->height * bytesPerPixel;
404            break;
405        case HAL_PIXEL_FORMAT_RGBA_8888:
406        case HAL_PIXEL_FORMAT_RGBX_8888:
407            // Single plane, 32bpp.
408            bytesPerPixel = 4;
409            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
410            pData = buffer->data;
411            dataSize = buffer->stride * buffer->height * bytesPerPixel;
412            break;
413        case HAL_PIXEL_FORMAT_RGB_565:
414            // Single plane, 16bpp.
415            bytesPerPixel = 2;
416            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
417            pData = buffer->data;
418            dataSize = buffer->stride * buffer->height * bytesPerPixel;
419            break;
420        case HAL_PIXEL_FORMAT_RGB_888:
421            // Single plane, 24bpp.
422            bytesPerPixel = 3;
423            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
424            pData = buffer->data;
425            dataSize = buffer->stride * buffer->height * bytesPerPixel;
426            break;
427        default:
428            jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
429                                 "Pixel format: 0x%x is unsupported", fmt);
430            break;
431    }
432
433    *base = pData;
434    *size = dataSize;
435}
436
437static jint Image_imageGetPixelStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx)
438{
439    ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
440    ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0), "Index is out of range:%d", idx);
441
442    int pixelStride = 0;
443    ALOG_ASSERT(buffer != NULL, "buffer is NULL");
444
445    int32_t fmt = buffer->format;
446    switch (fmt) {
447        case HAL_PIXEL_FORMAT_YCbCr_420_888:
448            pixelStride = (idx == 0) ? 1 : buffer->chromaStep;
449            break;
450        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
451            pixelStride = (idx == 0) ? 1 : 2;
452            break;
453        case HAL_PIXEL_FORMAT_Y8:
454            // Single plane 8bpp data.
455            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
456            pixelStride;
457            break;
458        case HAL_PIXEL_FORMAT_YV12:
459            pixelStride = 1;
460            break;
461        case HAL_PIXEL_FORMAT_BLOB:
462            // Used for JPEG data, single plane, row and pixel strides are 0
463            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
464            pixelStride = 0;
465            break;
466        case HAL_PIXEL_FORMAT_Y16:
467        case HAL_PIXEL_FORMAT_RAW_SENSOR:
468        case HAL_PIXEL_FORMAT_RGB_565:
469            // Single plane 16bpp data.
470            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
471            pixelStride = 2;
472            break;
473        case HAL_PIXEL_FORMAT_RGBA_8888:
474        case HAL_PIXEL_FORMAT_RGBX_8888:
475            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
476            pixelStride = 4;
477            break;
478        case HAL_PIXEL_FORMAT_RGB_888:
479            // Single plane, 24bpp.
480            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
481            pixelStride = 3;
482            break;
483        default:
484            jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
485                                 "Pixel format: 0x%x is unsupported", fmt);
486            break;
487    }
488
489    return pixelStride;
490}
491
492static jint Image_imageGetRowStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx)
493{
494    ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
495    ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0));
496
497    int rowStride = 0;
498    ALOG_ASSERT(buffer != NULL, "buffer is NULL");
499
500    int32_t fmt = buffer->format;
501
502    switch (fmt) {
503        case HAL_PIXEL_FORMAT_YCbCr_420_888:
504            rowStride = (idx == 0) ? buffer->stride : buffer->chromaStride;
505            break;
506        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
507            rowStride = buffer->width;
508            break;
509        case HAL_PIXEL_FORMAT_YV12:
510            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
511                                "Stride is not 16 pixel aligned %d", buffer->stride);
512            rowStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16);
513            break;
514        case HAL_PIXEL_FORMAT_BLOB:
515            // Used for JPEG data, single plane, row and pixel strides are 0
516            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
517            rowStride = 0;
518            break;
519        case HAL_PIXEL_FORMAT_Y8:
520            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
521            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
522                                "Stride is not 16 pixel aligned %d", buffer->stride);
523            rowStride = buffer->stride;
524            break;
525        case HAL_PIXEL_FORMAT_Y16:
526        case HAL_PIXEL_FORMAT_RAW_SENSOR:
527            // In native side, strides are specified in pixels, not in bytes.
528            // Single plane 16bpp bayer data. even width/height,
529            // row stride multiple of 16 pixels (32 bytes)
530            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
531            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
532                                "Stride is not 16 pixel aligned %d", buffer->stride);
533            rowStride = buffer->stride * 2;
534            break;
535        case HAL_PIXEL_FORMAT_RGB_565:
536            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
537            rowStride = buffer->stride * 2;
538            break;
539        case HAL_PIXEL_FORMAT_RGBA_8888:
540        case HAL_PIXEL_FORMAT_RGBX_8888:
541            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
542            rowStride = buffer->stride * 4;
543            break;
544        case HAL_PIXEL_FORMAT_RGB_888:
545            // Single plane, 24bpp.
546            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
547            rowStride = buffer->stride * 3;
548            break;
549        default:
550            ALOGE("%s Pixel format: 0x%x is unsupported", __FUNCTION__, fmt);
551            jniThrowException(env, "java/lang/UnsupportedOperationException",
552                              "unsupported buffer format");
553          break;
554    }
555
556    return rowStride;
557}
558
559// ----------------------------------------------------------------------------
560
561static void ImageReader_classInit(JNIEnv* env, jclass clazz)
562{
563    ALOGV("%s:", __FUNCTION__);
564
565    jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage");
566    LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
567                        "can't find android/graphics/ImageReader$SurfaceImage");
568    gSurfaceImageClassInfo.mLockedBuffer = env->GetFieldID(
569            imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J");
570    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mLockedBuffer == NULL,
571                        "can't find android/graphics/ImageReader.%s",
572                        ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID);
573
574    gSurfaceImageClassInfo.mTimestamp = env->GetFieldID(
575            imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J");
576    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL,
577                        "can't find android/graphics/ImageReader.%s",
578                        ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID);
579
580    gImageReaderClassInfo.mNativeContext = env->GetFieldID(
581            clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J");
582    LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL,
583                        "can't find android/graphics/ImageReader.%s",
584                          ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID);
585
586    gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(
587            clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
588    LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL,
589                        "can't find android/graphics/ImageReader.postEventFromNative");
590
591    jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane");
592    LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
593    // FindClass only gives a local reference of jclass object.
594    gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
595    gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
596            "(Landroid/media/ImageReader$SurfaceImage;III)V");
597    LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
598            "Can not find SurfacePlane constructor");
599}
600
601static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
602                             jint width, jint height, jint format, jint maxImages)
603{
604    status_t res;
605    int nativeFormat;
606
607    ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
608          __FUNCTION__, width, height, format, maxImages);
609
610    nativeFormat = Image_getPixelFormat(env, format);
611
612    sp<IGraphicBufferProducer> gbProducer;
613    sp<IGraphicBufferConsumer> gbConsumer;
614    BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
615    sp<CpuConsumer> consumer = new CpuConsumer(gbConsumer, maxImages,
616                                               /*controlledByApp*/true);
617    // TODO: throw dvm exOutOfMemoryError?
618    if (consumer == NULL) {
619        jniThrowRuntimeException(env, "Failed to allocate native CpuConsumer");
620        return;
621    }
622
623    jclass clazz = env->GetObjectClass(thiz);
624    if (clazz == NULL) {
625        jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader");
626        return;
627    }
628    sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
629    ctx->setCpuConsumer(consumer);
630    ctx->setProducer(gbProducer);
631    consumer->setFrameAvailableListener(ctx);
632    ImageReader_setNativeContext(env, thiz, ctx);
633    ctx->setBufferFormat(nativeFormat);
634    ctx->setBufferWidth(width);
635    ctx->setBufferHeight(height);
636
637    // Set the width/height/format to the CpuConsumer
638    res = consumer->setDefaultBufferSize(width, height);
639    if (res != OK) {
640        jniThrowException(env, "java/lang/IllegalStateException",
641                          "Failed to set CpuConsumer buffer size");
642        return;
643    }
644    res = consumer->setDefaultBufferFormat(nativeFormat);
645    if (res != OK) {
646        jniThrowException(env, "java/lang/IllegalStateException",
647                          "Failed to set CpuConsumer buffer format");
648    }
649}
650
651static void ImageReader_close(JNIEnv* env, jobject thiz)
652{
653    ALOGV("%s:", __FUNCTION__);
654
655    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
656    if (ctx == NULL) {
657        // ImageReader is already closed.
658        return;
659    }
660
661    CpuConsumer* consumer = ImageReader_getCpuConsumer(env, thiz);
662    if (consumer != NULL) {
663        consumer->abandon();
664        consumer->setFrameAvailableListener(NULL);
665    }
666    ImageReader_setNativeContext(env, thiz, NULL);
667}
668
669static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image)
670{
671    ALOGV("%s:", __FUNCTION__);
672    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
673    if (ctx == NULL) {
674        ALOGW("ImageReader#close called before Image#close, consider calling Image#close first");
675        return;
676    }
677
678    CpuConsumer* consumer = ctx->getCpuConsumer();
679    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image);
680    if (!buffer) {
681        ALOGW("Image already released!!!");
682        return;
683    }
684    consumer->unlockBuffer(*buffer);
685    Image_setBuffer(env, image, NULL);
686    ctx->returnLockedBuffer(buffer);
687}
688
689static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz,
690                                             jobject image)
691{
692    ALOGV("%s:", __FUNCTION__);
693    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
694    if (ctx == NULL) {
695        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
696        return -1;
697    }
698
699    CpuConsumer* consumer = ctx->getCpuConsumer();
700    CpuConsumer::LockedBuffer* buffer = ctx->getLockedBuffer();
701    if (buffer == NULL) {
702        ALOGW("Unable to acquire a lockedBuffer, very likely client tries to lock more than"
703            " maxImages buffers");
704        return ACQUIRE_MAX_IMAGES;
705    }
706    status_t res = consumer->lockNextBuffer(buffer);
707    if (res != NO_ERROR) {
708        ctx->returnLockedBuffer(buffer);
709        if (res != BAD_VALUE /*no buffers*/) {
710            if (res == NOT_ENOUGH_DATA) {
711                return ACQUIRE_MAX_IMAGES;
712            } else {
713                ALOGE("%s Fail to lockNextBuffer with error: %d ",
714                      __FUNCTION__, res);
715                jniThrowExceptionFmt(env, "java/lang/AssertionError",
716                          "Unknown error (%d) when we tried to lock buffer.",
717                          res);
718            }
719        }
720        return ACQUIRE_NO_BUFFERS;
721    }
722
723    if (buffer->format == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
724        jniThrowException(env, "java/lang/UnsupportedOperationException",
725                "NV21 format is not supported by ImageReader");
726        return -1;
727    }
728
729    // Check if the left-top corner of the crop rect is origin, we currently assume this point is
730    // zero, will revist this once this assumption turns out problematic.
731    Point lt = buffer->crop.leftTop();
732    if (lt.x != 0 || lt.y != 0) {
733        jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
734                "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
735        return -1;
736    }
737
738    // Check if the producer buffer configurations match what ImageReader configured.
739    // We want to fail for the very first image because this case is too bad.
740    int outputWidth = buffer->width;
741    int outputHeight = buffer->height;
742
743    // Correct width/height when crop is set.
744    if (!buffer->crop.isEmpty()) {
745        outputWidth = buffer->crop.getWidth();
746        outputHeight = buffer->crop.getHeight();
747    }
748
749    int imageReaderWidth = ctx->getBufferWidth();
750    int imageReaderHeight = ctx->getBufferHeight();
751    if ((buffer->format != HAL_PIXEL_FORMAT_BLOB) &&
752            (imageReaderWidth != outputWidth || imageReaderHeight > outputHeight)) {
753        /**
754         * For video decoder, the buffer height is actually the vertical stride,
755         * which is always >= actual image height. For future, decoder need provide
756         * right crop rectangle to CpuConsumer to indicate the actual image height,
757         * see bug 9563986. After this bug is fixed, we can enforce the height equal
758         * check. Right now, only make sure buffer height is no less than ImageReader
759         * height.
760         */
761        jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
762                "Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d",
763                outputWidth, outputHeight, imageReaderWidth, imageReaderHeight);
764        return -1;
765    }
766
767    int imgReaderFmt = ctx->getBufferFormat();
768    int bufFmt = buffer->format;
769    if (imgReaderFmt != bufFmt) {
770        // Special casing for when producer switches to a format compatible with flexible YUV
771        // (HAL_PIXEL_FORMAT_YCbCr_420_888).
772        if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && (bufFmt ==
773                HAL_PIXEL_FORMAT_YCrCb_420_SP || bufFmt == HAL_PIXEL_FORMAT_YV12)) {
774            ctx->setBufferFormat(bufFmt);
775            ALOGV("%s: Overriding buffer format YUV_420_888 to %x.", __FUNCTION__, bufFmt);
776        } else {
777            // Return the buffer to the queue.
778            consumer->unlockBuffer(*buffer);
779            ctx->returnLockedBuffer(buffer);
780
781            // Throw exception
782            ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
783                    buffer->format, ctx->getBufferFormat());
784            String8 msg;
785            msg.appendFormat("The producer output buffer format 0x%x doesn't "
786                    "match the ImageReader's configured buffer format 0x%x.",
787                    buffer->format, ctx->getBufferFormat());
788            jniThrowException(env, "java/lang/UnsupportedOperationException",
789                    msg.string());
790            return -1;
791        }
792    }
793    // Set SurfaceImage instance member variables
794    Image_setBuffer(env, image, buffer);
795    env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp,
796            static_cast<jlong>(buffer->timestamp));
797
798    return ACQUIRE_SUCCESS;
799}
800
801static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
802{
803    ALOGV("%s: ", __FUNCTION__);
804
805    IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz);
806    if (gbp == NULL) {
807        jniThrowRuntimeException(env, "CpuConsumer is uninitialized");
808        return NULL;
809    }
810
811    // Wrap the IGBP in a Java-language Surface.
812    return android_view_Surface_createFromIGraphicBufferProducer(env, gbp);
813}
814
815static jobject Image_createSurfacePlane(JNIEnv* env, jobject thiz, int idx)
816{
817    int rowStride, pixelStride;
818    ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
819
820    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
821
822    ALOG_ASSERT(buffer != NULL);
823    if (buffer == NULL) {
824        jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
825    }
826    rowStride = Image_imageGetRowStride(env, buffer, idx);
827    pixelStride = Image_imageGetPixelStride(env, buffer, idx);
828
829    jobject surfPlaneObj = env->NewObject(gSurfacePlaneClassInfo.clazz,
830            gSurfacePlaneClassInfo.ctor, thiz, idx, rowStride, pixelStride);
831
832    return surfPlaneObj;
833}
834
835static jobject Image_getByteBuffer(JNIEnv* env, jobject thiz, int idx)
836{
837    uint8_t *base = NULL;
838    uint32_t size = 0;
839    jobject byteBuffer;
840
841    ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
842
843    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
844
845    if (buffer == NULL) {
846        jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
847    }
848
849    // Create byteBuffer from native buffer
850    Image_getLockedBufferInfo(env, buffer, idx, &base, &size);
851    byteBuffer = env->NewDirectByteBuffer(base, size);
852    // TODO: throw dvm exOutOfMemoryError?
853    if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
854        jniThrowException(env, "java/lang/IllegalStateException", "Failed to allocate ByteBuffer");
855    }
856
857    return byteBuffer;
858}
859
860} // extern "C"
861
862// ----------------------------------------------------------------------------
863
864static JNINativeMethod gImageReaderMethods[] = {
865    {"nativeClassInit",        "()V",                        (void*)ImageReader_classInit },
866    {"nativeInit",             "(Ljava/lang/Object;IIII)V",  (void*)ImageReader_init },
867    {"nativeClose",            "()V",                        (void*)ImageReader_close },
868    {"nativeReleaseImage",     "(Landroid/media/Image;)V",   (void*)ImageReader_imageRelease },
869    {"nativeImageSetup",       "(Landroid/media/Image;)I",    (void*)ImageReader_imageSetup },
870    {"nativeGetSurface",       "()Landroid/view/Surface;",   (void*)ImageReader_getSurface },
871};
872
873static JNINativeMethod gImageMethods[] = {
874    {"nativeImageGetBuffer",   "(I)Ljava/nio/ByteBuffer;",   (void*)Image_getByteBuffer },
875    {"nativeCreatePlane",      "(I)Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",
876                                                             (void*)Image_createSurfacePlane },
877};
878
879int register_android_media_ImageReader(JNIEnv *env) {
880
881    int ret1 = AndroidRuntime::registerNativeMethods(env,
882                   "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods));
883
884    int ret2 = AndroidRuntime::registerNativeMethods(env,
885                   "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods));
886
887    return (ret1 || ret2);
888}
889