android_media_ImageReader.cpp revision 4fb442617cdea52b11ed622b5cf0490337928ec1 
1/*
2 * Copyright 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "ImageReader_JNI"
19#include <utils/Log.h>
20#include <utils/misc.h>
21#include <utils/List.h>
22#include <utils/String8.h>
23
24#include <cstdio>
25
26#include <gui/CpuConsumer.h>
27#include <gui/Surface.h>
28#include <camera3.h>
29
30#include <android_runtime/AndroidRuntime.h>
31#include <android_runtime/android_view_Surface.h>
32
33#include <jni.h>
34#include <JNIHelp.h>
35
36#include <stdint.h>
37#include <inttypes.h>
38
39#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
40
41#define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID       "mNativeContext"
42#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID   "mLockedBuffer"
43#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID       "mTimestamp"
44
45// ----------------------------------------------------------------------------
46
47using namespace android;
48
49enum {
50    IMAGE_READER_MAX_NUM_PLANES = 3,
51};
52
53enum {
54    ACQUIRE_SUCCESS = 0,
55    ACQUIRE_NO_BUFFERS = 1,
56    ACQUIRE_MAX_IMAGES = 2,
57};
58
59static struct {
60    jfieldID mNativeContext;
61    jmethodID postEventFromNative;
62} gImageReaderClassInfo;
63
64static struct {
65    jfieldID mLockedBuffer;
66    jfieldID mTimestamp;
67} gSurfaceImageClassInfo;
68
69static struct {
70    jclass clazz;
71    jmethodID ctor;
72} gSurfacePlaneClassInfo;
73
74// ----------------------------------------------------------------------------
75
76class JNIImageReaderContext : public CpuConsumer::FrameAvailableListener
77{
78public:
79    JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages);
80
81    virtual ~JNIImageReaderContext();
82
83    virtual void onFrameAvailable(const BufferItem& item);
84
85    CpuConsumer::LockedBuffer* getLockedBuffer();
86
87    void returnLockedBuffer(CpuConsumer::LockedBuffer* buffer);
88
89    void setCpuConsumer(const sp<CpuConsumer>& consumer) { mConsumer = consumer; }
90    CpuConsumer* getCpuConsumer() { return mConsumer.get(); }
91
92    void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; }
93    IGraphicBufferProducer* getProducer() { return mProducer.get(); }
94
95    void setBufferFormat(int format) { mFormat = format; }
96    int getBufferFormat() { return mFormat; }
97
98    void setBufferWidth(int width) { mWidth = width; }
99    int getBufferWidth() { return mWidth; }
100
101    void setBufferHeight(int height) { mHeight = height; }
102    int getBufferHeight() { return mHeight; }
103
104private:
105    static JNIEnv* getJNIEnv(bool* needsDetach);
106    static void detachJNI();
107
108    List<CpuConsumer::LockedBuffer*> mBuffers;
109    sp<CpuConsumer> mConsumer;
110    sp<IGraphicBufferProducer> mProducer;
111    jobject mWeakThiz;
112    jclass mClazz;
113    int mFormat;
114    int mWidth;
115    int mHeight;
116};
117
118JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env,
119        jobject weakThiz, jclass clazz, int maxImages) :
120    mWeakThiz(env->NewGlobalRef(weakThiz)),
121    mClazz((jclass)env->NewGlobalRef(clazz)) {
122    for (int i = 0; i < maxImages; i++) {
123        CpuConsumer::LockedBuffer *buffer = new CpuConsumer::LockedBuffer;
124        mBuffers.push_back(buffer);
125    }
126}
127
128JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) {
129    LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
130    *needsDetach = false;
131    JNIEnv* env = AndroidRuntime::getJNIEnv();
132    if (env == NULL) {
133        JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
134        JavaVM* vm = AndroidRuntime::getJavaVM();
135        int result = vm->AttachCurrentThread(&env, (void*) &args);
136        if (result != JNI_OK) {
137            ALOGE("thread attach failed: %#x", result);
138            return NULL;
139        }
140        *needsDetach = true;
141    }
142    return env;
143}
144
145void JNIImageReaderContext::detachJNI() {
146    JavaVM* vm = AndroidRuntime::getJavaVM();
147    int result = vm->DetachCurrentThread();
148    if (result != JNI_OK) {
149        ALOGE("thread detach failed: %#x", result);
150    }
151}
152
153CpuConsumer::LockedBuffer* JNIImageReaderContext::getLockedBuffer() {
154    if (mBuffers.empty()) {
155        return NULL;
156    }
157    // Return a LockedBuffer pointer and remove it from the list
158    List<CpuConsumer::LockedBuffer*>::iterator it = mBuffers.begin();
159    CpuConsumer::LockedBuffer* buffer = *it;
160    mBuffers.erase(it);
161    return buffer;
162}
163
164void JNIImageReaderContext::returnLockedBuffer(CpuConsumer::LockedBuffer* buffer) {
165    mBuffers.push_back(buffer);
166}
167
168JNIImageReaderContext::~JNIImageReaderContext() {
169    bool needsDetach = false;
170    JNIEnv* env = getJNIEnv(&needsDetach);
171    if (env != NULL) {
172        env->DeleteGlobalRef(mWeakThiz);
173        env->DeleteGlobalRef(mClazz);
174    } else {
175        ALOGW("leaking JNI object references");
176    }
177    if (needsDetach) {
178        detachJNI();
179    }
180
181    // Delete LockedBuffers
182    for (List<CpuConsumer::LockedBuffer *>::iterator it = mBuffers.begin();
183            it != mBuffers.end(); it++) {
184        delete *it;
185    }
186    mBuffers.clear();
187    mConsumer.clear();
188}
189
190void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
191{
192    ALOGV("%s: frame available", __FUNCTION__);
193    bool needsDetach = false;
194    JNIEnv* env = getJNIEnv(&needsDetach);
195    if (env != NULL) {
196        env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
197    } else {
198        ALOGW("onFrameAvailable event will not posted");
199    }
200    if (needsDetach) {
201        detachJNI();
202    }
203}
204
205// ----------------------------------------------------------------------------
206
207extern "C" {
208
209static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz)
210{
211    JNIImageReaderContext *ctx;
212    ctx = reinterpret_cast<JNIImageReaderContext *>
213              (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext));
214    return ctx;
215}
216
217static CpuConsumer* ImageReader_getCpuConsumer(JNIEnv* env, jobject thiz)
218{
219    ALOGV("%s:", __FUNCTION__);
220    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
221    if (ctx == NULL) {
222        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
223        return NULL;
224    }
225    return ctx->getCpuConsumer();
226}
227
228static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz)
229{
230    ALOGV("%s:", __FUNCTION__);
231    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
232    if (ctx == NULL) {
233        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
234        return NULL;
235    }
236    return ctx->getProducer();
237}
238
239static void ImageReader_setNativeContext(JNIEnv* env,
240        jobject thiz, sp<JNIImageReaderContext> ctx)
241{
242    ALOGV("%s:", __FUNCTION__);
243    JNIImageReaderContext* const p = ImageReader_getContext(env, thiz);
244    if (ctx != 0) {
245        ctx->incStrong((void*)ImageReader_setNativeContext);
246    }
247    if (p) {
248        p->decStrong((void*)ImageReader_setNativeContext);
249    }
250    env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext,
251            reinterpret_cast<jlong>(ctx.get()));
252}
253
254static CpuConsumer::LockedBuffer* Image_getLockedBuffer(JNIEnv* env, jobject image)
255{
256    return reinterpret_cast<CpuConsumer::LockedBuffer*>(
257            env->GetLongField(image, gSurfaceImageClassInfo.mLockedBuffer));
258}
259
260static void Image_setBuffer(JNIEnv* env, jobject thiz,
261        const CpuConsumer::LockedBuffer* buffer)
262{
263    env->SetLongField(thiz, gSurfaceImageClassInfo.mLockedBuffer, reinterpret_cast<jlong>(buffer));
264}
265
266// Some formats like JPEG defined with different values between android.graphics.ImageFormat and
267// graphics.h, need convert to the one defined in graphics.h here.
268static int Image_getPixelFormat(JNIEnv* env, int format)
269{
270    int jpegFormat;
271    jfieldID fid;
272
273    ALOGV("%s: format = 0x%x", __FUNCTION__, format);
274
275    jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat");
276    ALOG_ASSERT(imageFormatClazz != NULL);
277
278    fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I");
279    jpegFormat = env->GetStaticIntField(imageFormatClazz, fid);
280
281    // Translate the JPEG to BLOB for camera purpose.
282    if (format == jpegFormat) {
283        format = HAL_PIXEL_FORMAT_BLOB;
284    }
285
286    return format;
287}
288
289static uint32_t Image_getJpegSize(CpuConsumer::LockedBuffer* buffer, bool usingRGBAOverride)
290{
291    ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
292    uint32_t size = 0;
293    uint32_t width = buffer->width;
294    uint8_t* jpegBuffer = buffer->data;
295
296    if (usingRGBAOverride) {
297        width = (buffer->width + buffer->stride * (buffer->height - 1)) * 4;
298    }
299
300    // First check for JPEG transport header at the end of the buffer
301    uint8_t* header = jpegBuffer + (width - sizeof(struct camera3_jpeg_blob));
302    struct camera3_jpeg_blob *blob = (struct camera3_jpeg_blob*)(header);
303    if (blob->jpeg_blob_id == CAMERA3_JPEG_BLOB_ID) {
304        size = blob->jpeg_size;
305        ALOGV("%s: Jpeg size = %d", __FUNCTION__, size);
306    }
307
308    // failed to find size, default to whole buffer
309    if (size == 0) {
310        /*
311         * This is a problem because not including the JPEG header
312         * means that in certain rare situations a regular JPEG blob
313         * will be misidentified as having a header, in which case
314         * we will get a garbage size value.
315         */
316        ALOGW("%s: No JPEG header detected, defaulting to size=width=%d",
317                __FUNCTION__, width);
318        size = width;
319    }
320
321    return size;
322}
323
324static bool usingRGBAToJpegOverride(int32_t bufferFormat, int32_t readerCtxFormat) {
325    return readerCtxFormat == HAL_PIXEL_FORMAT_BLOB && bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888;
326}
327
328static int32_t applyFormatOverrides(int32_t bufferFormat, int32_t readerCtxFormat)
329{
330    // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW
331    // write limitations for some platforms (b/17379185).
332    if (usingRGBAToJpegOverride(bufferFormat, readerCtxFormat)) {
333        return HAL_PIXEL_FORMAT_BLOB;
334    }
335    return bufferFormat;
336}
337
338static void Image_getLockedBufferInfo(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx,
339                                uint8_t **base, uint32_t *size, int32_t readerFormat)
340{
341    ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
342    ALOG_ASSERT(base != NULL, "base is NULL!!!");
343    ALOG_ASSERT(size != NULL, "size is NULL!!!");
344    ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0));
345
346    ALOGV("%s: buffer: %p", __FUNCTION__, buffer);
347
348    uint32_t dataSize, ySize, cSize, cStride;
349    uint8_t *cb, *cr;
350    uint8_t *pData = NULL;
351    int bytesPerPixel = 0;
352
353    dataSize = ySize = cSize = cStride = 0;
354    int32_t fmt = buffer->flexFormat;
355
356    bool usingRGBAOverride = usingRGBAToJpegOverride(fmt, readerFormat);
357    fmt = applyFormatOverrides(fmt, readerFormat);
358    switch (fmt) {
359        case HAL_PIXEL_FORMAT_YCbCr_420_888:
360            pData =
361                (idx == 0) ?
362                    buffer->data :
363                (idx == 1) ?
364                    buffer->dataCb :
365                buffer->dataCr;
366            // only map until last pixel
367            if (idx == 0) {
368                dataSize = buffer->stride * (buffer->height - 1) + buffer->width;
369            } else {
370                dataSize = buffer->chromaStride * (buffer->height / 2 - 1) +
371                        buffer->chromaStep * (buffer->width / 2 - 1) + 1;
372            }
373            break;
374        // NV21
375        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
376            cr = buffer->data + (buffer->stride * buffer->height);
377            cb = cr + 1;
378            // only map until last pixel
379            ySize = buffer->width * (buffer->height - 1) + buffer->width;
380            cSize = buffer->width * (buffer->height / 2 - 1) + buffer->width - 1;
381
382            pData =
383                (idx == 0) ?
384                    buffer->data :
385                (idx == 1) ?
386                    cb:
387                cr;
388
389            dataSize = (idx == 0) ? ySize : cSize;
390            break;
391        case HAL_PIXEL_FORMAT_YV12:
392            // Y and C stride need to be 16 pixel aligned.
393            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
394                                "Stride is not 16 pixel aligned %d", buffer->stride);
395
396            ySize = buffer->stride * buffer->height;
397            cStride = ALIGN(buffer->stride / 2, 16);
398            cr = buffer->data + ySize;
399            cSize = cStride * buffer->height / 2;
400            cb = cr + cSize;
401
402            pData =
403                (idx == 0) ?
404                    buffer->data :
405                (idx == 1) ?
406                    cb :
407                cr;
408            dataSize = (idx == 0) ? ySize : cSize;
409            break;
410        case HAL_PIXEL_FORMAT_Y8:
411            // Single plane, 8bpp.
412            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
413
414            pData = buffer->data;
415            dataSize = buffer->stride * buffer->height;
416            break;
417        case HAL_PIXEL_FORMAT_Y16:
418            bytesPerPixel = 2;
419            // Single plane, 16bpp, strides are specified in pixels, not in bytes
420            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
421
422            pData = buffer->data;
423            dataSize = buffer->stride * buffer->height * bytesPerPixel;
424            break;
425        case HAL_PIXEL_FORMAT_BLOB:
426            // Used for JPEG data, height must be 1, width == size, single plane.
427            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
428            ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height);
429
430            pData = buffer->data;
431            dataSize = Image_getJpegSize(buffer, usingRGBAOverride);
432            break;
433        case HAL_PIXEL_FORMAT_RAW_SENSOR:
434            // Single plane 16bpp bayer data.
435            bytesPerPixel = 2;
436            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
437            pData = buffer->data;
438            dataSize = buffer->stride * buffer->height * bytesPerPixel;
439            break;
440        case HAL_PIXEL_FORMAT_RAW10:
441            // Single plane 10bpp bayer data.
442            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
443            LOG_ALWAYS_FATAL_IF(buffer->width % 4,
444                                "Width is not multiple of 4 %d", buffer->width);
445            LOG_ALWAYS_FATAL_IF(buffer->height % 2,
446                                "Height is not even %d", buffer->height);
447            LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 10 / 8),
448                                "stride (%d) should be at least %d",
449                                buffer->stride, buffer->width * 10 / 8);
450            pData = buffer->data;
451            dataSize = buffer->stride * buffer->height;
452            break;
453        case HAL_PIXEL_FORMAT_RGBA_8888:
454        case HAL_PIXEL_FORMAT_RGBX_8888:
455            // Single plane, 32bpp.
456            bytesPerPixel = 4;
457            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
458            pData = buffer->data;
459            dataSize = buffer->stride * buffer->height * bytesPerPixel;
460            break;
461        case HAL_PIXEL_FORMAT_RGB_565:
462            // Single plane, 16bpp.
463            bytesPerPixel = 2;
464            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
465            pData = buffer->data;
466            dataSize = buffer->stride * buffer->height * bytesPerPixel;
467            break;
468        case HAL_PIXEL_FORMAT_RGB_888:
469            // Single plane, 24bpp.
470            bytesPerPixel = 3;
471            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
472            pData = buffer->data;
473            dataSize = buffer->stride * buffer->height * bytesPerPixel;
474            break;
475        default:
476            jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
477                                 "Pixel format: 0x%x is unsupported", fmt);
478            break;
479    }
480
481    *base = pData;
482    *size = dataSize;
483}
484
485static jint Image_imageGetPixelStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx,
486        int32_t readerFormat)
487{
488    ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
489    ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0), "Index is out of range:%d", idx);
490
491    int pixelStride = 0;
492    ALOG_ASSERT(buffer != NULL, "buffer is NULL");
493
494    int32_t fmt = buffer->flexFormat;
495
496    fmt = applyFormatOverrides(fmt, readerFormat);
497
498    switch (fmt) {
499        case HAL_PIXEL_FORMAT_YCbCr_420_888:
500            pixelStride = (idx == 0) ? 1 : buffer->chromaStep;
501            break;
502        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
503            pixelStride = (idx == 0) ? 1 : 2;
504            break;
505        case HAL_PIXEL_FORMAT_Y8:
506            // Single plane 8bpp data.
507            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
508            pixelStride;
509            break;
510        case HAL_PIXEL_FORMAT_YV12:
511            pixelStride = 1;
512            break;
513        case HAL_PIXEL_FORMAT_BLOB:
514        case HAL_PIXEL_FORMAT_RAW10:
515            // Blob is used for JPEG data, RAW10 is used for 10-bit raw data, they are
516            // single plane, row and pixel strides are 0.
517            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
518            pixelStride = 0;
519            break;
520        case HAL_PIXEL_FORMAT_Y16:
521        case HAL_PIXEL_FORMAT_RAW_SENSOR:
522        case HAL_PIXEL_FORMAT_RGB_565:
523            // Single plane 16bpp data.
524            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
525            pixelStride = 2;
526            break;
527        case HAL_PIXEL_FORMAT_RGBA_8888:
528        case HAL_PIXEL_FORMAT_RGBX_8888:
529            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
530            pixelStride = 4;
531            break;
532        case HAL_PIXEL_FORMAT_RGB_888:
533            // Single plane, 24bpp.
534            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
535            pixelStride = 3;
536            break;
537        default:
538            jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
539                                 "Pixel format: 0x%x is unsupported", fmt);
540            break;
541    }
542
543    return pixelStride;
544}
545
546static jint Image_imageGetRowStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx,
547        int32_t readerFormat)
548{
549    ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
550    ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0));
551
552    int rowStride = 0;
553    ALOG_ASSERT(buffer != NULL, "buffer is NULL");
554
555    int32_t fmt = buffer->flexFormat;
556
557    fmt = applyFormatOverrides(fmt, readerFormat);
558
559    switch (fmt) {
560        case HAL_PIXEL_FORMAT_YCbCr_420_888:
561            rowStride = (idx == 0) ? buffer->stride : buffer->chromaStride;
562            break;
563        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
564            rowStride = buffer->width;
565            break;
566        case HAL_PIXEL_FORMAT_YV12:
567            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
568                                "Stride is not 16 pixel aligned %d", buffer->stride);
569            rowStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16);
570            break;
571        case HAL_PIXEL_FORMAT_BLOB:
572            // Blob is used for JPEG data, RAW10 is used for 10-bit raw data, they are
573            // single plane, row and pixel strides are 0.
574            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
575            rowStride = 0;
576            break;
577        case HAL_PIXEL_FORMAT_RAW10:
578            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
579            rowStride = buffer->stride;
580            break;
581        case HAL_PIXEL_FORMAT_Y8:
582            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
583            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
584                                "Stride is not 16 pixel aligned %d", buffer->stride);
585            rowStride = buffer->stride;
586            break;
587        case HAL_PIXEL_FORMAT_Y16:
588        case HAL_PIXEL_FORMAT_RAW_SENSOR:
589            // In native side, strides are specified in pixels, not in bytes.
590            // Single plane 16bpp bayer data. even width/height,
591            // row stride multiple of 16 pixels (32 bytes)
592            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
593            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
594                                "Stride is not 16 pixel aligned %d", buffer->stride);
595            rowStride = buffer->stride * 2;
596            break;
597        case HAL_PIXEL_FORMAT_RGB_565:
598            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
599            rowStride = buffer->stride * 2;
600            break;
601        case HAL_PIXEL_FORMAT_RGBA_8888:
602        case HAL_PIXEL_FORMAT_RGBX_8888:
603            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
604            rowStride = buffer->stride * 4;
605            break;
606        case HAL_PIXEL_FORMAT_RGB_888:
607            // Single plane, 24bpp.
608            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
609            rowStride = buffer->stride * 3;
610            break;
611        default:
612            ALOGE("%s Pixel format: 0x%x is unsupported", __FUNCTION__, fmt);
613            jniThrowException(env, "java/lang/UnsupportedOperationException",
614                              "unsupported buffer format");
615          break;
616    }
617
618    return rowStride;
619}
620
621static int Image_getBufferWidth(CpuConsumer::LockedBuffer* buffer) {
622    if (buffer == NULL) return -1;
623
624    if (!buffer->crop.isEmpty()) {
625        return buffer->crop.getWidth();
626    }
627    return buffer->width;
628}
629
630static int Image_getBufferHeight(CpuConsumer::LockedBuffer* buffer) {
631    if (buffer == NULL) return -1;
632
633    if (!buffer->crop.isEmpty()) {
634        return buffer->crop.getHeight();
635    }
636    return buffer->height;
637}
638
639// ----------------------------------------------------------------------------
640
641static void ImageReader_classInit(JNIEnv* env, jclass clazz)
642{
643    ALOGV("%s:", __FUNCTION__);
644
645    jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage");
646    LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
647                        "can't find android/graphics/ImageReader$SurfaceImage");
648    gSurfaceImageClassInfo.mLockedBuffer = env->GetFieldID(
649            imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J");
650    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mLockedBuffer == NULL,
651                        "can't find android/graphics/ImageReader.%s",
652                        ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID);
653
654    gSurfaceImageClassInfo.mTimestamp = env->GetFieldID(
655            imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J");
656    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL,
657                        "can't find android/graphics/ImageReader.%s",
658                        ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID);
659
660    gImageReaderClassInfo.mNativeContext = env->GetFieldID(
661            clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J");
662    LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL,
663                        "can't find android/graphics/ImageReader.%s",
664                          ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID);
665
666    gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(
667            clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
668    LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL,
669                        "can't find android/graphics/ImageReader.postEventFromNative");
670
671    jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane");
672    LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
673    // FindClass only gives a local reference of jclass object.
674    gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
675    gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
676            "(Landroid/media/ImageReader$SurfaceImage;III)V");
677    LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
678            "Can not find SurfacePlane constructor");
679}
680
681static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
682                             jint width, jint height, jint format, jint maxImages)
683{
684    status_t res;
685    int nativeFormat;
686
687    ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
688          __FUNCTION__, width, height, format, maxImages);
689
690    nativeFormat = Image_getPixelFormat(env, format);
691
692    sp<IGraphicBufferProducer> gbProducer;
693    sp<IGraphicBufferConsumer> gbConsumer;
694    BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
695    sp<CpuConsumer> consumer = new CpuConsumer(gbConsumer, maxImages,
696                                               /*controlledByApp*/true);
697    // TODO: throw dvm exOutOfMemoryError?
698    if (consumer == NULL) {
699        jniThrowRuntimeException(env, "Failed to allocate native CpuConsumer");
700        return;
701    }
702
703    jclass clazz = env->GetObjectClass(thiz);
704    if (clazz == NULL) {
705        jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader");
706        return;
707    }
708    sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
709    ctx->setCpuConsumer(consumer);
710    ctx->setProducer(gbProducer);
711    consumer->setFrameAvailableListener(ctx);
712    ImageReader_setNativeContext(env, thiz, ctx);
713    ctx->setBufferFormat(nativeFormat);
714    ctx->setBufferWidth(width);
715    ctx->setBufferHeight(height);
716
717    // Set the width/height/format to the CpuConsumer
718    res = consumer->setDefaultBufferSize(width, height);
719    if (res != OK) {
720        jniThrowException(env, "java/lang/IllegalStateException",
721                          "Failed to set CpuConsumer buffer size");
722        return;
723    }
724    res = consumer->setDefaultBufferFormat(nativeFormat);
725    if (res != OK) {
726        jniThrowException(env, "java/lang/IllegalStateException",
727                          "Failed to set CpuConsumer buffer format");
728    }
729}
730
731static void ImageReader_close(JNIEnv* env, jobject thiz)
732{
733    ALOGV("%s:", __FUNCTION__);
734
735    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
736    if (ctx == NULL) {
737        // ImageReader is already closed.
738        return;
739    }
740
741    CpuConsumer* consumer = ImageReader_getCpuConsumer(env, thiz);
742    if (consumer != NULL) {
743        consumer->abandon();
744        consumer->setFrameAvailableListener(NULL);
745    }
746    ImageReader_setNativeContext(env, thiz, NULL);
747}
748
749static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image)
750{
751    ALOGV("%s:", __FUNCTION__);
752    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
753    if (ctx == NULL) {
754        ALOGW("ImageReader#close called before Image#close, consider calling Image#close first");
755        return;
756    }
757
758    CpuConsumer* consumer = ctx->getCpuConsumer();
759    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image);
760    if (!buffer) {
761        ALOGW("Image already released!!!");
762        return;
763    }
764    consumer->unlockBuffer(*buffer);
765    Image_setBuffer(env, image, NULL);
766    ctx->returnLockedBuffer(buffer);
767}
768
769static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz,
770                                             jobject image)
771{
772    ALOGV("%s:", __FUNCTION__);
773    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
774    if (ctx == NULL) {
775        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
776        return -1;
777    }
778
779    CpuConsumer* consumer = ctx->getCpuConsumer();
780    CpuConsumer::LockedBuffer* buffer = ctx->getLockedBuffer();
781    if (buffer == NULL) {
782        ALOGW("Unable to acquire a lockedBuffer, very likely client tries to lock more than"
783            " maxImages buffers");
784        return ACQUIRE_MAX_IMAGES;
785    }
786    status_t res = consumer->lockNextBuffer(buffer);
787    if (res != NO_ERROR) {
788        ctx->returnLockedBuffer(buffer);
789        if (res != BAD_VALUE /*no buffers*/) {
790            if (res == NOT_ENOUGH_DATA) {
791                return ACQUIRE_MAX_IMAGES;
792            } else {
793                ALOGE("%s Fail to lockNextBuffer with error: %d ",
794                      __FUNCTION__, res);
795                jniThrowExceptionFmt(env, "java/lang/AssertionError",
796                          "Unknown error (%d) when we tried to lock buffer.",
797                          res);
798            }
799        }
800        return ACQUIRE_NO_BUFFERS;
801    }
802
803    if (buffer->flexFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
804        jniThrowException(env, "java/lang/UnsupportedOperationException",
805                "NV21 format is not supported by ImageReader");
806        return -1;
807    }
808
809    // Check if the left-top corner of the crop rect is origin, we currently assume this point is
810    // zero, will revist this once this assumption turns out problematic.
811    Point lt = buffer->crop.leftTop();
812    if (lt.x != 0 || lt.y != 0) {
813        jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
814                "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
815        return -1;
816    }
817
818    // Check if the producer buffer configurations match what ImageReader configured.
819    int outputWidth = Image_getBufferWidth(buffer);
820    int outputHeight = Image_getBufferHeight(buffer);
821
822    int imgReaderFmt = ctx->getBufferFormat();
823    int imageReaderWidth = ctx->getBufferWidth();
824    int imageReaderHeight = ctx->getBufferHeight();
825    if ((buffer->format != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) &&
826            (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) {
827        ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d",
828                __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight);
829    }
830
831    int bufFmt = buffer->format;
832    if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888) {
833        bufFmt = buffer->flexFormat;
834    }
835    if (imgReaderFmt != bufFmt) {
836        if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && (bufFmt ==
837                HAL_PIXEL_FORMAT_YCrCb_420_SP || bufFmt == HAL_PIXEL_FORMAT_YV12)) {
838            // Special casing for when producer switches to a format compatible with flexible YUV
839            // (HAL_PIXEL_FORMAT_YCbCr_420_888).
840            ctx->setBufferFormat(bufFmt);
841            ALOGD("%s: Overriding buffer format YUV_420_888 to %x.", __FUNCTION__, bufFmt);
842        } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB && bufFmt == HAL_PIXEL_FORMAT_RGBA_8888) {
843            // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW
844            // write limitations for (b/17379185).
845            ALOGD("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__);
846        } else {
847            // Return the buffer to the queue.
848            consumer->unlockBuffer(*buffer);
849            ctx->returnLockedBuffer(buffer);
850
851            // Throw exception
852            ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
853                    buffer->format, ctx->getBufferFormat());
854            String8 msg;
855            msg.appendFormat("The producer output buffer format 0x%x doesn't "
856                    "match the ImageReader's configured buffer format 0x%x.",
857                    bufFmt, ctx->getBufferFormat());
858            jniThrowException(env, "java/lang/UnsupportedOperationException",
859                    msg.string());
860            return -1;
861        }
862    }
863    // Set SurfaceImage instance member variables
864    Image_setBuffer(env, image, buffer);
865    env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp,
866            static_cast<jlong>(buffer->timestamp));
867
868    return ACQUIRE_SUCCESS;
869}
870
871static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
872{
873    ALOGV("%s: ", __FUNCTION__);
874
875    IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz);
876    if (gbp == NULL) {
877        jniThrowRuntimeException(env, "CpuConsumer is uninitialized");
878        return NULL;
879    }
880
881    // Wrap the IGBP in a Java-language Surface.
882    return android_view_Surface_createFromIGraphicBufferProducer(env, gbp);
883}
884
885static jobject Image_createSurfacePlane(JNIEnv* env, jobject thiz, int idx, int readerFormat)
886{
887    int rowStride, pixelStride;
888    ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
889
890    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
891
892    ALOG_ASSERT(buffer != NULL);
893    if (buffer == NULL) {
894        jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
895    }
896
897    readerFormat = Image_getPixelFormat(env, readerFormat);
898
899    rowStride = Image_imageGetRowStride(env, buffer, idx, readerFormat);
900    pixelStride = Image_imageGetPixelStride(env, buffer, idx, readerFormat);
901
902    jobject surfPlaneObj = env->NewObject(gSurfacePlaneClassInfo.clazz,
903            gSurfacePlaneClassInfo.ctor, thiz, idx, rowStride, pixelStride);
904
905    return surfPlaneObj;
906}
907
908static jobject Image_getByteBuffer(JNIEnv* env, jobject thiz, int idx, int readerFormat)
909{
910    uint8_t *base = NULL;
911    uint32_t size = 0;
912    jobject byteBuffer;
913
914    ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
915
916    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
917
918    if (buffer == NULL) {
919        jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
920    }
921
922    readerFormat = Image_getPixelFormat(env, readerFormat);
923
924    // Create byteBuffer from native buffer
925    Image_getLockedBufferInfo(env, buffer, idx, &base, &size, readerFormat);
926
927    if (size > static_cast<uint32_t>(INT32_MAX)) {
928        // Byte buffer have 'int capacity', so check the range
929        jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
930                "Size too large for bytebuffer capacity %" PRIu32, size);
931        return NULL;
932    }
933
934    byteBuffer = env->NewDirectByteBuffer(base, size);
935    // TODO: throw dvm exOutOfMemoryError?
936    if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
937        jniThrowException(env, "java/lang/IllegalStateException", "Failed to allocate ByteBuffer");
938    }
939
940    return byteBuffer;
941}
942
943static jint Image_getWidth(JNIEnv* env, jobject thiz)
944{
945    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
946    return Image_getBufferWidth(buffer);
947}
948
949static jint Image_getHeight(JNIEnv* env, jobject thiz)
950{
951    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
952    return Image_getBufferHeight(buffer);
953}
954
955
956} // extern "C"
957
958// ----------------------------------------------------------------------------
959
960static JNINativeMethod gImageReaderMethods[] = {
961    {"nativeClassInit",        "()V",                        (void*)ImageReader_classInit },
962    {"nativeInit",             "(Ljava/lang/Object;IIII)V",  (void*)ImageReader_init },
963    {"nativeClose",            "()V",                        (void*)ImageReader_close },
964    {"nativeReleaseImage",     "(Landroid/media/Image;)V",   (void*)ImageReader_imageRelease },
965    {"nativeImageSetup",       "(Landroid/media/Image;)I",   (void*)ImageReader_imageSetup },
966    {"nativeGetSurface",       "()Landroid/view/Surface;",   (void*)ImageReader_getSurface },
967};
968
969static JNINativeMethod gImageMethods[] = {
970    {"nativeImageGetBuffer",   "(II)Ljava/nio/ByteBuffer;",   (void*)Image_getByteBuffer },
971    {"nativeCreatePlane",      "(II)Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",
972                                                              (void*)Image_createSurfacePlane },
973    {"nativeGetWidth",         "()I",                         (void*)Image_getWidth },
974    {"nativeGetHeight",        "()I",                         (void*)Image_getHeight },
975};
976
977int register_android_media_ImageReader(JNIEnv *env) {
978
979    int ret1 = AndroidRuntime::registerNativeMethods(env,
980                   "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods));
981
982    int ret2 = AndroidRuntime::registerNativeMethods(env,
983                   "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods));
984
985    return (ret1 || ret2);
986}
987