android_media_ImageReader.cpp revision 37ec99b19ce7a5d40d16ccc2f578aca66bd7a25d 
1/*
2 * Copyright 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "ImageReader_JNI"
19#include <utils/Log.h>
20#include <utils/misc.h>
21#include <utils/List.h>
22#include <utils/String8.h>
23
24#include <cstdio>
25
26#include <gui/CpuConsumer.h>
27#include <gui/Surface.h>
28#include <camera3.h>
29
30#include <android_runtime/AndroidRuntime.h>
31#include <android_runtime/android_view_Surface.h>
32
33#include <jni.h>
34#include <JNIHelp.h>
35
36#include <stdint.h>
37#include <inttypes.h>
38
39#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
40
41#define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID       "mNativeContext"
42#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID   "mLockedBuffer"
43#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID       "mTimestamp"
44
45// ----------------------------------------------------------------------------
46
47using namespace android;
48
49enum {
50    IMAGE_READER_MAX_NUM_PLANES = 3,
51};
52
53enum {
54    ACQUIRE_SUCCESS = 0,
55    ACQUIRE_NO_BUFFERS = 1,
56    ACQUIRE_MAX_IMAGES = 2,
57};
58
59static struct {
60    jfieldID mNativeContext;
61    jmethodID postEventFromNative;
62} gImageReaderClassInfo;
63
64static struct {
65    jfieldID mLockedBuffer;
66    jfieldID mTimestamp;
67} gSurfaceImageClassInfo;
68
69static struct {
70    jclass clazz;
71    jmethodID ctor;
72} gSurfacePlaneClassInfo;
73
74// ----------------------------------------------------------------------------
75
76class JNIImageReaderContext : public CpuConsumer::FrameAvailableListener
77{
78public:
79    JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages);
80
81    virtual ~JNIImageReaderContext();
82
83    virtual void onFrameAvailable(const BufferItem& item);
84
85    CpuConsumer::LockedBuffer* getLockedBuffer();
86
87    void returnLockedBuffer(CpuConsumer::LockedBuffer* buffer);
88
89    void setCpuConsumer(const sp<CpuConsumer>& consumer) { mConsumer = consumer; }
90    CpuConsumer* getCpuConsumer() { return mConsumer.get(); }
91
92    void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; }
93    IGraphicBufferProducer* getProducer() { return mProducer.get(); }
94
95    void setBufferFormat(int format) { mFormat = format; }
96    int getBufferFormat() { return mFormat; }
97
98    void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; }
99    android_dataspace getBufferDataspace() { return mDataSpace; }
100
101    void setBufferWidth(int width) { mWidth = width; }
102    int getBufferWidth() { return mWidth; }
103
104    void setBufferHeight(int height) { mHeight = height; }
105    int getBufferHeight() { return mHeight; }
106
107private:
108    static JNIEnv* getJNIEnv(bool* needsDetach);
109    static void detachJNI();
110
111    List<CpuConsumer::LockedBuffer*> mBuffers;
112    sp<CpuConsumer> mConsumer;
113    sp<IGraphicBufferProducer> mProducer;
114    jobject mWeakThiz;
115    jclass mClazz;
116    int mFormat;
117    android_dataspace mDataSpace;
118    int mWidth;
119    int mHeight;
120};
121
122JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env,
123        jobject weakThiz, jclass clazz, int maxImages) :
124    mWeakThiz(env->NewGlobalRef(weakThiz)),
125    mClazz((jclass)env->NewGlobalRef(clazz)) {
126    for (int i = 0; i < maxImages; i++) {
127        CpuConsumer::LockedBuffer *buffer = new CpuConsumer::LockedBuffer;
128        mBuffers.push_back(buffer);
129    }
130}
131
132JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) {
133    LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
134    *needsDetach = false;
135    JNIEnv* env = AndroidRuntime::getJNIEnv();
136    if (env == NULL) {
137        JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
138        JavaVM* vm = AndroidRuntime::getJavaVM();
139        int result = vm->AttachCurrentThread(&env, (void*) &args);
140        if (result != JNI_OK) {
141            ALOGE("thread attach failed: %#x", result);
142            return NULL;
143        }
144        *needsDetach = true;
145    }
146    return env;
147}
148
149void JNIImageReaderContext::detachJNI() {
150    JavaVM* vm = AndroidRuntime::getJavaVM();
151    int result = vm->DetachCurrentThread();
152    if (result != JNI_OK) {
153        ALOGE("thread detach failed: %#x", result);
154    }
155}
156
157CpuConsumer::LockedBuffer* JNIImageReaderContext::getLockedBuffer() {
158    if (mBuffers.empty()) {
159        return NULL;
160    }
161    // Return a LockedBuffer pointer and remove it from the list
162    List<CpuConsumer::LockedBuffer*>::iterator it = mBuffers.begin();
163    CpuConsumer::LockedBuffer* buffer = *it;
164    mBuffers.erase(it);
165    return buffer;
166}
167
168void JNIImageReaderContext::returnLockedBuffer(CpuConsumer::LockedBuffer* buffer) {
169    mBuffers.push_back(buffer);
170}
171
172JNIImageReaderContext::~JNIImageReaderContext() {
173    bool needsDetach = false;
174    JNIEnv* env = getJNIEnv(&needsDetach);
175    if (env != NULL) {
176        env->DeleteGlobalRef(mWeakThiz);
177        env->DeleteGlobalRef(mClazz);
178    } else {
179        ALOGW("leaking JNI object references");
180    }
181    if (needsDetach) {
182        detachJNI();
183    }
184
185    // Delete LockedBuffers
186    for (List<CpuConsumer::LockedBuffer *>::iterator it = mBuffers.begin();
187            it != mBuffers.end(); it++) {
188        delete *it;
189    }
190    mBuffers.clear();
191    mConsumer.clear();
192}
193
194void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
195{
196    ALOGV("%s: frame available", __FUNCTION__);
197    bool needsDetach = false;
198    JNIEnv* env = getJNIEnv(&needsDetach);
199    if (env != NULL) {
200        env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
201    } else {
202        ALOGW("onFrameAvailable event will not posted");
203    }
204    if (needsDetach) {
205        detachJNI();
206    }
207}
208
209// ----------------------------------------------------------------------------
210
211extern "C" {
212
213static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz)
214{
215    JNIImageReaderContext *ctx;
216    ctx = reinterpret_cast<JNIImageReaderContext *>
217              (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext));
218    return ctx;
219}
220
221static CpuConsumer* ImageReader_getCpuConsumer(JNIEnv* env, jobject thiz)
222{
223    ALOGV("%s:", __FUNCTION__);
224    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
225    if (ctx == NULL) {
226        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
227        return NULL;
228    }
229    return ctx->getCpuConsumer();
230}
231
232static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz)
233{
234    ALOGV("%s:", __FUNCTION__);
235    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
236    if (ctx == NULL) {
237        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
238        return NULL;
239    }
240    return ctx->getProducer();
241}
242
243static void ImageReader_setNativeContext(JNIEnv* env,
244        jobject thiz, sp<JNIImageReaderContext> ctx)
245{
246    ALOGV("%s:", __FUNCTION__);
247    JNIImageReaderContext* const p = ImageReader_getContext(env, thiz);
248    if (ctx != 0) {
249        ctx->incStrong((void*)ImageReader_setNativeContext);
250    }
251    if (p) {
252        p->decStrong((void*)ImageReader_setNativeContext);
253    }
254    env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext,
255            reinterpret_cast<jlong>(ctx.get()));
256}
257
258static CpuConsumer::LockedBuffer* Image_getLockedBuffer(JNIEnv* env, jobject image)
259{
260    return reinterpret_cast<CpuConsumer::LockedBuffer*>(
261            env->GetLongField(image, gSurfaceImageClassInfo.mLockedBuffer));
262}
263
264static void Image_setBuffer(JNIEnv* env, jobject thiz,
265        const CpuConsumer::LockedBuffer* buffer)
266{
267    env->SetLongField(thiz, gSurfaceImageClassInfo.mLockedBuffer, reinterpret_cast<jlong>(buffer));
268}
269
270static uint32_t Image_getJpegSize(CpuConsumer::LockedBuffer* buffer, bool usingRGBAOverride)
271{
272    ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
273    uint32_t size = 0;
274    uint32_t width = buffer->width;
275    uint8_t* jpegBuffer = buffer->data;
276
277    if (usingRGBAOverride) {
278        width = (buffer->width + buffer->stride * (buffer->height - 1)) * 4;
279    }
280
281    // First check for JPEG transport header at the end of the buffer
282    uint8_t* header = jpegBuffer + (width - sizeof(struct camera3_jpeg_blob));
283    struct camera3_jpeg_blob *blob = (struct camera3_jpeg_blob*)(header);
284    if (blob->jpeg_blob_id == CAMERA3_JPEG_BLOB_ID) {
285        size = blob->jpeg_size;
286        ALOGV("%s: Jpeg size = %d", __FUNCTION__, size);
287    }
288
289    // failed to find size, default to whole buffer
290    if (size == 0) {
291        /*
292         * This is a problem because not including the JPEG header
293         * means that in certain rare situations a regular JPEG blob
294         * will be misidentified as having a header, in which case
295         * we will get a garbage size value.
296         */
297        ALOGW("%s: No JPEG header detected, defaulting to size=width=%d",
298                __FUNCTION__, width);
299        size = width;
300    }
301
302    return size;
303}
304
305static bool usingRGBAToJpegOverride(int32_t bufferFormat, int32_t readerCtxFormat) {
306    return readerCtxFormat == HAL_PIXEL_FORMAT_BLOB && bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888;
307}
308
309static int32_t applyFormatOverrides(int32_t bufferFormat, int32_t readerCtxFormat)
310{
311    // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW
312    // write limitations for some platforms (b/17379185).
313    if (usingRGBAToJpegOverride(bufferFormat, readerCtxFormat)) {
314        return HAL_PIXEL_FORMAT_BLOB;
315    }
316    return bufferFormat;
317}
318
319static void Image_getLockedBufferInfo(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx,
320                                uint8_t **base, uint32_t *size, int32_t readerFormat)
321{
322    ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
323    ALOG_ASSERT(base != NULL, "base is NULL!!!");
324    ALOG_ASSERT(size != NULL, "size is NULL!!!");
325    ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0));
326
327    ALOGV("%s: buffer: %p", __FUNCTION__, buffer);
328
329    uint32_t dataSize, ySize, cSize, cStride;
330    uint8_t *cb, *cr;
331    uint8_t *pData = NULL;
332    int bytesPerPixel = 0;
333
334    dataSize = ySize = cSize = cStride = 0;
335    int32_t fmt = buffer->flexFormat;
336
337    bool usingRGBAOverride = usingRGBAToJpegOverride(fmt, readerFormat);
338    fmt = applyFormatOverrides(fmt, readerFormat);
339    switch (fmt) {
340        case HAL_PIXEL_FORMAT_YCbCr_420_888:
341            pData =
342                (idx == 0) ?
343                    buffer->data :
344                (idx == 1) ?
345                    buffer->dataCb :
346                buffer->dataCr;
347            // only map until last pixel
348            if (idx == 0) {
349                dataSize = buffer->stride * (buffer->height - 1) + buffer->width;
350            } else {
351                dataSize = buffer->chromaStride * (buffer->height / 2 - 1) +
352                        buffer->chromaStep * (buffer->width / 2 - 1) + 1;
353            }
354            break;
355        // NV21
356        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
357            cr = buffer->data + (buffer->stride * buffer->height);
358            cb = cr + 1;
359            // only map until last pixel
360            ySize = buffer->width * (buffer->height - 1) + buffer->width;
361            cSize = buffer->width * (buffer->height / 2 - 1) + buffer->width - 1;
362
363            pData =
364                (idx == 0) ?
365                    buffer->data :
366                (idx == 1) ?
367                    cb:
368                cr;
369
370            dataSize = (idx == 0) ? ySize : cSize;
371            break;
372        case HAL_PIXEL_FORMAT_YV12:
373            // Y and C stride need to be 16 pixel aligned.
374            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
375                                "Stride is not 16 pixel aligned %d", buffer->stride);
376
377            ySize = buffer->stride * buffer->height;
378            cStride = ALIGN(buffer->stride / 2, 16);
379            cr = buffer->data + ySize;
380            cSize = cStride * buffer->height / 2;
381            cb = cr + cSize;
382
383            pData =
384                (idx == 0) ?
385                    buffer->data :
386                (idx == 1) ?
387                    cb :
388                cr;
389            dataSize = (idx == 0) ? ySize : cSize;
390            break;
391        case HAL_PIXEL_FORMAT_Y8:
392            // Single plane, 8bpp.
393            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
394
395            pData = buffer->data;
396            dataSize = buffer->stride * buffer->height;
397            break;
398        case HAL_PIXEL_FORMAT_Y16:
399            bytesPerPixel = 2;
400            // Single plane, 16bpp, strides are specified in pixels, not in bytes
401            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
402
403            pData = buffer->data;
404            dataSize = buffer->stride * buffer->height * bytesPerPixel;
405            break;
406        case HAL_PIXEL_FORMAT_BLOB:
407            // Used for JPEG data, height must be 1, width == size, single plane.
408            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
409            ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height);
410
411            pData = buffer->data;
412            dataSize = Image_getJpegSize(buffer, usingRGBAOverride);
413            break;
414        case HAL_PIXEL_FORMAT_RAW16:
415            // Single plane 16bpp bayer data.
416            bytesPerPixel = 2;
417            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
418            pData = buffer->data;
419            dataSize = buffer->stride * buffer->height * bytesPerPixel;
420            break;
421        case HAL_PIXEL_FORMAT_RAW10:
422            // Single plane 10bpp bayer data.
423            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
424            LOG_ALWAYS_FATAL_IF(buffer->width % 4,
425                                "Width is not multiple of 4 %d", buffer->width);
426            LOG_ALWAYS_FATAL_IF(buffer->height % 2,
427                                "Height is not even %d", buffer->height);
428            LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 10 / 8),
429                                "stride (%d) should be at least %d",
430                                buffer->stride, buffer->width * 10 / 8);
431            pData = buffer->data;
432            dataSize = buffer->stride * buffer->height;
433            break;
434        case HAL_PIXEL_FORMAT_RAW12:
435            // Single plane 10bpp bayer data.
436            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
437            LOG_ALWAYS_FATAL_IF(buffer->width % 4,
438                                "Width is not multiple of 4 %d", buffer->width);
439            LOG_ALWAYS_FATAL_IF(buffer->height % 2,
440                                "Height is not even %d", buffer->height);
441            LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 12 / 8),
442                                "stride (%d) should be at least %d",
443                                buffer->stride, buffer->width * 12 / 8);
444            pData = buffer->data;
445            dataSize = buffer->stride * buffer->height;
446            break;
447        case HAL_PIXEL_FORMAT_RGBA_8888:
448        case HAL_PIXEL_FORMAT_RGBX_8888:
449            // Single plane, 32bpp.
450            bytesPerPixel = 4;
451            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
452            pData = buffer->data;
453            dataSize = buffer->stride * buffer->height * bytesPerPixel;
454            break;
455        case HAL_PIXEL_FORMAT_RGB_565:
456            // Single plane, 16bpp.
457            bytesPerPixel = 2;
458            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
459            pData = buffer->data;
460            dataSize = buffer->stride * buffer->height * bytesPerPixel;
461            break;
462        case HAL_PIXEL_FORMAT_RGB_888:
463            // Single plane, 24bpp.
464            bytesPerPixel = 3;
465            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
466            pData = buffer->data;
467            dataSize = buffer->stride * buffer->height * bytesPerPixel;
468            break;
469        default:
470            jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
471                                 "Pixel format: 0x%x is unsupported", fmt);
472            break;
473    }
474
475    *base = pData;
476    *size = dataSize;
477}
478
479static jint Image_imageGetPixelStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx,
480        int32_t halReaderFormat)
481{
482    ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
483    ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0), "Index is out of range:%d", idx);
484
485    int pixelStride = 0;
486    ALOG_ASSERT(buffer != NULL, "buffer is NULL");
487
488    int32_t fmt = buffer->flexFormat;
489
490    fmt = applyFormatOverrides(fmt, halReaderFormat);
491
492    switch (fmt) {
493        case HAL_PIXEL_FORMAT_YCbCr_420_888:
494            pixelStride = (idx == 0) ? 1 : buffer->chromaStep;
495            break;
496        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
497            pixelStride = (idx == 0) ? 1 : 2;
498            break;
499        case HAL_PIXEL_FORMAT_Y8:
500            // Single plane 8bpp data.
501            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
502            break;
503        case HAL_PIXEL_FORMAT_YV12:
504            pixelStride = 1;
505            break;
506        case HAL_PIXEL_FORMAT_BLOB:
507        case HAL_PIXEL_FORMAT_RAW10:
508        case HAL_PIXEL_FORMAT_RAW12:
509            // Blob is used for JPEG data, RAW10 and RAW12 is used for 10-bit and 12-bit raw data,
510            // those are single plane data with pixel stride 0 since they don't really have a
511            // well defined pixel stride
512            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
513            pixelStride = 0;
514            break;
515        case HAL_PIXEL_FORMAT_Y16:
516        case HAL_PIXEL_FORMAT_RAW16:
517        case HAL_PIXEL_FORMAT_RGB_565:
518            // Single plane 16bpp data.
519            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
520            pixelStride = 2;
521            break;
522        case HAL_PIXEL_FORMAT_RGBA_8888:
523        case HAL_PIXEL_FORMAT_RGBX_8888:
524            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
525            pixelStride = 4;
526            break;
527        case HAL_PIXEL_FORMAT_RGB_888:
528            // Single plane, 24bpp.
529            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
530            pixelStride = 3;
531            break;
532        default:
533            jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
534                                 "Pixel format: 0x%x is unsupported", fmt);
535            break;
536    }
537
538    return pixelStride;
539}
540
541static jint Image_imageGetRowStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx,
542        int32_t halReaderFormat)
543{
544    ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
545    ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0));
546
547    int rowStride = 0;
548    ALOG_ASSERT(buffer != NULL, "buffer is NULL");
549
550    int32_t fmt = buffer->flexFormat;
551
552    fmt = applyFormatOverrides(fmt, halReaderFormat);
553
554    switch (fmt) {
555        case HAL_PIXEL_FORMAT_YCbCr_420_888:
556            rowStride = (idx == 0) ? buffer->stride : buffer->chromaStride;
557            break;
558        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
559            rowStride = buffer->width;
560            break;
561        case HAL_PIXEL_FORMAT_YV12:
562            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
563                                "Stride is not 16 pixel aligned %d", buffer->stride);
564            rowStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16);
565            break;
566        case HAL_PIXEL_FORMAT_BLOB:
567            // Blob is used for JPEG data. It is single plane and has 0 row stride and
568            // 0 pixel stride
569            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
570            rowStride = 0;
571            break;
572        case HAL_PIXEL_FORMAT_RAW10:
573        case HAL_PIXEL_FORMAT_RAW12:
574            // RAW10 and RAW12 are used for 10-bit and 12-bit raw data, they are single plane
575            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
576            rowStride = buffer->stride;
577            break;
578        case HAL_PIXEL_FORMAT_Y8:
579            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
580            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
581                                "Stride is not 16 pixel aligned %d", buffer->stride);
582            rowStride = buffer->stride;
583            break;
584        case HAL_PIXEL_FORMAT_Y16:
585        case HAL_PIXEL_FORMAT_RAW16:
586            // In native side, strides are specified in pixels, not in bytes.
587            // Single plane 16bpp bayer data. even width/height,
588            // row stride multiple of 16 pixels (32 bytes)
589            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
590            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
591                                "Stride is not 16 pixel aligned %d", buffer->stride);
592            rowStride = buffer->stride * 2;
593            break;
594        case HAL_PIXEL_FORMAT_RGB_565:
595            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
596            rowStride = buffer->stride * 2;
597            break;
598        case HAL_PIXEL_FORMAT_RGBA_8888:
599        case HAL_PIXEL_FORMAT_RGBX_8888:
600            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
601            rowStride = buffer->stride * 4;
602            break;
603        case HAL_PIXEL_FORMAT_RGB_888:
604            // Single plane, 24bpp.
605            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
606            rowStride = buffer->stride * 3;
607            break;
608        default:
609            ALOGE("%s Pixel format: 0x%x is unsupported", __FUNCTION__, fmt);
610            jniThrowException(env, "java/lang/UnsupportedOperationException",
611                              "unsupported buffer format");
612          break;
613    }
614
615    return rowStride;
616}
617
618static int Image_getBufferWidth(CpuConsumer::LockedBuffer* buffer) {
619    if (buffer == NULL) return -1;
620
621    if (!buffer->crop.isEmpty()) {
622        return buffer->crop.getWidth();
623    }
624    return buffer->width;
625}
626
627static int Image_getBufferHeight(CpuConsumer::LockedBuffer* buffer) {
628    if (buffer == NULL) return -1;
629
630    if (!buffer->crop.isEmpty()) {
631        return buffer->crop.getHeight();
632    }
633    return buffer->height;
634}
635
636// ----------------------------------------------------------------------------
637
638static void ImageReader_classInit(JNIEnv* env, jclass clazz)
639{
640    ALOGV("%s:", __FUNCTION__);
641
642    jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage");
643    LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
644                        "can't find android/graphics/ImageReader$SurfaceImage");
645    gSurfaceImageClassInfo.mLockedBuffer = env->GetFieldID(
646            imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J");
647    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mLockedBuffer == NULL,
648                        "can't find android/graphics/ImageReader.%s",
649                        ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID);
650
651    gSurfaceImageClassInfo.mTimestamp = env->GetFieldID(
652            imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J");
653    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL,
654                        "can't find android/graphics/ImageReader.%s",
655                        ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID);
656
657    gImageReaderClassInfo.mNativeContext = env->GetFieldID(
658            clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J");
659    LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL,
660                        "can't find android/graphics/ImageReader.%s",
661                          ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID);
662
663    gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(
664            clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
665    LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL,
666                        "can't find android/graphics/ImageReader.postEventFromNative");
667
668    jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane");
669    LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
670    // FindClass only gives a local reference of jclass object.
671    gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
672    gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
673            "(Landroid/media/ImageReader$SurfaceImage;III)V");
674    LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
675            "Can not find SurfacePlane constructor");
676}
677
678static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
679                             jint width, jint height, jint format, jint maxImages)
680{
681    status_t res;
682    int nativeFormat;
683    android_dataspace nativeDataspace;
684
685    ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
686          __FUNCTION__, width, height, format, maxImages);
687
688    PublicFormat publicFormat = static_cast<PublicFormat>(format);
689    nativeFormat = android_view_Surface_mapPublicFormatToHalFormat(
690        publicFormat);
691    nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace(
692        publicFormat);
693
694    sp<IGraphicBufferProducer> gbProducer;
695    sp<IGraphicBufferConsumer> gbConsumer;
696    BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
697    sp<CpuConsumer> consumer = new CpuConsumer(gbConsumer, maxImages,
698                                               /*controlledByApp*/true);
699    // TODO: throw dvm exOutOfMemoryError?
700    if (consumer == NULL) {
701        jniThrowRuntimeException(env, "Failed to allocate native CpuConsumer");
702        return;
703    }
704
705    jclass clazz = env->GetObjectClass(thiz);
706    if (clazz == NULL) {
707        jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader");
708        return;
709    }
710    sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
711    ctx->setCpuConsumer(consumer);
712    ctx->setProducer(gbProducer);
713    consumer->setFrameAvailableListener(ctx);
714    ImageReader_setNativeContext(env, thiz, ctx);
715    ctx->setBufferFormat(nativeFormat);
716    ctx->setBufferDataspace(nativeDataspace);
717    ctx->setBufferWidth(width);
718    ctx->setBufferHeight(height);
719
720    // Set the width/height/format/dataspace to the CpuConsumer
721    res = consumer->setDefaultBufferSize(width, height);
722    if (res != OK) {
723        jniThrowException(env, "java/lang/IllegalStateException",
724                          "Failed to set CpuConsumer buffer size");
725        return;
726    }
727    res = consumer->setDefaultBufferFormat(nativeFormat);
728    if (res != OK) {
729        jniThrowException(env, "java/lang/IllegalStateException",
730                          "Failed to set CpuConsumer buffer format");
731    }
732    res = consumer->setDefaultBufferDataSpace(nativeDataspace);
733    if (res != OK) {
734        jniThrowException(env, "java/lang/IllegalStateException",
735                          "Failed to set CpuConsumer buffer dataSpace");
736    }
737
738}
739
740static void ImageReader_close(JNIEnv* env, jobject thiz)
741{
742    ALOGV("%s:", __FUNCTION__);
743
744    JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
745    if (ctx == NULL) {
746        // ImageReader is already closed.
747        return;
748    }
749
750    CpuConsumer* consumer = ImageReader_getCpuConsumer(env, thiz);
751    if (consumer != NULL) {
752        consumer->abandon();
753        consumer->setFrameAvailableListener(NULL);
754    }
755    ImageReader_setNativeContext(env, thiz, NULL);
756}
757
758static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image)
759{
760    ALOGV("%s:", __FUNCTION__);
761    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
762    if (ctx == NULL) {
763        ALOGW("ImageReader#close called before Image#close, consider calling Image#close first");
764        return;
765    }
766
767    CpuConsumer* consumer = ctx->getCpuConsumer();
768    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image);
769    if (!buffer) {
770        ALOGW("Image already released!!!");
771        return;
772    }
773    consumer->unlockBuffer(*buffer);
774    Image_setBuffer(env, image, NULL);
775    ctx->returnLockedBuffer(buffer);
776}
777
778static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz,
779                                             jobject image)
780{
781    ALOGV("%s:", __FUNCTION__);
782    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
783    if (ctx == NULL) {
784        jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
785        return -1;
786    }
787
788    CpuConsumer* consumer = ctx->getCpuConsumer();
789    CpuConsumer::LockedBuffer* buffer = ctx->getLockedBuffer();
790    if (buffer == NULL) {
791        ALOGW("Unable to acquire a lockedBuffer, very likely client tries to lock more than"
792            " maxImages buffers");
793        return ACQUIRE_MAX_IMAGES;
794    }
795    status_t res = consumer->lockNextBuffer(buffer);
796    if (res != NO_ERROR) {
797        ctx->returnLockedBuffer(buffer);
798        if (res != BAD_VALUE /*no buffers*/) {
799            if (res == NOT_ENOUGH_DATA) {
800                return ACQUIRE_MAX_IMAGES;
801            } else {
802                ALOGE("%s Fail to lockNextBuffer with error: %d ",
803                      __FUNCTION__, res);
804                jniThrowExceptionFmt(env, "java/lang/AssertionError",
805                          "Unknown error (%d) when we tried to lock buffer.",
806                          res);
807            }
808        }
809        return ACQUIRE_NO_BUFFERS;
810    }
811
812    if (buffer->flexFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
813        jniThrowException(env, "java/lang/UnsupportedOperationException",
814                "NV21 format is not supported by ImageReader");
815        return -1;
816    }
817
818    // Check if the left-top corner of the crop rect is origin, we currently assume this point is
819    // zero, will revist this once this assumption turns out problematic.
820    Point lt = buffer->crop.leftTop();
821    if (lt.x != 0 || lt.y != 0) {
822        jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
823                "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
824        return -1;
825    }
826
827    // Check if the producer buffer configurations match what ImageReader configured.
828    int outputWidth = Image_getBufferWidth(buffer);
829    int outputHeight = Image_getBufferHeight(buffer);
830
831    int imgReaderFmt = ctx->getBufferFormat();
832    int imageReaderWidth = ctx->getBufferWidth();
833    int imageReaderHeight = ctx->getBufferHeight();
834    if ((buffer->format != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) &&
835            (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) {
836        ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d",
837                __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight);
838    }
839
840    int bufFmt = buffer->format;
841    if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888) {
842        bufFmt = buffer->flexFormat;
843    }
844    if (imgReaderFmt != bufFmt) {
845        if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && (bufFmt ==
846                HAL_PIXEL_FORMAT_YCrCb_420_SP || bufFmt == HAL_PIXEL_FORMAT_YV12)) {
847            // Special casing for when producer switches to a format compatible with flexible YUV
848            // (HAL_PIXEL_FORMAT_YCbCr_420_888).
849            ctx->setBufferFormat(bufFmt);
850            ALOGD("%s: Overriding buffer format YUV_420_888 to %x.", __FUNCTION__, bufFmt);
851        } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB && bufFmt == HAL_PIXEL_FORMAT_RGBA_8888) {
852            // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW
853            // write limitations for (b/17379185).
854            ALOGD("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__);
855        } else {
856            // Return the buffer to the queue.
857            consumer->unlockBuffer(*buffer);
858            ctx->returnLockedBuffer(buffer);
859
860            // Throw exception
861            ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
862                    buffer->format, ctx->getBufferFormat());
863            String8 msg;
864            msg.appendFormat("The producer output buffer format 0x%x doesn't "
865                    "match the ImageReader's configured buffer format 0x%x.",
866                    bufFmt, ctx->getBufferFormat());
867            jniThrowException(env, "java/lang/UnsupportedOperationException",
868                    msg.string());
869            return -1;
870        }
871    }
872    // Set SurfaceImage instance member variables
873    Image_setBuffer(env, image, buffer);
874    env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp,
875            static_cast<jlong>(buffer->timestamp));
876
877    return ACQUIRE_SUCCESS;
878}
879
880static void ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) {
881    ALOGV("%s:", __FUNCTION__);
882    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
883    if (ctx == NULL) {
884        jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
885        return;
886    }
887
888    // CpuConsumer* consumer = ctx->getCpuConsumer();
889    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image);
890    if (!buffer) {
891        ALOGW("Image already released!!!");
892        return;
893    }
894
895    // TODO: need implement
896    jniThrowRuntimeException(env, "nativeDetachImage is not implemented yet!!!");
897}
898
899static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
900{
901    ALOGV("%s: ", __FUNCTION__);
902
903    IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz);
904    if (gbp == NULL) {
905        jniThrowRuntimeException(env, "CpuConsumer is uninitialized");
906        return NULL;
907    }
908
909    // Wrap the IGBP in a Java-language Surface.
910    return android_view_Surface_createFromIGraphicBufferProducer(env, gbp);
911}
912
913static jobject Image_createSurfacePlane(JNIEnv* env, jobject thiz, int idx, int readerFormat)
914{
915    int rowStride, pixelStride;
916    PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat);
917
918    ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
919
920    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
921
922    ALOG_ASSERT(buffer != NULL);
923    if (buffer == NULL) {
924        jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
925    }
926
927    int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat(
928        publicReaderFormat);
929
930    rowStride = Image_imageGetRowStride(env, buffer, idx, halReaderFormat);
931    pixelStride = Image_imageGetPixelStride(env, buffer, idx, halReaderFormat);
932
933    jobject surfPlaneObj = env->NewObject(gSurfacePlaneClassInfo.clazz,
934            gSurfacePlaneClassInfo.ctor, thiz, idx, rowStride, pixelStride);
935
936    return surfPlaneObj;
937}
938
939static jobject Image_getByteBuffer(JNIEnv* env, jobject thiz, int idx, int readerFormat)
940{
941    uint8_t *base = NULL;
942    uint32_t size = 0;
943    jobject byteBuffer;
944    PublicFormat readerPublicFormat = static_cast<PublicFormat>(readerFormat);
945
946    ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
947
948    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
949
950    if (buffer == NULL) {
951        jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
952    }
953
954    int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat(
955            readerPublicFormat);
956
957    // Create byteBuffer from native buffer
958    Image_getLockedBufferInfo(env, buffer, idx, &base, &size, readerHalFormat);
959
960    if (size > static_cast<uint32_t>(INT32_MAX)) {
961        // Byte buffer have 'int capacity', so check the range
962        jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
963                "Size too large for bytebuffer capacity %" PRIu32, size);
964        return NULL;
965    }
966
967    byteBuffer = env->NewDirectByteBuffer(base, size);
968    // TODO: throw dvm exOutOfMemoryError?
969    if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
970        jniThrowException(env, "java/lang/IllegalStateException", "Failed to allocate ByteBuffer");
971    }
972
973    return byteBuffer;
974}
975
976static jint Image_getWidth(JNIEnv* env, jobject thiz)
977{
978    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
979    return Image_getBufferWidth(buffer);
980}
981
982static jint Image_getHeight(JNIEnv* env, jobject thiz)
983{
984    CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
985    return Image_getBufferHeight(buffer);
986}
987
988
989} // extern "C"
990
991// ----------------------------------------------------------------------------
992
993static JNINativeMethod gImageReaderMethods[] = {
994    {"nativeClassInit",        "()V",                        (void*)ImageReader_classInit },
995    {"nativeInit",             "(Ljava/lang/Object;IIII)V",  (void*)ImageReader_init },
996    {"nativeClose",            "()V",                        (void*)ImageReader_close },
997    {"nativeReleaseImage",     "(Landroid/media/Image;)V",   (void*)ImageReader_imageRelease },
998    {"nativeImageSetup",       "(Landroid/media/Image;)I",   (void*)ImageReader_imageSetup },
999    {"nativeGetSurface",       "()Landroid/view/Surface;",   (void*)ImageReader_getSurface },
1000    {"nativeDetachImage",      "(Landroid/media/Image;)V",   (void*)ImageReader_detachImage },
1001};
1002
1003static JNINativeMethod gImageMethods[] = {
1004    {"nativeImageGetBuffer",   "(II)Ljava/nio/ByteBuffer;",   (void*)Image_getByteBuffer },
1005    {"nativeCreatePlane",      "(II)Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",
1006                                                              (void*)Image_createSurfacePlane },
1007    {"nativeGetWidth",         "()I",                         (void*)Image_getWidth },
1008    {"nativeGetHeight",        "()I",                         (void*)Image_getHeight },
1009};
1010
1011int register_android_media_ImageReader(JNIEnv *env) {
1012
1013    int ret1 = AndroidRuntime::registerNativeMethods(env,
1014                   "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods));
1015
1016    int ret2 = AndroidRuntime::registerNativeMethods(env,
1017                   "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods));
1018
1019    return (ret1 || ret2);
1020}
1021