android_media_ImageWriter.cpp revision d1cbc684053986c6c9efc6220bd79d654932b3d0 
1/*
2 * Copyright 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "ImageWriter_JNI"
19#include <utils/Log.h>
20#include <utils/String8.h>
21
22#include <gui/IProducerListener.h>
23#include <gui/Surface.h>
24#include <gui/CpuConsumer.h>
25#include <android_runtime/AndroidRuntime.h>
26#include <android_runtime/android_view_Surface.h>
27#include <camera3.h>
28
29#include <jni.h>
30#include <JNIHelp.h>
31
32#include <stdint.h>
33#include <inttypes.h>
34
35#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
36
37#define IMAGE_BUFFER_JNI_ID           "mNativeBuffer"
38
39// ----------------------------------------------------------------------------
40
41using namespace android;
42
43enum {
44    IMAGE_WRITER_MAX_NUM_PLANES = 3,
45};
46
47static struct {
48    jmethodID postEventFromNative;
49    jfieldID mWriterFormat;
50} gImageWriterClassInfo;
51
52static struct {
53    jfieldID mNativeBuffer;
54    jfieldID mNativeFenceFd;
55    jfieldID mPlanes;
56} gSurfaceImageClassInfo;
57
58static struct {
59    jclass clazz;
60    jmethodID ctor;
61} gSurfacePlaneClassInfo;
62
63typedef CpuConsumer::LockedBuffer LockedImage;
64
65// ----------------------------------------------------------------------------
66
67class JNIImageWriterContext : public BnProducerListener {
68public:
69    JNIImageWriterContext(JNIEnv* env, jobject weakThiz, jclass clazz);
70
71    virtual ~JNIImageWriterContext();
72
73    // Implementation of IProducerListener, used to notify the ImageWriter that the consumer
74    // has returned a buffer and it is ready for ImageWriter to dequeue.
75    virtual void onBufferReleased();
76
77    void setProducer(const sp<ANativeWindow>& producer) { mProducer = producer; }
78    ANativeWindow* getProducer() { return mProducer.get(); }
79
80    void setBufferFormat(int format) { mFormat = format; }
81    int getBufferFormat() { return mFormat; }
82
83    void setBufferWidth(int width) { mWidth = width; }
84    int getBufferWidth() { return mWidth; }
85
86    void setBufferHeight(int height) { mHeight = height; }
87    int getBufferHeight() { return mHeight; }
88
89private:
90    static JNIEnv* getJNIEnv(bool* needsDetach);
91    static void detachJNI();
92
93    sp<ANativeWindow> mProducer;
94    jobject mWeakThiz;
95    jclass mClazz;
96    int mFormat;
97    int mWidth;
98    int mHeight;
99};
100
101JNIImageWriterContext::JNIImageWriterContext(JNIEnv* env, jobject weakThiz, jclass clazz) :
102    mWeakThiz(env->NewGlobalRef(weakThiz)),
103    mClazz((jclass)env->NewGlobalRef(clazz)),
104    mFormat(0),
105    mWidth(-1),
106    mHeight(-1) {
107}
108
109JNIImageWriterContext::~JNIImageWriterContext() {
110    ALOGV("%s", __FUNCTION__);
111    bool needsDetach = false;
112    JNIEnv* env = getJNIEnv(&needsDetach);
113    if (env != NULL) {
114        env->DeleteGlobalRef(mWeakThiz);
115        env->DeleteGlobalRef(mClazz);
116    } else {
117        ALOGW("leaking JNI object references");
118    }
119    if (needsDetach) {
120        detachJNI();
121    }
122
123    mProducer.clear();
124}
125
126JNIEnv* JNIImageWriterContext::getJNIEnv(bool* needsDetach) {
127    ALOGV("%s", __FUNCTION__);
128    LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
129    *needsDetach = false;
130    JNIEnv* env = AndroidRuntime::getJNIEnv();
131    if (env == NULL) {
132        JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
133        JavaVM* vm = AndroidRuntime::getJavaVM();
134        int result = vm->AttachCurrentThread(&env, (void*) &args);
135        if (result != JNI_OK) {
136            ALOGE("thread attach failed: %#x", result);
137            return NULL;
138        }
139        *needsDetach = true;
140    }
141    return env;
142}
143
144void JNIImageWriterContext::detachJNI() {
145    ALOGV("%s", __FUNCTION__);
146    JavaVM* vm = AndroidRuntime::getJavaVM();
147    int result = vm->DetachCurrentThread();
148    if (result != JNI_OK) {
149        ALOGE("thread detach failed: %#x", result);
150    }
151}
152
153void JNIImageWriterContext::onBufferReleased() {
154    ALOGV("%s: buffer released", __FUNCTION__);
155    bool needsDetach = false;
156    JNIEnv* env = getJNIEnv(&needsDetach);
157    if (env != NULL) {
158        env->CallStaticVoidMethod(mClazz, gImageWriterClassInfo.postEventFromNative, mWeakThiz);
159    } else {
160        ALOGW("onBufferReleased event will not posted");
161    }
162    if (needsDetach) {
163        detachJNI();
164    }
165}
166
167// ----------------------------------------------------------------------------
168
169extern "C" {
170
171// -------------------------------Private method declarations--------------
172
173static bool isWritable(int format);
174static bool isPossiblyYUV(PixelFormat format);
175static void Image_setNativeContext(JNIEnv* env, jobject thiz,
176        sp<GraphicBuffer> buffer, int fenceFd);
177static void Image_getNativeContext(JNIEnv* env, jobject thiz,
178        GraphicBuffer** buffer, int* fenceFd);
179static void Image_unlockIfLocked(JNIEnv* env, jobject thiz);
180
181// --------------------------ImageWriter methods---------------------------------------
182
183static void ImageWriter_classInit(JNIEnv* env, jclass clazz) {
184    ALOGV("%s:", __FUNCTION__);
185    jclass imageClazz = env->FindClass("android/media/ImageWriter$WriterSurfaceImage");
186    LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
187            "can't find android/media/ImageWriter$WriterSurfaceImage");
188    gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID(
189            imageClazz, IMAGE_BUFFER_JNI_ID, "J");
190    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL,
191            "can't find android/media/ImageWriter$WriterSurfaceImage.%s", IMAGE_BUFFER_JNI_ID);
192
193    gSurfaceImageClassInfo.mNativeFenceFd = env->GetFieldID(
194            imageClazz, "mNativeFenceFd", "I");
195    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeFenceFd == NULL,
196            "can't find android/media/ImageWriter$WriterSurfaceImage.mNativeFenceFd");
197
198    gSurfaceImageClassInfo.mPlanes = env->GetFieldID(
199            imageClazz, "mPlanes", "[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;");
200    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL,
201            "can't find android/media/ImageWriter$WriterSurfaceImage.mPlanes");
202
203    gImageWriterClassInfo.postEventFromNative = env->GetStaticMethodID(
204            clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
205    LOG_ALWAYS_FATAL_IF(gImageWriterClassInfo.postEventFromNative == NULL,
206                        "can't find android/media/ImageWriter.postEventFromNative");
207
208    gImageWriterClassInfo.mWriterFormat = env->GetFieldID(
209            clazz, "mWriterFormat", "I");
210    LOG_ALWAYS_FATAL_IF(gImageWriterClassInfo.mWriterFormat == NULL,
211                        "can't find android/media/ImageWriter.mWriterFormat");
212
213    jclass planeClazz = env->FindClass("android/media/ImageWriter$WriterSurfaceImage$SurfacePlane");
214    LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
215    // FindClass only gives a local reference of jclass object.
216    gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
217    gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
218            "(Landroid/media/ImageWriter$WriterSurfaceImage;IILjava/nio/ByteBuffer;)V");
219    LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
220            "Can not find SurfacePlane constructor");
221}
222
223static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobject jsurface,
224        jint maxImages) {
225    status_t res;
226
227    ALOGV("%s: maxImages:%d", __FUNCTION__, maxImages);
228
229    sp<Surface> surface(android_view_Surface_getSurface(env, jsurface));
230    if (surface == NULL) {
231        jniThrowException(env,
232                "java/lang/IllegalArgumentException",
233                "The surface has been released");
234        return 0;
235     }
236    sp<IGraphicBufferProducer> bufferProducer = surface->getIGraphicBufferProducer();
237
238    jclass clazz = env->GetObjectClass(thiz);
239    if (clazz == NULL) {
240        jniThrowRuntimeException(env, "Can't find android/graphics/ImageWriter");
241        return 0;
242    }
243    sp<JNIImageWriterContext> ctx(new JNIImageWriterContext(env, weakThiz, clazz));
244
245    sp<Surface> producer = new Surface(bufferProducer, /*controlledByApp*/false);
246    ctx->setProducer(producer);
247    /**
248     * NATIVE_WINDOW_API_CPU isn't a good choice here, as it makes the bufferQueue not connectable
249     * after disconnect. MEDIA or CAMERA are treated the same internally. The producer listener
250     * will be cleared after disconnect call.
251     */
252    producer->connect(/*api*/NATIVE_WINDOW_API_CAMERA, /*listener*/ctx);
253    jlong nativeCtx = reinterpret_cast<jlong>(ctx.get());
254
255    // Get the dimension and format of the producer.
256    sp<ANativeWindow> anw = producer;
257    int32_t width, height, format;
258    if ((res = anw->query(anw.get(), NATIVE_WINDOW_WIDTH, &width)) != OK) {
259        ALOGE("%s: Query Surface width failed: %s (%d)", __FUNCTION__, strerror(-res), res);
260        jniThrowRuntimeException(env, "Failed to query Surface width");
261        return 0;
262    }
263    ctx->setBufferWidth(width);
264
265    if ((res = anw->query(anw.get(), NATIVE_WINDOW_HEIGHT, &height)) != OK) {
266        ALOGE("%s: Query Surface height failed: %s (%d)", __FUNCTION__, strerror(-res), res);
267        jniThrowRuntimeException(env, "Failed to query Surface height");
268        return 0;
269    }
270    ctx->setBufferHeight(height);
271
272    if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &format)) != OK) {
273        ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res);
274        jniThrowRuntimeException(env, "Failed to query Surface format");
275        return 0;
276    }
277    ctx->setBufferFormat(format);
278    env->SetIntField(thiz, gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(format));
279
280
281    if (isWritable(format)) {
282        res = native_window_set_usage(anw.get(), GRALLOC_USAGE_SW_WRITE_OFTEN);
283        if (res != OK) {
284            ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)",
285                    __FUNCTION__, GRALLOC_USAGE_SW_WRITE_OFTEN, format, strerror(-res), res);
286            jniThrowRuntimeException(env, "Failed to SW_WRITE_OFTEN configure usage");
287            return 0;
288        }
289    }
290
291    int minUndequeuedBufferCount = 0;
292    res = anw->query(anw.get(),
293                NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBufferCount);
294    if (res != OK) {
295        ALOGE("%s: Query producer undequeued buffer count failed: %s (%d)",
296                __FUNCTION__, strerror(-res), res);
297        jniThrowRuntimeException(env, "Query producer undequeued buffer count failed");
298        return 0;
299     }
300
301    size_t totalBufferCount = maxImages + minUndequeuedBufferCount;
302    res = native_window_set_buffer_count(anw.get(), totalBufferCount);
303    if (res != OK) {
304        ALOGE("%s: Set buffer count failed: %s (%d)", __FUNCTION__, strerror(-res), res);
305        jniThrowRuntimeException(env, "Set buffer count failed");
306        return 0;
307    }
308
309    if (ctx != 0) {
310        ctx->incStrong((void*)ImageWriter_init);
311    }
312    return nativeCtx;
313}
314
315static void ImageWriter_dequeueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) {
316    ALOGV("%s", __FUNCTION__);
317    JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
318    if (ctx == NULL || thiz == NULL) {
319        jniThrowException(env, "java/lang/IllegalStateException",
320                "ImageWriterContext is not initialized");
321        return;
322    }
323
324    sp<ANativeWindow> anw = ctx->getProducer();
325    android_native_buffer_t *anb = NULL;
326    int fenceFd = -1;
327    status_t res = anw->dequeueBuffer(anw.get(), &anb, &fenceFd);
328    if (res != OK) {
329        // TODO: handle different error cases here.
330        ALOGE("%s: Set buffer count failed: %s (%d)", __FUNCTION__, strerror(-res), res);
331        jniThrowRuntimeException(env, "dequeue buffer failed");
332        return;
333    }
334    // New GraphicBuffer object doesn't own the handle, thus the native buffer
335    // won't be freed when this object is destroyed.
336    sp<GraphicBuffer> buffer(new GraphicBuffer(anb, /*keepOwnership*/false));
337
338    // Note that:
339    // 1. No need to lock buffer now, will only lock it when the first getPlanes() is called.
340    // 2. Fence will be saved to mNativeFenceFd, and will consumed by lock/queue/cancel buffer
341    //    later.
342    // 3. need use lockAsync here, as it will handle the dequeued fence for us automatically.
343
344    // Finally, set the native info into image object.
345    Image_setNativeContext(env, image, buffer, fenceFd);
346}
347
348static void ImageWriter_close(JNIEnv* env, jobject thiz, jlong nativeCtx) {
349    ALOGV("%s:", __FUNCTION__);
350    JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
351    if (ctx == NULL || thiz == NULL) {
352        jniThrowException(env, "java/lang/IllegalStateException",
353                "ImageWriterContext is not initialized");
354        return;
355    }
356
357    ANativeWindow* producer = ctx->getProducer();
358    if (producer != NULL) {
359        /**
360         * NATIVE_WINDOW_API_CPU isn't a good choice here, as it makes the bufferQueue not
361         * connectable after disconnect. MEDIA or CAMERA are treated the same internally.
362         * The producer listener will be cleared after disconnect call.
363         */
364        status_t res = native_window_api_disconnect(producer, /*api*/NATIVE_WINDOW_API_CAMERA);
365        /**
366         * This is not an error. if client calling process dies, the window will
367         * also die and all calls to it will return DEAD_OBJECT, thus it's already
368         * "disconnected"
369         */
370        if (res == DEAD_OBJECT) {
371            ALOGW("%s: While disconnecting ImageWriter from native window, the"
372                    " native window died already", __FUNCTION__);
373        } else if (res != OK) {
374            ALOGE("%s: native window disconnect failed: %s (%d)",
375                    __FUNCTION__, strerror(-res), res);
376            jniThrowRuntimeException(env, "Native window disconnect failed");
377            return;
378        }
379    }
380
381    ctx->decStrong((void*)ImageWriter_init);
382}
383
384static void ImageWriter_cancelImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) {
385    ALOGV("%s", __FUNCTION__);
386    JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
387    if (ctx == NULL || thiz == NULL) {
388        jniThrowException(env, "java/lang/IllegalStateException",
389                "ImageWriterContext is not initialized");
390        return;
391    }
392
393    sp<ANativeWindow> anw = ctx->getProducer();
394
395    GraphicBuffer *buffer = NULL;
396    int fenceFd = -1;
397    Image_getNativeContext(env, image, &buffer, &fenceFd);
398    if (buffer == NULL) {
399        jniThrowException(env, "java/lang/IllegalStateException",
400                "Image is not initialized");
401        return;
402    }
403
404    // Unlock the image if it was locked
405    Image_unlockIfLocked(env, image);
406
407    anw->cancelBuffer(anw.get(), buffer, fenceFd);
408
409    Image_setNativeContext(env, image, NULL, -1);
410}
411
412static void ImageWriter_queueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image,
413        jlong timestampNs, jint left, jint top, jint right, jint bottom) {
414    ALOGV("%s", __FUNCTION__);
415    JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
416    if (ctx == NULL || thiz == NULL) {
417        jniThrowException(env, "java/lang/IllegalStateException",
418                "ImageWriterContext is not initialized");
419        return;
420    }
421
422    status_t res = OK;
423    sp<ANativeWindow> anw = ctx->getProducer();
424
425    GraphicBuffer *buffer = NULL;
426    int fenceFd = -1;
427    Image_getNativeContext(env, image, &buffer, &fenceFd);
428    if (buffer == NULL) {
429        jniThrowException(env, "java/lang/IllegalStateException",
430                "Image is not initialized");
431        return;
432    }
433
434    // Unlock image if it was locked.
435    Image_unlockIfLocked(env, image);
436
437    // Set timestamp
438    ALOGV("timestamp to be queued: %" PRId64, timestampNs);
439    res = native_window_set_buffers_timestamp(anw.get(), timestampNs);
440    if (res != OK) {
441        jniThrowRuntimeException(env, "Set timestamp failed");
442        return;
443    }
444
445    // Set crop
446    android_native_rect_t cropRect;
447    cropRect.left = left;
448    cropRect.top = top;
449    cropRect.right = right;
450    cropRect.bottom = bottom;
451    res = native_window_set_crop(anw.get(), &cropRect);
452    if (res != OK) {
453        jniThrowRuntimeException(env, "Set crop rect failed");
454        return;
455    }
456
457    // Finally, queue input buffer
458    res = anw->queueBuffer(anw.get(), buffer, fenceFd);
459    if (res != OK) {
460        jniThrowRuntimeException(env, "Queue input buffer failed");
461        return;
462    }
463
464    Image_setNativeContext(env, image, NULL, -1);
465}
466
467static void ImageWriter_attachImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) {
468    ALOGV("%s", __FUNCTION__);
469    JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
470    if (ctx == NULL || thiz == NULL) {
471        jniThrowException(env, "java/lang/IllegalStateException",
472                "ImageWriterContext is not initialized");
473        return;
474    }
475
476    sp<ANativeWindow> anw = ctx->getProducer();
477
478    GraphicBuffer *buffer = NULL;
479    int fenceFd = -1;
480    Image_getNativeContext(env, image, &buffer, &fenceFd);
481    if (buffer == NULL) {
482        jniThrowException(env, "java/lang/IllegalStateException",
483                "Image is not initialized");
484        return;
485    }
486
487    // TODO: need implement
488    jniThrowRuntimeException(env, "nativeAttachImage is not implement yet!!!");
489}
490
491// --------------------------Image methods---------------------------------------
492
493static void Image_getNativeContext(JNIEnv* env, jobject thiz,
494        GraphicBuffer** buffer, int* fenceFd) {
495    ALOGV("%s", __FUNCTION__);
496    if (buffer != NULL) {
497        GraphicBuffer *gb = reinterpret_cast<GraphicBuffer *>
498                  (env->GetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer));
499        *buffer = gb;
500    }
501
502    if (fenceFd != NULL) {
503        *fenceFd = reinterpret_cast<jint>(env->GetIntField(
504                thiz, gSurfaceImageClassInfo.mNativeFenceFd));
505    }
506}
507
508static void Image_setNativeContext(JNIEnv* env, jobject thiz,
509        sp<GraphicBuffer> buffer, int fenceFd) {
510    ALOGV("%s:", __FUNCTION__);
511    GraphicBuffer* p = NULL;
512    Image_getNativeContext(env, thiz, &p, /*fenceFd*/NULL);
513    if (buffer != 0) {
514        buffer->incStrong((void*)Image_setNativeContext);
515    }
516    if (p) {
517        p->decStrong((void*)Image_setNativeContext);
518    }
519    env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer,
520            reinterpret_cast<jlong>(buffer.get()));
521
522    env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast<jint>(fenceFd));
523}
524
525static void Image_unlockIfLocked(JNIEnv* env, jobject thiz) {
526    ALOGV("%s", __FUNCTION__);
527    GraphicBuffer* buffer;
528    Image_getNativeContext(env, thiz, &buffer, NULL);
529    if (buffer == NULL) {
530        jniThrowException(env, "java/lang/IllegalStateException",
531                "Image is not initialized");
532        return;
533    }
534
535    // Is locked?
536    bool isLocked = false;
537    jobject planes = env->GetObjectField(thiz, gSurfaceImageClassInfo.mPlanes);
538    isLocked = (planes != NULL);
539    if (isLocked) {
540        // no need to use fence here, as we it will be consumed by either concel or queue buffer.
541        status_t res = buffer->unlock();
542        if (res != OK) {
543            jniThrowRuntimeException(env, "unlock buffer failed");
544        }
545        ALOGV("Successfully unlocked the image");
546    }
547}
548
549static jint Image_getWidth(JNIEnv* env, jobject thiz) {
550    ALOGV("%s", __FUNCTION__);
551    GraphicBuffer* buffer;
552    Image_getNativeContext(env, thiz, &buffer, NULL);
553    if (buffer == NULL) {
554        jniThrowException(env, "java/lang/IllegalStateException",
555                "Image is not initialized");
556        return -1;
557    }
558
559    return buffer->getWidth();
560}
561
562static jint Image_getHeight(JNIEnv* env, jobject thiz) {
563    ALOGV("%s", __FUNCTION__);
564    GraphicBuffer* buffer;
565    Image_getNativeContext(env, thiz, &buffer, NULL);
566    if (buffer == NULL) {
567        jniThrowException(env, "java/lang/IllegalStateException",
568                "Image is not initialized");
569        return -1;
570    }
571
572    return buffer->getHeight();
573}
574
575// Some formats like JPEG defined with different values between android.graphics.ImageFormat and
576// graphics.h, need convert to the one defined in graphics.h here.
577static int Image_getPixelFormat(JNIEnv* env, int format) {
578    int jpegFormat;
579    jfieldID fid;
580
581    ALOGV("%s: format = 0x%x", __FUNCTION__, format);
582
583    jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat");
584    ALOG_ASSERT(imageFormatClazz != NULL);
585
586    fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I");
587    jpegFormat = env->GetStaticIntField(imageFormatClazz, fid);
588
589    // Translate the JPEG to BLOB for camera purpose.
590    if (format == jpegFormat) {
591        format = HAL_PIXEL_FORMAT_BLOB;
592    }
593
594    return format;
595}
596
597static jint Image_getFormat(JNIEnv* env, jobject thiz) {
598    ALOGV("%s", __FUNCTION__);
599    GraphicBuffer* buffer;
600    Image_getNativeContext(env, thiz, &buffer, NULL);
601    if (buffer == NULL) {
602        jniThrowException(env, "java/lang/IllegalStateException",
603                "Image is not initialized");
604        return 0;
605    }
606
607    return Image_getPixelFormat(env, buffer->getPixelFormat());
608}
609
610static void Image_setFenceFd(JNIEnv* env, jobject thiz, int fenceFd) {
611    ALOGV("%s:", __FUNCTION__);
612    env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast<jint>(fenceFd));
613}
614
615static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) {
616    ALOGV("%s", __FUNCTION__);
617    GraphicBuffer* buffer;
618    int fenceFd = -1;
619    Image_getNativeContext(env, thiz, &buffer, &fenceFd);
620    if (buffer == NULL) {
621        jniThrowException(env, "java/lang/IllegalStateException",
622                "Image is not initialized");
623        return;
624    }
625
626    void* pData = NULL;
627    android_ycbcr ycbcr = android_ycbcr();
628    status_t res;
629    int format = Image_getFormat(env, thiz);
630    int flexFormat = format;
631    if (isPossiblyYUV(format)) {
632        // ImageWriter doesn't use crop by itself, app sets it, use the no crop version.
633        res = buffer->lockAsyncYCbCr(GRALLOC_USAGE_SW_WRITE_OFTEN, &ycbcr, fenceFd);
634        // Clear the fenceFd as it is already consumed by lock call.
635        Image_setFenceFd(env, thiz, /*fenceFd*/-1);
636        if (res != OK) {
637            jniThrowRuntimeException(env, "lockAsyncYCbCr failed for YUV buffer");
638            return;
639        }
640        pData = ycbcr.y;
641        flexFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;
642    }
643
644    // lockAsyncYCbCr for YUV is unsuccessful.
645    if (pData == NULL) {
646        res = buffer->lockAsync(GRALLOC_USAGE_SW_WRITE_OFTEN, &pData, fenceFd);
647        if (res != OK) {
648            jniThrowRuntimeException(env, "lockAsync failed");
649            return;
650        }
651    }
652
653    image->data = reinterpret_cast<uint8_t*>(pData);
654    image->width = buffer->getWidth();
655    image->height = buffer->getHeight();
656    image->format = format;
657    image->flexFormat = flexFormat;
658    image->stride = (ycbcr.y != NULL) ? static_cast<uint32_t>(ycbcr.ystride) : buffer->getStride();
659
660    image->dataCb = reinterpret_cast<uint8_t*>(ycbcr.cb);
661    image->dataCr = reinterpret_cast<uint8_t*>(ycbcr.cr);
662    image->chromaStride = static_cast<uint32_t>(ycbcr.cstride);
663    image->chromaStep = static_cast<uint32_t>(ycbcr.chroma_step);
664    ALOGV("Successfully locked the image");
665    // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer,
666    // and we don't set them here.
667}
668
669static bool usingRGBAToJpegOverride(int32_t bufferFormat, int32_t writerCtxFormat) {
670    return writerCtxFormat == HAL_PIXEL_FORMAT_BLOB && bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888;
671}
672
673static int32_t applyFormatOverrides(int32_t bufferFormat, int32_t writerCtxFormat)
674{
675    // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW
676    // write limitations for some platforms (b/17379185).
677    if (usingRGBAToJpegOverride(bufferFormat, writerCtxFormat)) {
678        return HAL_PIXEL_FORMAT_BLOB;
679    }
680    return bufferFormat;
681}
682
683static uint32_t Image_getJpegSize(LockedImage* buffer, bool usingRGBAOverride) {
684    ALOGV("%s", __FUNCTION__);
685    ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
686    uint32_t size = 0;
687    uint32_t width = buffer->width;
688    uint8_t* jpegBuffer = buffer->data;
689
690    if (usingRGBAOverride) {
691        width = (buffer->width + buffer->stride * (buffer->height - 1)) * 4;
692    }
693
694    // First check for JPEG transport header at the end of the buffer
695    uint8_t* header = jpegBuffer + (width - sizeof(struct camera3_jpeg_blob));
696    struct camera3_jpeg_blob *blob = (struct camera3_jpeg_blob*)(header);
697    if (blob->jpeg_blob_id == CAMERA3_JPEG_BLOB_ID) {
698        size = blob->jpeg_size;
699        ALOGV("%s: Jpeg size = %d", __FUNCTION__, size);
700    }
701
702    // failed to find size, default to whole buffer
703    if (size == 0) {
704        /*
705         * This is a problem because not including the JPEG header
706         * means that in certain rare situations a regular JPEG blob
707         * will be misidentified as having a header, in which case
708         * we will get a garbage size value.
709         */
710        ALOGW("%s: No JPEG header detected, defaulting to size=width=%d",
711                __FUNCTION__, width);
712        size = width;
713    }
714
715    return size;
716}
717
718static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
719        int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {
720    ALOGV("%s", __FUNCTION__);
721    ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
722    ALOG_ASSERT(base != NULL, "base is NULL!!!");
723    ALOG_ASSERT(size != NULL, "size is NULL!!!");
724    ALOG_ASSERT(pixelStride != NULL, "pixelStride is NULL!!!");
725    ALOG_ASSERT(rowStride != NULL, "rowStride is NULL!!!");
726    ALOG_ASSERT((idx < IMAGE_WRITER_MAX_NUM_PLANES) && (idx >= 0));
727
728    ALOGV("%s: buffer: %p", __FUNCTION__, buffer);
729
730    uint32_t dataSize, ySize, cSize, cStride;
731    uint32_t pStride = 0, rStride = 0;
732    uint8_t *cb, *cr;
733    uint8_t *pData = NULL;
734    int bytesPerPixel = 0;
735
736    dataSize = ySize = cSize = cStride = 0;
737    int32_t fmt = buffer->flexFormat;
738
739    bool usingRGBAOverride = usingRGBAToJpegOverride(fmt, writerFormat);
740    fmt = applyFormatOverrides(fmt, writerFormat);
741    switch (fmt) {
742        case HAL_PIXEL_FORMAT_YCbCr_420_888:
743            pData =
744                (idx == 0) ?
745                    buffer->data :
746                (idx == 1) ?
747                    buffer->dataCb :
748                buffer->dataCr;
749            // only map until last pixel
750            if (idx == 0) {
751                pStride = 1;
752                rStride = buffer->stride;
753                dataSize = buffer->stride * (buffer->height - 1) + buffer->width;
754            } else {
755                pStride = buffer->chromaStep;
756                rStride = buffer->chromaStride;
757                dataSize = buffer->chromaStride * (buffer->height / 2 - 1) +
758                        buffer->chromaStep * (buffer->width / 2 - 1) + 1;
759            }
760            break;
761        // NV21
762        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
763            cr = buffer->data + (buffer->stride * buffer->height);
764            cb = cr + 1;
765            // only map until last pixel
766            ySize = buffer->width * (buffer->height - 1) + buffer->width;
767            cSize = buffer->width * (buffer->height / 2 - 1) + buffer->width - 1;
768
769            pData =
770                (idx == 0) ?
771                    buffer->data :
772                (idx == 1) ?
773                    cb:
774                cr;
775
776            dataSize = (idx == 0) ? ySize : cSize;
777            pStride = (idx == 0) ? 1 : 2;
778            rStride = buffer->width;
779            break;
780        case HAL_PIXEL_FORMAT_YV12:
781            // Y and C stride need to be 16 pixel aligned.
782            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
783                                "Stride is not 16 pixel aligned %d", buffer->stride);
784
785            ySize = buffer->stride * buffer->height;
786            cStride = ALIGN(buffer->stride / 2, 16);
787            cr = buffer->data + ySize;
788            cSize = cStride * buffer->height / 2;
789            cb = cr + cSize;
790
791            pData =
792                (idx == 0) ?
793                    buffer->data :
794                (idx == 1) ?
795                    cb :
796                cr;
797            dataSize = (idx == 0) ? ySize : cSize;
798            pStride = 1;
799            rStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16);
800            break;
801        case HAL_PIXEL_FORMAT_Y8:
802            // Single plane, 8bpp.
803            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
804
805            pData = buffer->data;
806            dataSize = buffer->stride * buffer->height;
807            pStride = 1;
808            rStride = buffer->stride;
809            break;
810        case HAL_PIXEL_FORMAT_Y16:
811            bytesPerPixel = 2;
812            // Single plane, 16bpp, strides are specified in pixels, not in bytes
813            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
814
815            pData = buffer->data;
816            dataSize = buffer->stride * buffer->height * bytesPerPixel;
817            pStride = bytesPerPixel;
818            rStride = buffer->stride * 2;
819            break;
820        case HAL_PIXEL_FORMAT_BLOB:
821            // Used for JPEG data, height must be 1, width == size, single plane.
822            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
823            ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height);
824
825            pData = buffer->data;
826            dataSize = Image_getJpegSize(buffer, usingRGBAOverride);
827            pStride = bytesPerPixel;
828            rowStride = 0;
829            break;
830        case HAL_PIXEL_FORMAT_RAW16:
831            // Single plane 16bpp bayer data.
832            bytesPerPixel = 2;
833            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
834            pData = buffer->data;
835            dataSize = buffer->stride * buffer->height * bytesPerPixel;
836            pStride = bytesPerPixel;
837            rStride = buffer->stride * 2;
838            break;
839        case HAL_PIXEL_FORMAT_RAW10:
840            // Single plane 10bpp bayer data.
841            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
842            LOG_ALWAYS_FATAL_IF(buffer->width % 4,
843                                "Width is not multiple of 4 %d", buffer->width);
844            LOG_ALWAYS_FATAL_IF(buffer->height % 2,
845                                "Height is not even %d", buffer->height);
846            LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 10 / 8),
847                                "stride (%d) should be at least %d",
848                                buffer->stride, buffer->width * 10 / 8);
849            pData = buffer->data;
850            dataSize = buffer->stride * buffer->height;
851            pStride = 0;
852            rStride = buffer->stride;
853            break;
854        case HAL_PIXEL_FORMAT_RGBA_8888:
855        case HAL_PIXEL_FORMAT_RGBX_8888:
856            // Single plane, 32bpp.
857            bytesPerPixel = 4;
858            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
859            pData = buffer->data;
860            dataSize = buffer->stride * buffer->height * bytesPerPixel;
861            pStride = bytesPerPixel;
862            rStride = buffer->stride * 4;
863            break;
864        case HAL_PIXEL_FORMAT_RGB_565:
865            // Single plane, 16bpp.
866            bytesPerPixel = 2;
867            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
868            pData = buffer->data;
869            dataSize = buffer->stride * buffer->height * bytesPerPixel;
870            pStride = bytesPerPixel;
871            rStride = buffer->stride * 2;
872            break;
873        case HAL_PIXEL_FORMAT_RGB_888:
874            // Single plane, 24bpp.
875            bytesPerPixel = 3;
876            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
877            pData = buffer->data;
878            dataSize = buffer->stride * buffer->height * bytesPerPixel;
879            pStride = bytesPerPixel;
880            rStride = buffer->stride * 3;
881            break;
882        default:
883            jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
884                                 "Pixel format: 0x%x is unsupported", fmt);
885            break;
886    }
887
888    *base = pData;
889    *size = dataSize;
890    *pixelStride = pStride;
891    *rowStride = rStride;
892}
893
894static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
895        int numPlanes, int writerFormat) {
896    ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes);
897    int rowStride, pixelStride;
898    uint8_t *pData;
899    uint32_t dataSize;
900    jobject byteBuffer;
901
902    int format = Image_getFormat(env, thiz);
903    if (!isWritable(format) && numPlanes > 0) {
904        String8 msg;
905        msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
906                " must be 0", format, numPlanes);
907        jniThrowException(env, "java/lang/IllegalArgumentException", msg.string());
908        return NULL;
909    }
910
911    jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz,
912            /*initial_element*/NULL);
913    if (surfacePlanes == NULL) {
914        jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays,"
915                " probably out of memory");
916        return NULL;
917    }
918
919    // Buildup buffer info: rowStride, pixelStride and byteBuffers.
920    LockedImage lockedImg = LockedImage();
921    Image_getLockedImage(env, thiz, &lockedImg);
922
923    // Create all SurfacePlanes
924    writerFormat = Image_getPixelFormat(env, writerFormat);
925    for (int i = 0; i < numPlanes; i++) {
926        Image_getLockedImageInfo(env, &lockedImg, i, writerFormat,
927                &pData, &dataSize, &pixelStride, &rowStride);
928        byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
929        if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
930            jniThrowException(env, "java/lang/IllegalStateException",
931                    "Failed to allocate ByteBuffer");
932            return NULL;
933        }
934
935        // Finally, create this SurfacePlane.
936        jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
937                    gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
938        env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
939    }
940
941    return surfacePlanes;
942}
943
944// -------------------------------Private convenience methods--------------------
945
946// Check if buffer with this format is writable. Generally speaking, the opaque formats
947// like IMPLEMENTATION_DEFINED is not writable, as the actual buffer formats and layouts
948// are unknown to frameworks.
949static bool isWritable(int format) {
950    // Assume all other formats are writable.
951    return !(format == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED ||
952            format == HAL_PIXEL_FORMAT_RAW_OPAQUE);
953}
954
955static bool isPossiblyYUV(PixelFormat format) {
956    switch (static_cast<int>(format)) {
957        case HAL_PIXEL_FORMAT_RGBA_8888:
958        case HAL_PIXEL_FORMAT_RGBX_8888:
959        case HAL_PIXEL_FORMAT_RGB_888:
960        case HAL_PIXEL_FORMAT_RGB_565:
961        case HAL_PIXEL_FORMAT_BGRA_8888:
962        case HAL_PIXEL_FORMAT_Y8:
963        case HAL_PIXEL_FORMAT_Y16:
964        case HAL_PIXEL_FORMAT_RAW16:
965        case HAL_PIXEL_FORMAT_RAW10:
966        case HAL_PIXEL_FORMAT_RAW_OPAQUE:
967        case HAL_PIXEL_FORMAT_BLOB:
968        case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
969            return false;
970
971        case HAL_PIXEL_FORMAT_YV12:
972        case HAL_PIXEL_FORMAT_YCbCr_420_888:
973        case HAL_PIXEL_FORMAT_YCbCr_422_SP:
974        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
975        case HAL_PIXEL_FORMAT_YCbCr_422_I:
976        default:
977            return true;
978    }
979}
980
981} // extern "C"
982
983// ----------------------------------------------------------------------------
984
985static JNINativeMethod gImageWriterMethods[] = {
986    {"nativeClassInit",         "()V",                        (void*)ImageWriter_classInit },
987    {"nativeInit",              "(Ljava/lang/Object;Landroid/view/Surface;I)J",
988                                                              (void*)ImageWriter_init },
989    {"nativeClose",              "(J)V",                       (void*)ImageWriter_close },
990    {"nativeAttachImage",       "(JLandroid/media/Image;)V",  (void*)ImageWriter_attachImage },
991    {"nativeDequeueInputImage", "(JLandroid/media/Image;)V",  (void*)ImageWriter_dequeueImage },
992    {"nativeQueueInputImage",   "(JLandroid/media/Image;JIIII)V",  (void*)ImageWriter_queueImage },
993    {"cancelImage",             "(JLandroid/media/Image;)V",   (void*)ImageWriter_cancelImage },
994};
995
996static JNINativeMethod gImageMethods[] = {
997    {"nativeCreatePlanes",      "(II)[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;",
998                                                              (void*)Image_createSurfacePlanes },
999    {"nativeGetWidth",         "()I",                         (void*)Image_getWidth },
1000    {"nativeGetHeight",        "()I",                         (void*)Image_getHeight },
1001    {"nativeGetFormat",        "()I",                         (void*)Image_getFormat },
1002};
1003
1004int register_android_media_ImageWriter(JNIEnv *env) {
1005
1006    int ret1 = AndroidRuntime::registerNativeMethods(env,
1007                   "android/media/ImageWriter", gImageWriterMethods, NELEM(gImageWriterMethods));
1008
1009    int ret2 = AndroidRuntime::registerNativeMethods(env,
1010                   "android/media/ImageWriter$WriterSurfaceImage", gImageMethods, NELEM(gImageMethods));
1011
1012    return (ret1 || ret2);
1013}
1014
1015