android_media_ImageWriter.cpp revision e0ee63046ad062040aafc977585fb461a2acf666
1/*
2 * Copyright 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "ImageWriter_JNI"
19#include <utils/Log.h>
20#include <utils/String8.h>
21
22#include <gui/IProducerListener.h>
23#include <gui/Surface.h>
24#include <gui/CpuConsumer.h>
25#include <android_runtime/AndroidRuntime.h>
26#include <android_runtime/android_view_Surface.h>
27#include <camera3.h>
28
29#include <jni.h>
30#include <JNIHelp.h>
31
32#include <stdint.h>
33#include <inttypes.h>
34
35#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
36
37#define IMAGE_BUFFER_JNI_ID           "mNativeBuffer"
38
39// ----------------------------------------------------------------------------
40
41using namespace android;
42
43enum {
44    IMAGE_WRITER_MAX_NUM_PLANES = 3,
45};
46
47static struct {
48    jmethodID postEventFromNative;
49    jfieldID mWriterFormat;
50} gImageWriterClassInfo;
51
52static struct {
53    jfieldID mNativeBuffer;
54    jfieldID mNativeFenceFd;
55    jfieldID mPlanes;
56} gSurfaceImageClassInfo;
57
58static struct {
59    jclass clazz;
60    jmethodID ctor;
61} gSurfacePlaneClassInfo;
62
63typedef CpuConsumer::LockedBuffer LockedImage;
64
65// ----------------------------------------------------------------------------
66
67class JNIImageWriterContext : public BnProducerListener {
68public:
69    JNIImageWriterContext(JNIEnv* env, jobject weakThiz, jclass clazz);
70
71    virtual ~JNIImageWriterContext();
72
73    // Implementation of IProducerListener, used to notify the ImageWriter that the consumer
74    // has returned a buffer and it is ready for ImageWriter to dequeue.
75    virtual void onBufferReleased();
76
77    void setProducer(const sp<Surface>& producer) { mProducer = producer; }
78    Surface* getProducer() { return mProducer.get(); }
79
80    void setBufferFormat(int format) { mFormat = format; }
81    int getBufferFormat() { return mFormat; }
82
83    void setBufferWidth(int width) { mWidth = width; }
84    int getBufferWidth() { return mWidth; }
85
86    void setBufferHeight(int height) { mHeight = height; }
87    int getBufferHeight() { return mHeight; }
88
89private:
90    static JNIEnv* getJNIEnv(bool* needsDetach);
91    static void detachJNI();
92
93    sp<Surface> mProducer;
94    jobject mWeakThiz;
95    jclass mClazz;
96    int mFormat;
97    int mWidth;
98    int mHeight;
99};
100
101JNIImageWriterContext::JNIImageWriterContext(JNIEnv* env, jobject weakThiz, jclass clazz) :
102    mWeakThiz(env->NewGlobalRef(weakThiz)),
103    mClazz((jclass)env->NewGlobalRef(clazz)),
104    mFormat(0),
105    mWidth(-1),
106    mHeight(-1) {
107}
108
109JNIImageWriterContext::~JNIImageWriterContext() {
110    ALOGV("%s", __FUNCTION__);
111    bool needsDetach = false;
112    JNIEnv* env = getJNIEnv(&needsDetach);
113    if (env != NULL) {
114        env->DeleteGlobalRef(mWeakThiz);
115        env->DeleteGlobalRef(mClazz);
116    } else {
117        ALOGW("leaking JNI object references");
118    }
119    if (needsDetach) {
120        detachJNI();
121    }
122
123    mProducer.clear();
124}
125
126JNIEnv* JNIImageWriterContext::getJNIEnv(bool* needsDetach) {
127    ALOGV("%s", __FUNCTION__);
128    LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
129    *needsDetach = false;
130    JNIEnv* env = AndroidRuntime::getJNIEnv();
131    if (env == NULL) {
132        JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
133        JavaVM* vm = AndroidRuntime::getJavaVM();
134        int result = vm->AttachCurrentThread(&env, (void*) &args);
135        if (result != JNI_OK) {
136            ALOGE("thread attach failed: %#x", result);
137            return NULL;
138        }
139        *needsDetach = true;
140    }
141    return env;
142}
143
144void JNIImageWriterContext::detachJNI() {
145    ALOGV("%s", __FUNCTION__);
146    JavaVM* vm = AndroidRuntime::getJavaVM();
147    int result = vm->DetachCurrentThread();
148    if (result != JNI_OK) {
149        ALOGE("thread detach failed: %#x", result);
150    }
151}
152
153void JNIImageWriterContext::onBufferReleased() {
154    ALOGV("%s: buffer released", __FUNCTION__);
155    bool needsDetach = false;
156    JNIEnv* env = getJNIEnv(&needsDetach);
157    if (env != NULL) {
158        // Detach the buffer every time when a buffer consumption is done,
159        // need let this callback give a BufferItem, then only detach if it was attached to this
160        // Writer. Do the detach unconditionally for opaque format now. see b/19977520
161        if (mFormat == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED) {
162            sp<Fence> fence;
163            sp<GraphicBuffer> buffer;
164            ALOGV("%s: One buffer is detached", __FUNCTION__);
165            mProducer->detachNextBuffer(&buffer, &fence);
166        }
167
168        env->CallStaticVoidMethod(mClazz, gImageWriterClassInfo.postEventFromNative, mWeakThiz);
169    } else {
170        ALOGW("onBufferReleased event will not posted");
171    }
172
173    if (needsDetach) {
174        detachJNI();
175    }
176}
177
178// ----------------------------------------------------------------------------
179
180extern "C" {
181
182// -------------------------------Private method declarations--------------
183
184static bool isPossiblyYUV(PixelFormat format);
185static void Image_setNativeContext(JNIEnv* env, jobject thiz,
186        sp<GraphicBuffer> buffer, int fenceFd);
187static void Image_getNativeContext(JNIEnv* env, jobject thiz,
188        GraphicBuffer** buffer, int* fenceFd);
189static void Image_unlockIfLocked(JNIEnv* env, jobject thiz);
190static bool isFormatOpaque(int format);
191
192// --------------------------ImageWriter methods---------------------------------------
193
194static void ImageWriter_classInit(JNIEnv* env, jclass clazz) {
195    ALOGV("%s:", __FUNCTION__);
196    jclass imageClazz = env->FindClass("android/media/ImageWriter$WriterSurfaceImage");
197    LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
198            "can't find android/media/ImageWriter$WriterSurfaceImage");
199    gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID(
200            imageClazz, IMAGE_BUFFER_JNI_ID, "J");
201    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL,
202            "can't find android/media/ImageWriter$WriterSurfaceImage.%s", IMAGE_BUFFER_JNI_ID);
203
204    gSurfaceImageClassInfo.mNativeFenceFd = env->GetFieldID(
205            imageClazz, "mNativeFenceFd", "I");
206    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeFenceFd == NULL,
207            "can't find android/media/ImageWriter$WriterSurfaceImage.mNativeFenceFd");
208
209    gSurfaceImageClassInfo.mPlanes = env->GetFieldID(
210            imageClazz, "mPlanes", "[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;");
211    LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL,
212            "can't find android/media/ImageWriter$WriterSurfaceImage.mPlanes");
213
214    gImageWriterClassInfo.postEventFromNative = env->GetStaticMethodID(
215            clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
216    LOG_ALWAYS_FATAL_IF(gImageWriterClassInfo.postEventFromNative == NULL,
217                        "can't find android/media/ImageWriter.postEventFromNative");
218
219    gImageWriterClassInfo.mWriterFormat = env->GetFieldID(
220            clazz, "mWriterFormat", "I");
221    LOG_ALWAYS_FATAL_IF(gImageWriterClassInfo.mWriterFormat == NULL,
222                        "can't find android/media/ImageWriter.mWriterFormat");
223
224    jclass planeClazz = env->FindClass("android/media/ImageWriter$WriterSurfaceImage$SurfacePlane");
225    LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
226    // FindClass only gives a local reference of jclass object.
227    gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
228    gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
229            "(Landroid/media/ImageWriter$WriterSurfaceImage;IILjava/nio/ByteBuffer;)V");
230    LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
231            "Can not find SurfacePlane constructor");
232}
233
234static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobject jsurface,
235        jint maxImages) {
236    status_t res;
237
238    ALOGV("%s: maxImages:%d", __FUNCTION__, maxImages);
239
240    sp<Surface> surface(android_view_Surface_getSurface(env, jsurface));
241    if (surface == NULL) {
242        jniThrowException(env,
243                "java/lang/IllegalArgumentException",
244                "The surface has been released");
245        return 0;
246     }
247    sp<IGraphicBufferProducer> bufferProducer = surface->getIGraphicBufferProducer();
248
249    jclass clazz = env->GetObjectClass(thiz);
250    if (clazz == NULL) {
251        jniThrowRuntimeException(env, "Can't find android/graphics/ImageWriter");
252        return 0;
253    }
254    sp<JNIImageWriterContext> ctx(new JNIImageWriterContext(env, weakThiz, clazz));
255
256    sp<Surface> producer = new Surface(bufferProducer, /*controlledByApp*/false);
257    ctx->setProducer(producer);
258    /**
259     * NATIVE_WINDOW_API_CPU isn't a good choice here, as it makes the bufferQueue not connectable
260     * after disconnect. MEDIA or CAMERA are treated the same internally. The producer listener
261     * will be cleared after disconnect call.
262     */
263    producer->connect(/*api*/NATIVE_WINDOW_API_CAMERA, /*listener*/ctx);
264    jlong nativeCtx = reinterpret_cast<jlong>(ctx.get());
265
266    // Get the dimension and format of the producer.
267    sp<ANativeWindow> anw = producer;
268    int32_t width, height, format;
269    if ((res = anw->query(anw.get(), NATIVE_WINDOW_WIDTH, &width)) != OK) {
270        ALOGE("%s: Query Surface width failed: %s (%d)", __FUNCTION__, strerror(-res), res);
271        jniThrowRuntimeException(env, "Failed to query Surface width");
272        return 0;
273    }
274    ctx->setBufferWidth(width);
275
276    if ((res = anw->query(anw.get(), NATIVE_WINDOW_HEIGHT, &height)) != OK) {
277        ALOGE("%s: Query Surface height failed: %s (%d)", __FUNCTION__, strerror(-res), res);
278        jniThrowRuntimeException(env, "Failed to query Surface height");
279        return 0;
280    }
281    ctx->setBufferHeight(height);
282
283    if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &format)) != OK) {
284        ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res);
285        jniThrowRuntimeException(env, "Failed to query Surface format");
286        return 0;
287    }
288    ctx->setBufferFormat(format);
289    env->SetIntField(thiz, gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(format));
290
291
292    if (!isFormatOpaque(format)) {
293        res = native_window_set_usage(anw.get(), GRALLOC_USAGE_SW_WRITE_OFTEN);
294        if (res != OK) {
295            ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)",
296                  __FUNCTION__, static_cast<unsigned int>(GRALLOC_USAGE_SW_WRITE_OFTEN),
297                  format, strerror(-res), res);
298            jniThrowRuntimeException(env, "Failed to SW_WRITE_OFTEN configure usage");
299            return 0;
300        }
301    }
302
303    int minUndequeuedBufferCount = 0;
304    res = anw->query(anw.get(),
305                NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBufferCount);
306    if (res != OK) {
307        ALOGE("%s: Query producer undequeued buffer count failed: %s (%d)",
308                __FUNCTION__, strerror(-res), res);
309        jniThrowRuntimeException(env, "Query producer undequeued buffer count failed");
310        return 0;
311     }
312
313    size_t totalBufferCount = maxImages + minUndequeuedBufferCount;
314    res = native_window_set_buffer_count(anw.get(), totalBufferCount);
315    if (res != OK) {
316        ALOGE("%s: Set buffer count failed: %s (%d)", __FUNCTION__, strerror(-res), res);
317        jniThrowRuntimeException(env, "Set buffer count failed");
318        return 0;
319    }
320
321    if (ctx != 0) {
322        ctx->incStrong((void*)ImageWriter_init);
323    }
324    return nativeCtx;
325}
326
327static void ImageWriter_dequeueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) {
328    ALOGV("%s", __FUNCTION__);
329    JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
330    if (ctx == NULL || thiz == NULL) {
331        jniThrowException(env, "java/lang/IllegalStateException",
332                "ImageWriterContext is not initialized");
333        return;
334    }
335
336    sp<ANativeWindow> anw = ctx->getProducer();
337    android_native_buffer_t *anb = NULL;
338    int fenceFd = -1;
339    status_t res = anw->dequeueBuffer(anw.get(), &anb, &fenceFd);
340    if (res != OK) {
341        ALOGE("%s: Dequeue buffer failed: %s (%d)", __FUNCTION__, strerror(-res), res);
342        switch (res) {
343            case NO_INIT:
344                jniThrowException(env, "java/lang/IllegalStateException",
345                    "Surface has been abandoned");
346                break;
347            default:
348                // TODO: handle other error cases here.
349                jniThrowRuntimeException(env, "dequeue buffer failed");
350        }
351        return;
352    }
353    // New GraphicBuffer object doesn't own the handle, thus the native buffer
354    // won't be freed when this object is destroyed.
355    sp<GraphicBuffer> buffer(new GraphicBuffer(anb, /*keepOwnership*/false));
356
357    // Note that:
358    // 1. No need to lock buffer now, will only lock it when the first getPlanes() is called.
359    // 2. Fence will be saved to mNativeFenceFd, and will consumed by lock/queue/cancel buffer
360    //    later.
361    // 3. need use lockAsync here, as it will handle the dequeued fence for us automatically.
362
363    // Finally, set the native info into image object.
364    Image_setNativeContext(env, image, buffer, fenceFd);
365}
366
367static void ImageWriter_close(JNIEnv* env, jobject thiz, jlong nativeCtx) {
368    ALOGV("%s:", __FUNCTION__);
369    JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
370    if (ctx == NULL || thiz == NULL) {
371        // ImageWriter is already closed.
372        return;
373    }
374
375    ANativeWindow* producer = ctx->getProducer();
376    if (producer != NULL) {
377        /**
378         * NATIVE_WINDOW_API_CPU isn't a good choice here, as it makes the bufferQueue not
379         * connectable after disconnect. MEDIA or CAMERA are treated the same internally.
380         * The producer listener will be cleared after disconnect call.
381         */
382        status_t res = native_window_api_disconnect(producer, /*api*/NATIVE_WINDOW_API_CAMERA);
383        /**
384         * This is not an error. if client calling process dies, the window will
385         * also die and all calls to it will return DEAD_OBJECT, thus it's already
386         * "disconnected"
387         */
388        if (res == DEAD_OBJECT) {
389            ALOGW("%s: While disconnecting ImageWriter from native window, the"
390                    " native window died already", __FUNCTION__);
391        } else if (res != OK) {
392            ALOGE("%s: native window disconnect failed: %s (%d)",
393                    __FUNCTION__, strerror(-res), res);
394            jniThrowRuntimeException(env, "Native window disconnect failed");
395            return;
396        }
397    }
398
399    ctx->decStrong((void*)ImageWriter_init);
400}
401
402static void ImageWriter_cancelImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) {
403    ALOGV("%s", __FUNCTION__);
404    JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
405    if (ctx == NULL || thiz == NULL) {
406        jniThrowException(env, "java/lang/IllegalStateException",
407                "ImageWriterContext is not initialized");
408        return;
409    }
410
411    sp<ANativeWindow> anw = ctx->getProducer();
412
413    GraphicBuffer *buffer = NULL;
414    int fenceFd = -1;
415    Image_getNativeContext(env, image, &buffer, &fenceFd);
416    if (buffer == NULL) {
417        jniThrowException(env, "java/lang/IllegalStateException",
418                "Image is not initialized");
419        return;
420    }
421
422    // Unlock the image if it was locked
423    Image_unlockIfLocked(env, image);
424
425    anw->cancelBuffer(anw.get(), buffer, fenceFd);
426
427    Image_setNativeContext(env, image, NULL, -1);
428}
429
430static void ImageWriter_queueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image,
431        jlong timestampNs, jint left, jint top, jint right, jint bottom) {
432    ALOGV("%s", __FUNCTION__);
433    JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
434    if (ctx == NULL || thiz == NULL) {
435        jniThrowException(env, "java/lang/IllegalStateException",
436                "ImageWriterContext is not initialized");
437        return;
438    }
439
440    status_t res = OK;
441    sp<ANativeWindow> anw = ctx->getProducer();
442
443    GraphicBuffer *buffer = NULL;
444    int fenceFd = -1;
445    Image_getNativeContext(env, image, &buffer, &fenceFd);
446    if (buffer == NULL) {
447        jniThrowException(env, "java/lang/IllegalStateException",
448                "Image is not initialized");
449        return;
450    }
451
452    // Unlock image if it was locked.
453    Image_unlockIfLocked(env, image);
454
455    // Set timestamp
456    ALOGV("timestamp to be queued: %" PRId64, timestampNs);
457    res = native_window_set_buffers_timestamp(anw.get(), timestampNs);
458    if (res != OK) {
459        jniThrowRuntimeException(env, "Set timestamp failed");
460        return;
461    }
462
463    // Set crop
464    android_native_rect_t cropRect;
465    cropRect.left = left;
466    cropRect.top = top;
467    cropRect.right = right;
468    cropRect.bottom = bottom;
469    res = native_window_set_crop(anw.get(), &cropRect);
470    if (res != OK) {
471        jniThrowRuntimeException(env, "Set crop rect failed");
472        return;
473    }
474
475    // Finally, queue input buffer
476    res = anw->queueBuffer(anw.get(), buffer, fenceFd);
477    if (res != OK) {
478        ALOGE("%s: Queue buffer failed: %s (%d)", __FUNCTION__, strerror(-res), res);
479        switch (res) {
480            case NO_INIT:
481                jniThrowException(env, "java/lang/IllegalStateException",
482                    "Surface has been abandoned");
483                break;
484            default:
485                // TODO: handle other error cases here.
486                jniThrowRuntimeException(env, "Queue input buffer failed");
487        }
488        return;
489    }
490
491    // Clear the image native context: end of this image's lifecycle in public API.
492    Image_setNativeContext(env, image, NULL, -1);
493}
494
495static jint ImageWriter_attachAndQueueImage(JNIEnv* env, jobject thiz, jlong nativeCtx,
496        jlong nativeBuffer, jint imageFormat, jlong timestampNs, jint left, jint top,
497        jint right, jint bottom) {
498    ALOGV("%s", __FUNCTION__);
499    JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
500    if (ctx == NULL || thiz == NULL) {
501        jniThrowException(env, "java/lang/IllegalStateException",
502                "ImageWriterContext is not initialized");
503        return -1;
504    }
505
506    sp<Surface> surface = ctx->getProducer();
507    status_t res = OK;
508    if (!isFormatOpaque(imageFormat)) {
509        // TODO: need implement, see b/19962027
510        jniThrowRuntimeException(env,
511                "nativeAttachImage for non-opaque image is not implement yet!!!");
512        return -1;
513    }
514
515    if (!isFormatOpaque(ctx->getBufferFormat())) {
516        jniThrowException(env, "java/lang/IllegalStateException",
517                "Trying to attach an opaque image into a non-opaque ImageWriter");
518        return -1;
519    }
520
521    // Image is guaranteed to be from ImageReader at this point, so it is safe to
522    // cast to BufferItem pointer.
523    BufferItem* opaqueBuffer = reinterpret_cast<BufferItem*>(nativeBuffer);
524    if (opaqueBuffer == NULL) {
525        jniThrowException(env, "java/lang/IllegalStateException",
526                "Image is not initialized or already closed");
527        return -1;
528    }
529
530    // Step 1. Attach Image
531    res = surface->attachBuffer(opaqueBuffer->mGraphicBuffer.get());
532    if (res != OK) {
533        ALOGE("Attach image failed: %s (%d)", strerror(-res), res);
534        switch (res) {
535            case NO_INIT:
536                jniThrowException(env, "java/lang/IllegalStateException",
537                    "Surface has been abandoned");
538                break;
539            default:
540                // TODO: handle other error cases here.
541                jniThrowRuntimeException(env, "nativeAttachImage failed!!!");
542        }
543        return res;
544    }
545    sp < ANativeWindow > anw = surface;
546
547    // Step 2. Set timestamp and crop. Note that we do not need unlock the image because
548    // it was not locked.
549    ALOGV("timestamp to be queued: %" PRId64, timestampNs);
550    res = native_window_set_buffers_timestamp(anw.get(), timestampNs);
551    if (res != OK) {
552        jniThrowRuntimeException(env, "Set timestamp failed");
553        return res;
554    }
555
556    android_native_rect_t cropRect;
557    cropRect.left = left;
558    cropRect.top = top;
559    cropRect.right = right;
560    cropRect.bottom = bottom;
561    res = native_window_set_crop(anw.get(), &cropRect);
562    if (res != OK) {
563        jniThrowRuntimeException(env, "Set crop rect failed");
564        return res;
565    }
566
567    // Step 3. Queue Image.
568    res = anw->queueBuffer(anw.get(), opaqueBuffer->mGraphicBuffer.get(), /*fenceFd*/
569            -1);
570    if (res != OK) {
571        ALOGE("%s: Queue buffer failed: %s (%d)", __FUNCTION__, strerror(-res), res);
572        switch (res) {
573            case NO_INIT:
574                jniThrowException(env, "java/lang/IllegalStateException",
575                    "Surface has been abandoned");
576                break;
577            default:
578                // TODO: handle other error cases here.
579                jniThrowRuntimeException(env, "Queue input buffer failed");
580        }
581        return res;
582    }
583
584    // Do not set the image native context. Since it would overwrite the existing native context
585    // of the image that is from ImageReader, the subsequent image close will run into issues.
586
587    return res;
588}
589
590// --------------------------Image methods---------------------------------------
591
592static void Image_getNativeContext(JNIEnv* env, jobject thiz,
593        GraphicBuffer** buffer, int* fenceFd) {
594    ALOGV("%s", __FUNCTION__);
595    if (buffer != NULL) {
596        GraphicBuffer *gb = reinterpret_cast<GraphicBuffer *>
597                  (env->GetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer));
598        *buffer = gb;
599    }
600
601    if (fenceFd != NULL) {
602        *fenceFd = reinterpret_cast<jint>(env->GetIntField(
603                thiz, gSurfaceImageClassInfo.mNativeFenceFd));
604    }
605}
606
607static void Image_setNativeContext(JNIEnv* env, jobject thiz,
608        sp<GraphicBuffer> buffer, int fenceFd) {
609    ALOGV("%s:", __FUNCTION__);
610    GraphicBuffer* p = NULL;
611    Image_getNativeContext(env, thiz, &p, /*fenceFd*/NULL);
612    if (buffer != 0) {
613        buffer->incStrong((void*)Image_setNativeContext);
614    }
615    if (p) {
616        p->decStrong((void*)Image_setNativeContext);
617    }
618    env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer,
619            reinterpret_cast<jlong>(buffer.get()));
620
621    env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast<jint>(fenceFd));
622}
623
624static void Image_unlockIfLocked(JNIEnv* env, jobject thiz) {
625    ALOGV("%s", __FUNCTION__);
626    GraphicBuffer* buffer;
627    Image_getNativeContext(env, thiz, &buffer, NULL);
628    if (buffer == NULL) {
629        jniThrowException(env, "java/lang/IllegalStateException",
630                "Image is not initialized");
631        return;
632    }
633
634    // Is locked?
635    bool isLocked = false;
636    jobject planes = NULL;
637    if (!isFormatOpaque(buffer->getPixelFormat())) {
638        planes = env->GetObjectField(thiz, gSurfaceImageClassInfo.mPlanes);
639    }
640    isLocked = (planes != NULL);
641    if (isLocked) {
642        // no need to use fence here, as we it will be consumed by either cancel or queue buffer.
643        status_t res = buffer->unlock();
644        if (res != OK) {
645            jniThrowRuntimeException(env, "unlock buffer failed");
646        }
647        ALOGV("Successfully unlocked the image");
648    }
649}
650
651static jint Image_getWidth(JNIEnv* env, jobject thiz) {
652    ALOGV("%s", __FUNCTION__);
653    GraphicBuffer* buffer;
654    Image_getNativeContext(env, thiz, &buffer, NULL);
655    if (buffer == NULL) {
656        jniThrowException(env, "java/lang/IllegalStateException",
657                "Image is not initialized");
658        return -1;
659    }
660
661    return buffer->getWidth();
662}
663
664static jint Image_getHeight(JNIEnv* env, jobject thiz) {
665    ALOGV("%s", __FUNCTION__);
666    GraphicBuffer* buffer;
667    Image_getNativeContext(env, thiz, &buffer, NULL);
668    if (buffer == NULL) {
669        jniThrowException(env, "java/lang/IllegalStateException",
670                "Image is not initialized");
671        return -1;
672    }
673
674    return buffer->getHeight();
675}
676
677// Some formats like JPEG defined with different values between android.graphics.ImageFormat and
678// graphics.h, need convert to the one defined in graphics.h here.
679static int Image_getPixelFormat(JNIEnv* env, int format) {
680    int jpegFormat;
681    jfieldID fid;
682
683    ALOGV("%s: format = 0x%x", __FUNCTION__, format);
684
685    jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat");
686    ALOG_ASSERT(imageFormatClazz != NULL);
687
688    fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I");
689    jpegFormat = env->GetStaticIntField(imageFormatClazz, fid);
690
691    // Translate the JPEG to BLOB for camera purpose.
692    if (format == jpegFormat) {
693        format = HAL_PIXEL_FORMAT_BLOB;
694    }
695
696    return format;
697}
698
699static jint Image_getFormat(JNIEnv* env, jobject thiz) {
700    ALOGV("%s", __FUNCTION__);
701    GraphicBuffer* buffer;
702    Image_getNativeContext(env, thiz, &buffer, NULL);
703    if (buffer == NULL) {
704        jniThrowException(env, "java/lang/IllegalStateException",
705                "Image is not initialized");
706        return 0;
707    }
708
709    return Image_getPixelFormat(env, buffer->getPixelFormat());
710}
711
712static void Image_setFenceFd(JNIEnv* env, jobject thiz, int fenceFd) {
713    ALOGV("%s:", __FUNCTION__);
714    env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast<jint>(fenceFd));
715}
716
717static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) {
718    ALOGV("%s", __FUNCTION__);
719    GraphicBuffer* buffer;
720    int fenceFd = -1;
721    Image_getNativeContext(env, thiz, &buffer, &fenceFd);
722    if (buffer == NULL) {
723        jniThrowException(env, "java/lang/IllegalStateException",
724                "Image is not initialized");
725        return;
726    }
727
728    void* pData = NULL;
729    android_ycbcr ycbcr = android_ycbcr();
730    status_t res;
731    int format = Image_getFormat(env, thiz);
732    int flexFormat = format;
733    if (isPossiblyYUV(format)) {
734        // ImageWriter doesn't use crop by itself, app sets it, use the no crop version.
735        res = buffer->lockAsyncYCbCr(GRALLOC_USAGE_SW_WRITE_OFTEN, &ycbcr, fenceFd);
736        // Clear the fenceFd as it is already consumed by lock call.
737        Image_setFenceFd(env, thiz, /*fenceFd*/-1);
738        if (res != OK) {
739            jniThrowRuntimeException(env, "lockAsyncYCbCr failed for YUV buffer");
740            return;
741        }
742        pData = ycbcr.y;
743        flexFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;
744    }
745
746    // lockAsyncYCbCr for YUV is unsuccessful.
747    if (pData == NULL) {
748        res = buffer->lockAsync(GRALLOC_USAGE_SW_WRITE_OFTEN, &pData, fenceFd);
749        if (res != OK) {
750            jniThrowRuntimeException(env, "lockAsync failed");
751            return;
752        }
753    }
754
755    image->data = reinterpret_cast<uint8_t*>(pData);
756    image->width = buffer->getWidth();
757    image->height = buffer->getHeight();
758    image->format = format;
759    image->flexFormat = flexFormat;
760    image->stride = (ycbcr.y != NULL) ? static_cast<uint32_t>(ycbcr.ystride) : buffer->getStride();
761
762    image->dataCb = reinterpret_cast<uint8_t*>(ycbcr.cb);
763    image->dataCr = reinterpret_cast<uint8_t*>(ycbcr.cr);
764    image->chromaStride = static_cast<uint32_t>(ycbcr.cstride);
765    image->chromaStep = static_cast<uint32_t>(ycbcr.chroma_step);
766    ALOGV("Successfully locked the image");
767    // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer,
768    // and we don't set them here.
769}
770
771static bool usingRGBAToJpegOverride(int32_t bufferFormat, int32_t writerCtxFormat) {
772    return writerCtxFormat == HAL_PIXEL_FORMAT_BLOB && bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888;
773}
774
775static int32_t applyFormatOverrides(int32_t bufferFormat, int32_t writerCtxFormat)
776{
777    // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW
778    // write limitations for some platforms (b/17379185).
779    if (usingRGBAToJpegOverride(bufferFormat, writerCtxFormat)) {
780        return HAL_PIXEL_FORMAT_BLOB;
781    }
782    return bufferFormat;
783}
784
785static uint32_t Image_getJpegSize(LockedImage* buffer, bool usingRGBAOverride) {
786    ALOGV("%s", __FUNCTION__);
787    ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
788    uint32_t size = 0;
789    uint32_t width = buffer->width;
790    uint8_t* jpegBuffer = buffer->data;
791
792    if (usingRGBAOverride) {
793        width = (buffer->width + buffer->stride * (buffer->height - 1)) * 4;
794    }
795
796    // First check for JPEG transport header at the end of the buffer
797    uint8_t* header = jpegBuffer + (width - sizeof(struct camera3_jpeg_blob));
798    struct camera3_jpeg_blob *blob = (struct camera3_jpeg_blob*)(header);
799    if (blob->jpeg_blob_id == CAMERA3_JPEG_BLOB_ID) {
800        size = blob->jpeg_size;
801        ALOGV("%s: Jpeg size = %d", __FUNCTION__, size);
802    }
803
804    // failed to find size, default to whole buffer
805    if (size == 0) {
806        /*
807         * This is a problem because not including the JPEG header
808         * means that in certain rare situations a regular JPEG blob
809         * will be misidentified as having a header, in which case
810         * we will get a garbage size value.
811         */
812        ALOGW("%s: No JPEG header detected, defaulting to size=width=%d",
813                __FUNCTION__, width);
814        size = width;
815    }
816
817    return size;
818}
819
820static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
821        int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {
822    ALOGV("%s", __FUNCTION__);
823    ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
824    ALOG_ASSERT(base != NULL, "base is NULL!!!");
825    ALOG_ASSERT(size != NULL, "size is NULL!!!");
826    ALOG_ASSERT(pixelStride != NULL, "pixelStride is NULL!!!");
827    ALOG_ASSERT(rowStride != NULL, "rowStride is NULL!!!");
828    ALOG_ASSERT((idx < IMAGE_WRITER_MAX_NUM_PLANES) && (idx >= 0));
829
830    ALOGV("%s: buffer: %p", __FUNCTION__, buffer);
831
832    uint32_t dataSize, ySize, cSize, cStride;
833    uint32_t pStride = 0, rStride = 0;
834    uint8_t *cb, *cr;
835    uint8_t *pData = NULL;
836    int bytesPerPixel = 0;
837
838    dataSize = ySize = cSize = cStride = 0;
839    int32_t fmt = buffer->flexFormat;
840
841    bool usingRGBAOverride = usingRGBAToJpegOverride(fmt, writerFormat);
842    fmt = applyFormatOverrides(fmt, writerFormat);
843    switch (fmt) {
844        case HAL_PIXEL_FORMAT_YCbCr_420_888:
845            pData =
846                (idx == 0) ?
847                    buffer->data :
848                (idx == 1) ?
849                    buffer->dataCb :
850                buffer->dataCr;
851            // only map until last pixel
852            if (idx == 0) {
853                pStride = 1;
854                rStride = buffer->stride;
855                dataSize = buffer->stride * (buffer->height - 1) + buffer->width;
856            } else {
857                pStride = buffer->chromaStep;
858                rStride = buffer->chromaStride;
859                dataSize = buffer->chromaStride * (buffer->height / 2 - 1) +
860                        buffer->chromaStep * (buffer->width / 2 - 1) + 1;
861            }
862            break;
863        // NV21
864        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
865            cr = buffer->data + (buffer->stride * buffer->height);
866            cb = cr + 1;
867            // only map until last pixel
868            ySize = buffer->width * (buffer->height - 1) + buffer->width;
869            cSize = buffer->width * (buffer->height / 2 - 1) + buffer->width - 1;
870
871            pData =
872                (idx == 0) ?
873                    buffer->data :
874                (idx == 1) ?
875                    cb:
876                cr;
877
878            dataSize = (idx == 0) ? ySize : cSize;
879            pStride = (idx == 0) ? 1 : 2;
880            rStride = buffer->width;
881            break;
882        case HAL_PIXEL_FORMAT_YV12:
883            // Y and C stride need to be 16 pixel aligned.
884            LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
885                                "Stride is not 16 pixel aligned %d", buffer->stride);
886
887            ySize = buffer->stride * buffer->height;
888            cStride = ALIGN(buffer->stride / 2, 16);
889            cr = buffer->data + ySize;
890            cSize = cStride * buffer->height / 2;
891            cb = cr + cSize;
892
893            pData =
894                (idx == 0) ?
895                    buffer->data :
896                (idx == 1) ?
897                    cb :
898                cr;
899            dataSize = (idx == 0) ? ySize : cSize;
900            pStride = 1;
901            rStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16);
902            break;
903        case HAL_PIXEL_FORMAT_Y8:
904            // Single plane, 8bpp.
905            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
906
907            pData = buffer->data;
908            dataSize = buffer->stride * buffer->height;
909            pStride = 1;
910            rStride = buffer->stride;
911            break;
912        case HAL_PIXEL_FORMAT_Y16:
913            bytesPerPixel = 2;
914            // Single plane, 16bpp, strides are specified in pixels, not in bytes
915            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
916
917            pData = buffer->data;
918            dataSize = buffer->stride * buffer->height * bytesPerPixel;
919            pStride = bytesPerPixel;
920            rStride = buffer->stride * 2;
921            break;
922        case HAL_PIXEL_FORMAT_BLOB:
923            // Used for JPEG data, height must be 1, width == size, single plane.
924            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
925            ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height);
926
927            pData = buffer->data;
928            dataSize = Image_getJpegSize(buffer, usingRGBAOverride);
929            pStride = bytesPerPixel;
930            rowStride = 0;
931            break;
932        case HAL_PIXEL_FORMAT_RAW16:
933            // Single plane 16bpp bayer data.
934            bytesPerPixel = 2;
935            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
936            pData = buffer->data;
937            dataSize = buffer->stride * buffer->height * bytesPerPixel;
938            pStride = bytesPerPixel;
939            rStride = buffer->stride * 2;
940            break;
941        case HAL_PIXEL_FORMAT_RAW10:
942            // Single plane 10bpp bayer data.
943            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
944            LOG_ALWAYS_FATAL_IF(buffer->width % 4,
945                                "Width is not multiple of 4 %d", buffer->width);
946            LOG_ALWAYS_FATAL_IF(buffer->height % 2,
947                                "Height is not even %d", buffer->height);
948            LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 10 / 8),
949                                "stride (%d) should be at least %d",
950                                buffer->stride, buffer->width * 10 / 8);
951            pData = buffer->data;
952            dataSize = buffer->stride * buffer->height;
953            pStride = 0;
954            rStride = buffer->stride;
955            break;
956        case HAL_PIXEL_FORMAT_RGBA_8888:
957        case HAL_PIXEL_FORMAT_RGBX_8888:
958            // Single plane, 32bpp.
959            bytesPerPixel = 4;
960            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
961            pData = buffer->data;
962            dataSize = buffer->stride * buffer->height * bytesPerPixel;
963            pStride = bytesPerPixel;
964            rStride = buffer->stride * 4;
965            break;
966        case HAL_PIXEL_FORMAT_RGB_565:
967            // Single plane, 16bpp.
968            bytesPerPixel = 2;
969            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
970            pData = buffer->data;
971            dataSize = buffer->stride * buffer->height * bytesPerPixel;
972            pStride = bytesPerPixel;
973            rStride = buffer->stride * 2;
974            break;
975        case HAL_PIXEL_FORMAT_RGB_888:
976            // Single plane, 24bpp.
977            bytesPerPixel = 3;
978            ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
979            pData = buffer->data;
980            dataSize = buffer->stride * buffer->height * bytesPerPixel;
981            pStride = bytesPerPixel;
982            rStride = buffer->stride * 3;
983            break;
984        default:
985            jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
986                                 "Pixel format: 0x%x is unsupported", fmt);
987            break;
988    }
989
990    *base = pData;
991    *size = dataSize;
992    *pixelStride = pStride;
993    *rowStride = rStride;
994}
995
996static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
997        int numPlanes, int writerFormat) {
998    ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes);
999    int rowStride, pixelStride;
1000    uint8_t *pData;
1001    uint32_t dataSize;
1002    jobject byteBuffer;
1003
1004    int format = Image_getFormat(env, thiz);
1005    if (isFormatOpaque(format) && numPlanes > 0) {
1006        String8 msg;
1007        msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
1008                " must be 0", format, numPlanes);
1009        jniThrowException(env, "java/lang/IllegalArgumentException", msg.string());
1010        return NULL;
1011    }
1012
1013    jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz,
1014            /*initial_element*/NULL);
1015    if (surfacePlanes == NULL) {
1016        jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays,"
1017                " probably out of memory");
1018        return NULL;
1019    }
1020    if (isFormatOpaque(format)) {
1021        return surfacePlanes;
1022    }
1023
1024    // Buildup buffer info: rowStride, pixelStride and byteBuffers.
1025    LockedImage lockedImg = LockedImage();
1026    Image_getLockedImage(env, thiz, &lockedImg);
1027
1028    // Create all SurfacePlanes
1029    writerFormat = Image_getPixelFormat(env, writerFormat);
1030    for (int i = 0; i < numPlanes; i++) {
1031        Image_getLockedImageInfo(env, &lockedImg, i, writerFormat,
1032                &pData, &dataSize, &pixelStride, &rowStride);
1033        byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
1034        if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
1035            jniThrowException(env, "java/lang/IllegalStateException",
1036                    "Failed to allocate ByteBuffer");
1037            return NULL;
1038        }
1039
1040        // Finally, create this SurfacePlane.
1041        jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
1042                    gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
1043        env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
1044    }
1045
1046    return surfacePlanes;
1047}
1048
1049// -------------------------------Private convenience methods--------------------
1050
1051static bool isFormatOpaque(int format) {
1052    // Only treat IMPLEMENTATION_DEFINED as an opaque format for now.
1053    return format == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
1054}
1055
1056static bool isPossiblyYUV(PixelFormat format) {
1057    switch (static_cast<int>(format)) {
1058        case HAL_PIXEL_FORMAT_RGBA_8888:
1059        case HAL_PIXEL_FORMAT_RGBX_8888:
1060        case HAL_PIXEL_FORMAT_RGB_888:
1061        case HAL_PIXEL_FORMAT_RGB_565:
1062        case HAL_PIXEL_FORMAT_BGRA_8888:
1063        case HAL_PIXEL_FORMAT_Y8:
1064        case HAL_PIXEL_FORMAT_Y16:
1065        case HAL_PIXEL_FORMAT_RAW16:
1066        case HAL_PIXEL_FORMAT_RAW10:
1067        case HAL_PIXEL_FORMAT_RAW_OPAQUE:
1068        case HAL_PIXEL_FORMAT_BLOB:
1069        case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
1070            return false;
1071
1072        case HAL_PIXEL_FORMAT_YV12:
1073        case HAL_PIXEL_FORMAT_YCbCr_420_888:
1074        case HAL_PIXEL_FORMAT_YCbCr_422_SP:
1075        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
1076        case HAL_PIXEL_FORMAT_YCbCr_422_I:
1077        default:
1078            return true;
1079    }
1080}
1081
1082} // extern "C"
1083
1084// ----------------------------------------------------------------------------
1085
1086static JNINativeMethod gImageWriterMethods[] = {
1087    {"nativeClassInit",         "()V",                        (void*)ImageWriter_classInit },
1088    {"nativeInit",              "(Ljava/lang/Object;Landroid/view/Surface;I)J",
1089                                                              (void*)ImageWriter_init },
1090    {"nativeClose",              "(J)V",                      (void*)ImageWriter_close },
1091    {"nativeAttachAndQueueImage", "(JJIJIIII)I",          (void*)ImageWriter_attachAndQueueImage },
1092    {"nativeDequeueInputImage", "(JLandroid/media/Image;)V",  (void*)ImageWriter_dequeueImage },
1093    {"nativeQueueInputImage",   "(JLandroid/media/Image;JIIII)V",  (void*)ImageWriter_queueImage },
1094    {"cancelImage",             "(JLandroid/media/Image;)V",   (void*)ImageWriter_cancelImage },
1095};
1096
1097static JNINativeMethod gImageMethods[] = {
1098    {"nativeCreatePlanes",      "(II)[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;",
1099                                                              (void*)Image_createSurfacePlanes },
1100    {"nativeGetWidth",         "()I",                         (void*)Image_getWidth },
1101    {"nativeGetHeight",        "()I",                         (void*)Image_getHeight },
1102    {"nativeGetFormat",        "()I",                         (void*)Image_getFormat },
1103};
1104
1105int register_android_media_ImageWriter(JNIEnv *env) {
1106
1107    int ret1 = AndroidRuntime::registerNativeMethods(env,
1108                   "android/media/ImageWriter", gImageWriterMethods, NELEM(gImageWriterMethods));
1109
1110    int ret2 = AndroidRuntime::registerNativeMethods(env,
1111                   "android/media/ImageWriter$WriterSurfaceImage", gImageMethods, NELEM(gImageMethods));
1112
1113    return (ret1 || ret2);
1114}
1115
1116