android_media_ImageWriter.cpp revision a58271419ee5b4d8d2a6e25c602a0c40719f33e0
1/* 2 * Copyright 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "ImageWriter_JNI" 19#include <utils/Log.h> 20#include <utils/String8.h> 21 22#include <gui/IProducerListener.h> 23#include <gui/Surface.h> 24#include <gui/CpuConsumer.h> 25#include <android_runtime/AndroidRuntime.h> 26#include <android_runtime/android_view_Surface.h> 27#include <camera3.h> 28 29#include <jni.h> 30#include <JNIHelp.h> 31 32#include <stdint.h> 33#include <inttypes.h> 34 35#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) ) 36 37#define IMAGE_BUFFER_JNI_ID "mNativeBuffer" 38 39// ---------------------------------------------------------------------------- 40 41using namespace android; 42 43enum { 44 IMAGE_WRITER_MAX_NUM_PLANES = 3, 45}; 46 47static struct { 48 jmethodID postEventFromNative; 49 jfieldID mWriterFormat; 50} gImageWriterClassInfo; 51 52static struct { 53 jfieldID mNativeBuffer; 54 jfieldID mNativeFenceFd; 55 jfieldID mPlanes; 56} gSurfaceImageClassInfo; 57 58static struct { 59 jclass clazz; 60 jmethodID ctor; 61} gSurfacePlaneClassInfo; 62 63typedef CpuConsumer::LockedBuffer LockedImage; 64 65// ---------------------------------------------------------------------------- 66 67class JNIImageWriterContext : public BnProducerListener { 68public: 69 JNIImageWriterContext(JNIEnv* env, jobject weakThiz, jclass clazz); 70 71 virtual ~JNIImageWriterContext(); 72 73 // Implementation of IProducerListener, used to notify the ImageWriter that the consumer 74 // has returned a buffer and it is ready for ImageWriter to dequeue. 75 virtual void onBufferReleased(); 76 77 void setProducer(const sp<Surface>& producer) { mProducer = producer; } 78 Surface* getProducer() { return mProducer.get(); } 79 80 void setBufferFormat(int format) { mFormat = format; } 81 int getBufferFormat() { return mFormat; } 82 83 void setBufferWidth(int width) { mWidth = width; } 84 int getBufferWidth() { return mWidth; } 85 86 void setBufferHeight(int height) { mHeight = height; } 87 int getBufferHeight() { return mHeight; } 88 89private: 90 static JNIEnv* getJNIEnv(bool* needsDetach); 91 static void detachJNI(); 92 93 sp<Surface> mProducer; 94 jobject mWeakThiz; 95 jclass mClazz; 96 int mFormat; 97 int mWidth; 98 int mHeight; 99}; 100 101JNIImageWriterContext::JNIImageWriterContext(JNIEnv* env, jobject weakThiz, jclass clazz) : 102 mWeakThiz(env->NewGlobalRef(weakThiz)), 103 mClazz((jclass)env->NewGlobalRef(clazz)), 104 mFormat(0), 105 mWidth(-1), 106 mHeight(-1) { 107} 108 109JNIImageWriterContext::~JNIImageWriterContext() { 110 ALOGV("%s", __FUNCTION__); 111 bool needsDetach = false; 112 JNIEnv* env = getJNIEnv(&needsDetach); 113 if (env != NULL) { 114 env->DeleteGlobalRef(mWeakThiz); 115 env->DeleteGlobalRef(mClazz); 116 } else { 117 ALOGW("leaking JNI object references"); 118 } 119 if (needsDetach) { 120 detachJNI(); 121 } 122 123 mProducer.clear(); 124} 125 126JNIEnv* JNIImageWriterContext::getJNIEnv(bool* needsDetach) { 127 ALOGV("%s", __FUNCTION__); 128 LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!"); 129 *needsDetach = false; 130 JNIEnv* env = AndroidRuntime::getJNIEnv(); 131 if (env == NULL) { 132 JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL}; 133 JavaVM* vm = AndroidRuntime::getJavaVM(); 134 int result = vm->AttachCurrentThread(&env, (void*) &args); 135 if (result != JNI_OK) { 136 ALOGE("thread attach failed: %#x", result); 137 return NULL; 138 } 139 *needsDetach = true; 140 } 141 return env; 142} 143 144void JNIImageWriterContext::detachJNI() { 145 ALOGV("%s", __FUNCTION__); 146 JavaVM* vm = AndroidRuntime::getJavaVM(); 147 int result = vm->DetachCurrentThread(); 148 if (result != JNI_OK) { 149 ALOGE("thread detach failed: %#x", result); 150 } 151} 152 153void JNIImageWriterContext::onBufferReleased() { 154 ALOGV("%s: buffer released", __FUNCTION__); 155 bool needsDetach = false; 156 JNIEnv* env = getJNIEnv(&needsDetach); 157 if (env != NULL) { 158 // Detach the buffer every time when a buffer consumption is done, 159 // need let this callback give a BufferItem, then only detach if it was attached to this 160 // Writer. Do the detach unconditionally for opaque format now. see b/19977520 161 if (mFormat == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED) { 162 sp<Fence> fence; 163 ANativeWindowBuffer* buffer; 164 ALOGV("%s: One buffer is detached", __FUNCTION__); 165 mProducer->detachNextBuffer(&buffer, &fence); 166 } 167 168 env->CallStaticVoidMethod(mClazz, gImageWriterClassInfo.postEventFromNative, mWeakThiz); 169 } else { 170 ALOGW("onBufferReleased event will not posted"); 171 } 172 173 if (needsDetach) { 174 detachJNI(); 175 } 176} 177 178// ---------------------------------------------------------------------------- 179 180extern "C" { 181 182// -------------------------------Private method declarations-------------- 183 184static bool isPossiblyYUV(PixelFormat format); 185static void Image_setNativeContext(JNIEnv* env, jobject thiz, 186 sp<GraphicBuffer> buffer, int fenceFd); 187static void Image_getNativeContext(JNIEnv* env, jobject thiz, 188 GraphicBuffer** buffer, int* fenceFd); 189static void Image_unlockIfLocked(JNIEnv* env, jobject thiz); 190static bool isFormatOpaque(int format); 191 192// --------------------------ImageWriter methods--------------------------------------- 193 194static void ImageWriter_classInit(JNIEnv* env, jclass clazz) { 195 ALOGV("%s:", __FUNCTION__); 196 jclass imageClazz = env->FindClass("android/media/ImageWriter$WriterSurfaceImage"); 197 LOG_ALWAYS_FATAL_IF(imageClazz == NULL, 198 "can't find android/media/ImageWriter$WriterSurfaceImage"); 199 gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID( 200 imageClazz, IMAGE_BUFFER_JNI_ID, "J"); 201 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL, 202 "can't find android/media/ImageWriter$WriterSurfaceImage.%s", IMAGE_BUFFER_JNI_ID); 203 204 gSurfaceImageClassInfo.mNativeFenceFd = env->GetFieldID( 205 imageClazz, "mNativeFenceFd", "I"); 206 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeFenceFd == NULL, 207 "can't find android/media/ImageWriter$WriterSurfaceImage.mNativeFenceFd"); 208 209 gSurfaceImageClassInfo.mPlanes = env->GetFieldID( 210 imageClazz, "mPlanes", "[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;"); 211 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL, 212 "can't find android/media/ImageWriter$WriterSurfaceImage.mPlanes"); 213 214 gImageWriterClassInfo.postEventFromNative = env->GetStaticMethodID( 215 clazz, "postEventFromNative", "(Ljava/lang/Object;)V"); 216 LOG_ALWAYS_FATAL_IF(gImageWriterClassInfo.postEventFromNative == NULL, 217 "can't find android/media/ImageWriter.postEventFromNative"); 218 219 gImageWriterClassInfo.mWriterFormat = env->GetFieldID( 220 clazz, "mWriterFormat", "I"); 221 LOG_ALWAYS_FATAL_IF(gImageWriterClassInfo.mWriterFormat == NULL, 222 "can't find android/media/ImageWriter.mWriterFormat"); 223 224 jclass planeClazz = env->FindClass("android/media/ImageWriter$WriterSurfaceImage$SurfacePlane"); 225 LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class"); 226 // FindClass only gives a local reference of jclass object. 227 gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz); 228 gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>", 229 "(Landroid/media/ImageWriter$WriterSurfaceImage;IILjava/nio/ByteBuffer;)V"); 230 LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL, 231 "Can not find SurfacePlane constructor"); 232} 233 234static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobject jsurface, 235 jint maxImages) { 236 status_t res; 237 238 ALOGV("%s: maxImages:%d", __FUNCTION__, maxImages); 239 240 sp<Surface> surface(android_view_Surface_getSurface(env, jsurface)); 241 if (surface == NULL) { 242 jniThrowException(env, 243 "java/lang/IllegalArgumentException", 244 "The surface has been released"); 245 return 0; 246 } 247 sp<IGraphicBufferProducer> bufferProducer = surface->getIGraphicBufferProducer(); 248 249 jclass clazz = env->GetObjectClass(thiz); 250 if (clazz == NULL) { 251 jniThrowRuntimeException(env, "Can't find android/graphics/ImageWriter"); 252 return 0; 253 } 254 sp<JNIImageWriterContext> ctx(new JNIImageWriterContext(env, weakThiz, clazz)); 255 256 sp<Surface> producer = new Surface(bufferProducer, /*controlledByApp*/false); 257 ctx->setProducer(producer); 258 /** 259 * NATIVE_WINDOW_API_CPU isn't a good choice here, as it makes the bufferQueue not connectable 260 * after disconnect. MEDIA or CAMERA are treated the same internally. The producer listener 261 * will be cleared after disconnect call. 262 */ 263 producer->connect(/*api*/NATIVE_WINDOW_API_CAMERA, /*listener*/ctx); 264 jlong nativeCtx = reinterpret_cast<jlong>(ctx.get()); 265 266 // Get the dimension and format of the producer. 267 sp<ANativeWindow> anw = producer; 268 int32_t width, height, format; 269 if ((res = anw->query(anw.get(), NATIVE_WINDOW_WIDTH, &width)) != OK) { 270 ALOGE("%s: Query Surface width failed: %s (%d)", __FUNCTION__, strerror(-res), res); 271 jniThrowRuntimeException(env, "Failed to query Surface width"); 272 return 0; 273 } 274 ctx->setBufferWidth(width); 275 276 if ((res = anw->query(anw.get(), NATIVE_WINDOW_HEIGHT, &height)) != OK) { 277 ALOGE("%s: Query Surface height failed: %s (%d)", __FUNCTION__, strerror(-res), res); 278 jniThrowRuntimeException(env, "Failed to query Surface height"); 279 return 0; 280 } 281 ctx->setBufferHeight(height); 282 283 if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &format)) != OK) { 284 ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res); 285 jniThrowRuntimeException(env, "Failed to query Surface format"); 286 return 0; 287 } 288 ctx->setBufferFormat(format); 289 env->SetIntField(thiz, gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(format)); 290 291 292 if (!isFormatOpaque(format)) { 293 res = native_window_set_usage(anw.get(), GRALLOC_USAGE_SW_WRITE_OFTEN); 294 if (res != OK) { 295 ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)", 296 __FUNCTION__, GRALLOC_USAGE_SW_WRITE_OFTEN, format, strerror(-res), res); 297 jniThrowRuntimeException(env, "Failed to SW_WRITE_OFTEN configure usage"); 298 return 0; 299 } 300 } 301 302 int minUndequeuedBufferCount = 0; 303 res = anw->query(anw.get(), 304 NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBufferCount); 305 if (res != OK) { 306 ALOGE("%s: Query producer undequeued buffer count failed: %s (%d)", 307 __FUNCTION__, strerror(-res), res); 308 jniThrowRuntimeException(env, "Query producer undequeued buffer count failed"); 309 return 0; 310 } 311 312 size_t totalBufferCount = maxImages + minUndequeuedBufferCount; 313 res = native_window_set_buffer_count(anw.get(), totalBufferCount); 314 if (res != OK) { 315 ALOGE("%s: Set buffer count failed: %s (%d)", __FUNCTION__, strerror(-res), res); 316 jniThrowRuntimeException(env, "Set buffer count failed"); 317 return 0; 318 } 319 320 if (ctx != 0) { 321 ctx->incStrong((void*)ImageWriter_init); 322 } 323 return nativeCtx; 324} 325 326static void ImageWriter_dequeueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) { 327 ALOGV("%s", __FUNCTION__); 328 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx); 329 if (ctx == NULL || thiz == NULL) { 330 jniThrowException(env, "java/lang/IllegalStateException", 331 "ImageWriterContext is not initialized"); 332 return; 333 } 334 335 sp<ANativeWindow> anw = ctx->getProducer(); 336 android_native_buffer_t *anb = NULL; 337 int fenceFd = -1; 338 status_t res = anw->dequeueBuffer(anw.get(), &anb, &fenceFd); 339 if (res != OK) { 340 // TODO: handle different error cases here. 341 ALOGE("%s: Dequeue buffer failed: %s (%d)", __FUNCTION__, strerror(-res), res); 342 jniThrowRuntimeException(env, "dequeue buffer failed"); 343 return; 344 } 345 // New GraphicBuffer object doesn't own the handle, thus the native buffer 346 // won't be freed when this object is destroyed. 347 sp<GraphicBuffer> buffer(new GraphicBuffer(anb, /*keepOwnership*/false)); 348 349 // Note that: 350 // 1. No need to lock buffer now, will only lock it when the first getPlanes() is called. 351 // 2. Fence will be saved to mNativeFenceFd, and will consumed by lock/queue/cancel buffer 352 // later. 353 // 3. need use lockAsync here, as it will handle the dequeued fence for us automatically. 354 355 // Finally, set the native info into image object. 356 Image_setNativeContext(env, image, buffer, fenceFd); 357} 358 359static void ImageWriter_close(JNIEnv* env, jobject thiz, jlong nativeCtx) { 360 ALOGV("%s:", __FUNCTION__); 361 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx); 362 if (ctx == NULL || thiz == NULL) { 363 jniThrowException(env, "java/lang/IllegalStateException", 364 "ImageWriterContext is not initialized"); 365 return; 366 } 367 368 ANativeWindow* producer = ctx->getProducer(); 369 if (producer != NULL) { 370 /** 371 * NATIVE_WINDOW_API_CPU isn't a good choice here, as it makes the bufferQueue not 372 * connectable after disconnect. MEDIA or CAMERA are treated the same internally. 373 * The producer listener will be cleared after disconnect call. 374 */ 375 status_t res = native_window_api_disconnect(producer, /*api*/NATIVE_WINDOW_API_CAMERA); 376 /** 377 * This is not an error. if client calling process dies, the window will 378 * also die and all calls to it will return DEAD_OBJECT, thus it's already 379 * "disconnected" 380 */ 381 if (res == DEAD_OBJECT) { 382 ALOGW("%s: While disconnecting ImageWriter from native window, the" 383 " native window died already", __FUNCTION__); 384 } else if (res != OK) { 385 ALOGE("%s: native window disconnect failed: %s (%d)", 386 __FUNCTION__, strerror(-res), res); 387 jniThrowRuntimeException(env, "Native window disconnect failed"); 388 return; 389 } 390 } 391 392 ctx->decStrong((void*)ImageWriter_init); 393} 394 395static void ImageWriter_cancelImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) { 396 ALOGV("%s", __FUNCTION__); 397 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx); 398 if (ctx == NULL || thiz == NULL) { 399 jniThrowException(env, "java/lang/IllegalStateException", 400 "ImageWriterContext is not initialized"); 401 return; 402 } 403 404 sp<ANativeWindow> anw = ctx->getProducer(); 405 406 GraphicBuffer *buffer = NULL; 407 int fenceFd = -1; 408 Image_getNativeContext(env, image, &buffer, &fenceFd); 409 if (buffer == NULL) { 410 jniThrowException(env, "java/lang/IllegalStateException", 411 "Image is not initialized"); 412 return; 413 } 414 415 // Unlock the image if it was locked 416 Image_unlockIfLocked(env, image); 417 418 anw->cancelBuffer(anw.get(), buffer, fenceFd); 419 420 Image_setNativeContext(env, image, NULL, -1); 421} 422 423static void ImageWriter_queueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image, 424 jlong timestampNs, jint left, jint top, jint right, jint bottom) { 425 ALOGV("%s", __FUNCTION__); 426 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx); 427 if (ctx == NULL || thiz == NULL) { 428 jniThrowException(env, "java/lang/IllegalStateException", 429 "ImageWriterContext is not initialized"); 430 return; 431 } 432 433 status_t res = OK; 434 sp<ANativeWindow> anw = ctx->getProducer(); 435 436 GraphicBuffer *buffer = NULL; 437 int fenceFd = -1; 438 Image_getNativeContext(env, image, &buffer, &fenceFd); 439 if (buffer == NULL) { 440 jniThrowException(env, "java/lang/IllegalStateException", 441 "Image is not initialized"); 442 return; 443 } 444 445 // Unlock image if it was locked. 446 Image_unlockIfLocked(env, image); 447 448 // Set timestamp 449 ALOGV("timestamp to be queued: %" PRId64, timestampNs); 450 res = native_window_set_buffers_timestamp(anw.get(), timestampNs); 451 if (res != OK) { 452 jniThrowRuntimeException(env, "Set timestamp failed"); 453 return; 454 } 455 456 // Set crop 457 android_native_rect_t cropRect; 458 cropRect.left = left; 459 cropRect.top = top; 460 cropRect.right = right; 461 cropRect.bottom = bottom; 462 res = native_window_set_crop(anw.get(), &cropRect); 463 if (res != OK) { 464 jniThrowRuntimeException(env, "Set crop rect failed"); 465 return; 466 } 467 468 // Finally, queue input buffer 469 res = anw->queueBuffer(anw.get(), buffer, fenceFd); 470 if (res != OK) { 471 jniThrowRuntimeException(env, "Queue input buffer failed"); 472 return; 473 } 474 475 // Clear the image native context: end of this image's lifecycle in public API. 476 Image_setNativeContext(env, image, NULL, -1); 477} 478 479static jint ImageWriter_attachAndQueueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, 480 jlong nativeBuffer, jint imageFormat, jlong timestampNs, jint left, jint top, 481 jint right, jint bottom) { 482 ALOGV("%s", __FUNCTION__); 483 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx); 484 if (ctx == NULL || thiz == NULL) { 485 jniThrowException(env, "java/lang/IllegalStateException", 486 "ImageWriterContext is not initialized"); 487 return -1; 488 } 489 490 sp<Surface> surface = ctx->getProducer(); 491 status_t res = OK; 492 if (!isFormatOpaque(imageFormat)) { 493 // TODO: need implement, see b/19962027 494 jniThrowRuntimeException(env, 495 "nativeAttachImage for non-opaque image is not implement yet!!!"); 496 return -1; 497 } 498 499 if (!isFormatOpaque(ctx->getBufferFormat())) { 500 jniThrowException(env, "java/lang/IllegalStateException", 501 "Trying to attach an opaque image into a non-opaque ImageWriter"); 502 return -1; 503 } 504 505 // Image is guaranteed to be from ImageReader at this point, so it is safe to 506 // cast to BufferItem pointer. 507 BufferItem* opaqueBuffer = reinterpret_cast<BufferItem*>(nativeBuffer); 508 if (opaqueBuffer == NULL) { 509 jniThrowException(env, "java/lang/IllegalStateException", 510 "Image is not initialized or already closed"); 511 return -1; 512 } 513 514 // Step 1. Attach Image 515 res = surface->attachBuffer(opaqueBuffer->mGraphicBuffer.get()); 516 if (res != OK) { 517 // TODO: handle different error case separately. 518 ALOGE("Attach image failed: %s (%d)", strerror(-res), res); 519 jniThrowRuntimeException(env, "nativeAttachImage failed!!!"); 520 return res; 521 } 522 sp < ANativeWindow > anw = surface; 523 524 // Step 2. Set timestamp and crop. Note that we do not need unlock the image because 525 // it was not locked. 526 ALOGV("timestamp to be queued: %" PRId64, timestampNs); 527 res = native_window_set_buffers_timestamp(anw.get(), timestampNs); 528 if (res != OK) { 529 jniThrowRuntimeException(env, "Set timestamp failed"); 530 return res; 531 } 532 533 android_native_rect_t cropRect; 534 cropRect.left = left; 535 cropRect.top = top; 536 cropRect.right = right; 537 cropRect.bottom = bottom; 538 res = native_window_set_crop(anw.get(), &cropRect); 539 if (res != OK) { 540 jniThrowRuntimeException(env, "Set crop rect failed"); 541 return res; 542 } 543 544 // Step 3. Queue Image. 545 res = anw->queueBuffer(anw.get(), opaqueBuffer->mGraphicBuffer.get(), /*fenceFd*/ 546 -1); 547 if (res != OK) { 548 jniThrowRuntimeException(env, "Queue input buffer failed"); 549 return res; 550 } 551 552 // Do not set the image native context. Since it would overwrite the existing native context 553 // of the image that is from ImageReader, the subsequent image close will run into issues. 554 555 return res; 556} 557 558// --------------------------Image methods--------------------------------------- 559 560static void Image_getNativeContext(JNIEnv* env, jobject thiz, 561 GraphicBuffer** buffer, int* fenceFd) { 562 ALOGV("%s", __FUNCTION__); 563 if (buffer != NULL) { 564 GraphicBuffer *gb = reinterpret_cast<GraphicBuffer *> 565 (env->GetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer)); 566 *buffer = gb; 567 } 568 569 if (fenceFd != NULL) { 570 *fenceFd = reinterpret_cast<jint>(env->GetIntField( 571 thiz, gSurfaceImageClassInfo.mNativeFenceFd)); 572 } 573} 574 575static void Image_setNativeContext(JNIEnv* env, jobject thiz, 576 sp<GraphicBuffer> buffer, int fenceFd) { 577 ALOGV("%s:", __FUNCTION__); 578 GraphicBuffer* p = NULL; 579 Image_getNativeContext(env, thiz, &p, /*fenceFd*/NULL); 580 if (buffer != 0) { 581 buffer->incStrong((void*)Image_setNativeContext); 582 } 583 if (p) { 584 p->decStrong((void*)Image_setNativeContext); 585 } 586 env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, 587 reinterpret_cast<jlong>(buffer.get())); 588 589 env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast<jint>(fenceFd)); 590} 591 592static void Image_unlockIfLocked(JNIEnv* env, jobject thiz) { 593 ALOGV("%s", __FUNCTION__); 594 GraphicBuffer* buffer; 595 Image_getNativeContext(env, thiz, &buffer, NULL); 596 if (buffer == NULL) { 597 jniThrowException(env, "java/lang/IllegalStateException", 598 "Image is not initialized"); 599 return; 600 } 601 602 // Is locked? 603 bool isLocked = false; 604 jobject planes = NULL; 605 if (!isFormatOpaque(buffer->getPixelFormat())) { 606 planes = env->GetObjectField(thiz, gSurfaceImageClassInfo.mPlanes); 607 } 608 isLocked = (planes != NULL); 609 if (isLocked) { 610 // no need to use fence here, as we it will be consumed by either cancel or queue buffer. 611 status_t res = buffer->unlock(); 612 if (res != OK) { 613 jniThrowRuntimeException(env, "unlock buffer failed"); 614 } 615 ALOGV("Successfully unlocked the image"); 616 } 617} 618 619static jint Image_getWidth(JNIEnv* env, jobject thiz) { 620 ALOGV("%s", __FUNCTION__); 621 GraphicBuffer* buffer; 622 Image_getNativeContext(env, thiz, &buffer, NULL); 623 if (buffer == NULL) { 624 jniThrowException(env, "java/lang/IllegalStateException", 625 "Image is not initialized"); 626 return -1; 627 } 628 629 return buffer->getWidth(); 630} 631 632static jint Image_getHeight(JNIEnv* env, jobject thiz) { 633 ALOGV("%s", __FUNCTION__); 634 GraphicBuffer* buffer; 635 Image_getNativeContext(env, thiz, &buffer, NULL); 636 if (buffer == NULL) { 637 jniThrowException(env, "java/lang/IllegalStateException", 638 "Image is not initialized"); 639 return -1; 640 } 641 642 return buffer->getHeight(); 643} 644 645// Some formats like JPEG defined with different values between android.graphics.ImageFormat and 646// graphics.h, need convert to the one defined in graphics.h here. 647static int Image_getPixelFormat(JNIEnv* env, int format) { 648 int jpegFormat; 649 jfieldID fid; 650 651 ALOGV("%s: format = 0x%x", __FUNCTION__, format); 652 653 jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat"); 654 ALOG_ASSERT(imageFormatClazz != NULL); 655 656 fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I"); 657 jpegFormat = env->GetStaticIntField(imageFormatClazz, fid); 658 659 // Translate the JPEG to BLOB for camera purpose. 660 if (format == jpegFormat) { 661 format = HAL_PIXEL_FORMAT_BLOB; 662 } 663 664 return format; 665} 666 667static jint Image_getFormat(JNIEnv* env, jobject thiz) { 668 ALOGV("%s", __FUNCTION__); 669 GraphicBuffer* buffer; 670 Image_getNativeContext(env, thiz, &buffer, NULL); 671 if (buffer == NULL) { 672 jniThrowException(env, "java/lang/IllegalStateException", 673 "Image is not initialized"); 674 return 0; 675 } 676 677 return Image_getPixelFormat(env, buffer->getPixelFormat()); 678} 679 680static void Image_setFenceFd(JNIEnv* env, jobject thiz, int fenceFd) { 681 ALOGV("%s:", __FUNCTION__); 682 env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast<jint>(fenceFd)); 683} 684 685static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) { 686 ALOGV("%s", __FUNCTION__); 687 GraphicBuffer* buffer; 688 int fenceFd = -1; 689 Image_getNativeContext(env, thiz, &buffer, &fenceFd); 690 if (buffer == NULL) { 691 jniThrowException(env, "java/lang/IllegalStateException", 692 "Image is not initialized"); 693 return; 694 } 695 696 void* pData = NULL; 697 android_ycbcr ycbcr = android_ycbcr(); 698 status_t res; 699 int format = Image_getFormat(env, thiz); 700 int flexFormat = format; 701 if (isPossiblyYUV(format)) { 702 // ImageWriter doesn't use crop by itself, app sets it, use the no crop version. 703 res = buffer->lockAsyncYCbCr(GRALLOC_USAGE_SW_WRITE_OFTEN, &ycbcr, fenceFd); 704 // Clear the fenceFd as it is already consumed by lock call. 705 Image_setFenceFd(env, thiz, /*fenceFd*/-1); 706 if (res != OK) { 707 jniThrowRuntimeException(env, "lockAsyncYCbCr failed for YUV buffer"); 708 return; 709 } 710 pData = ycbcr.y; 711 flexFormat = HAL_PIXEL_FORMAT_YCbCr_420_888; 712 } 713 714 // lockAsyncYCbCr for YUV is unsuccessful. 715 if (pData == NULL) { 716 res = buffer->lockAsync(GRALLOC_USAGE_SW_WRITE_OFTEN, &pData, fenceFd); 717 if (res != OK) { 718 jniThrowRuntimeException(env, "lockAsync failed"); 719 return; 720 } 721 } 722 723 image->data = reinterpret_cast<uint8_t*>(pData); 724 image->width = buffer->getWidth(); 725 image->height = buffer->getHeight(); 726 image->format = format; 727 image->flexFormat = flexFormat; 728 image->stride = (ycbcr.y != NULL) ? static_cast<uint32_t>(ycbcr.ystride) : buffer->getStride(); 729 730 image->dataCb = reinterpret_cast<uint8_t*>(ycbcr.cb); 731 image->dataCr = reinterpret_cast<uint8_t*>(ycbcr.cr); 732 image->chromaStride = static_cast<uint32_t>(ycbcr.cstride); 733 image->chromaStep = static_cast<uint32_t>(ycbcr.chroma_step); 734 ALOGV("Successfully locked the image"); 735 // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer, 736 // and we don't set them here. 737} 738 739static bool usingRGBAToJpegOverride(int32_t bufferFormat, int32_t writerCtxFormat) { 740 return writerCtxFormat == HAL_PIXEL_FORMAT_BLOB && bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888; 741} 742 743static int32_t applyFormatOverrides(int32_t bufferFormat, int32_t writerCtxFormat) 744{ 745 // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW 746 // write limitations for some platforms (b/17379185). 747 if (usingRGBAToJpegOverride(bufferFormat, writerCtxFormat)) { 748 return HAL_PIXEL_FORMAT_BLOB; 749 } 750 return bufferFormat; 751} 752 753static uint32_t Image_getJpegSize(LockedImage* buffer, bool usingRGBAOverride) { 754 ALOGV("%s", __FUNCTION__); 755 ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!"); 756 uint32_t size = 0; 757 uint32_t width = buffer->width; 758 uint8_t* jpegBuffer = buffer->data; 759 760 if (usingRGBAOverride) { 761 width = (buffer->width + buffer->stride * (buffer->height - 1)) * 4; 762 } 763 764 // First check for JPEG transport header at the end of the buffer 765 uint8_t* header = jpegBuffer + (width - sizeof(struct camera3_jpeg_blob)); 766 struct camera3_jpeg_blob *blob = (struct camera3_jpeg_blob*)(header); 767 if (blob->jpeg_blob_id == CAMERA3_JPEG_BLOB_ID) { 768 size = blob->jpeg_size; 769 ALOGV("%s: Jpeg size = %d", __FUNCTION__, size); 770 } 771 772 // failed to find size, default to whole buffer 773 if (size == 0) { 774 /* 775 * This is a problem because not including the JPEG header 776 * means that in certain rare situations a regular JPEG blob 777 * will be misidentified as having a header, in which case 778 * we will get a garbage size value. 779 */ 780 ALOGW("%s: No JPEG header detected, defaulting to size=width=%d", 781 __FUNCTION__, width); 782 size = width; 783 } 784 785 return size; 786} 787 788static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx, 789 int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) { 790 ALOGV("%s", __FUNCTION__); 791 ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!"); 792 ALOG_ASSERT(base != NULL, "base is NULL!!!"); 793 ALOG_ASSERT(size != NULL, "size is NULL!!!"); 794 ALOG_ASSERT(pixelStride != NULL, "pixelStride is NULL!!!"); 795 ALOG_ASSERT(rowStride != NULL, "rowStride is NULL!!!"); 796 ALOG_ASSERT((idx < IMAGE_WRITER_MAX_NUM_PLANES) && (idx >= 0)); 797 798 ALOGV("%s: buffer: %p", __FUNCTION__, buffer); 799 800 uint32_t dataSize, ySize, cSize, cStride; 801 uint32_t pStride = 0, rStride = 0; 802 uint8_t *cb, *cr; 803 uint8_t *pData = NULL; 804 int bytesPerPixel = 0; 805 806 dataSize = ySize = cSize = cStride = 0; 807 int32_t fmt = buffer->flexFormat; 808 809 bool usingRGBAOverride = usingRGBAToJpegOverride(fmt, writerFormat); 810 fmt = applyFormatOverrides(fmt, writerFormat); 811 switch (fmt) { 812 case HAL_PIXEL_FORMAT_YCbCr_420_888: 813 pData = 814 (idx == 0) ? 815 buffer->data : 816 (idx == 1) ? 817 buffer->dataCb : 818 buffer->dataCr; 819 // only map until last pixel 820 if (idx == 0) { 821 pStride = 1; 822 rStride = buffer->stride; 823 dataSize = buffer->stride * (buffer->height - 1) + buffer->width; 824 } else { 825 pStride = buffer->chromaStep; 826 rStride = buffer->chromaStride; 827 dataSize = buffer->chromaStride * (buffer->height / 2 - 1) + 828 buffer->chromaStep * (buffer->width / 2 - 1) + 1; 829 } 830 break; 831 // NV21 832 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 833 cr = buffer->data + (buffer->stride * buffer->height); 834 cb = cr + 1; 835 // only map until last pixel 836 ySize = buffer->width * (buffer->height - 1) + buffer->width; 837 cSize = buffer->width * (buffer->height / 2 - 1) + buffer->width - 1; 838 839 pData = 840 (idx == 0) ? 841 buffer->data : 842 (idx == 1) ? 843 cb: 844 cr; 845 846 dataSize = (idx == 0) ? ySize : cSize; 847 pStride = (idx == 0) ? 1 : 2; 848 rStride = buffer->width; 849 break; 850 case HAL_PIXEL_FORMAT_YV12: 851 // Y and C stride need to be 16 pixel aligned. 852 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 853 "Stride is not 16 pixel aligned %d", buffer->stride); 854 855 ySize = buffer->stride * buffer->height; 856 cStride = ALIGN(buffer->stride / 2, 16); 857 cr = buffer->data + ySize; 858 cSize = cStride * buffer->height / 2; 859 cb = cr + cSize; 860 861 pData = 862 (idx == 0) ? 863 buffer->data : 864 (idx == 1) ? 865 cb : 866 cr; 867 dataSize = (idx == 0) ? ySize : cSize; 868 pStride = 1; 869 rStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16); 870 break; 871 case HAL_PIXEL_FORMAT_Y8: 872 // Single plane, 8bpp. 873 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 874 875 pData = buffer->data; 876 dataSize = buffer->stride * buffer->height; 877 pStride = 1; 878 rStride = buffer->stride; 879 break; 880 case HAL_PIXEL_FORMAT_Y16: 881 bytesPerPixel = 2; 882 // Single plane, 16bpp, strides are specified in pixels, not in bytes 883 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 884 885 pData = buffer->data; 886 dataSize = buffer->stride * buffer->height * bytesPerPixel; 887 pStride = bytesPerPixel; 888 rStride = buffer->stride * 2; 889 break; 890 case HAL_PIXEL_FORMAT_BLOB: 891 // Used for JPEG data, height must be 1, width == size, single plane. 892 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 893 ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height); 894 895 pData = buffer->data; 896 dataSize = Image_getJpegSize(buffer, usingRGBAOverride); 897 pStride = bytesPerPixel; 898 rowStride = 0; 899 break; 900 case HAL_PIXEL_FORMAT_RAW16: 901 // Single plane 16bpp bayer data. 902 bytesPerPixel = 2; 903 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 904 pData = buffer->data; 905 dataSize = buffer->stride * buffer->height * bytesPerPixel; 906 pStride = bytesPerPixel; 907 rStride = buffer->stride * 2; 908 break; 909 case HAL_PIXEL_FORMAT_RAW10: 910 // Single plane 10bpp bayer data. 911 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 912 LOG_ALWAYS_FATAL_IF(buffer->width % 4, 913 "Width is not multiple of 4 %d", buffer->width); 914 LOG_ALWAYS_FATAL_IF(buffer->height % 2, 915 "Height is not even %d", buffer->height); 916 LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 10 / 8), 917 "stride (%d) should be at least %d", 918 buffer->stride, buffer->width * 10 / 8); 919 pData = buffer->data; 920 dataSize = buffer->stride * buffer->height; 921 pStride = 0; 922 rStride = buffer->stride; 923 break; 924 case HAL_PIXEL_FORMAT_RGBA_8888: 925 case HAL_PIXEL_FORMAT_RGBX_8888: 926 // Single plane, 32bpp. 927 bytesPerPixel = 4; 928 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 929 pData = buffer->data; 930 dataSize = buffer->stride * buffer->height * bytesPerPixel; 931 pStride = bytesPerPixel; 932 rStride = buffer->stride * 4; 933 break; 934 case HAL_PIXEL_FORMAT_RGB_565: 935 // Single plane, 16bpp. 936 bytesPerPixel = 2; 937 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 938 pData = buffer->data; 939 dataSize = buffer->stride * buffer->height * bytesPerPixel; 940 pStride = bytesPerPixel; 941 rStride = buffer->stride * 2; 942 break; 943 case HAL_PIXEL_FORMAT_RGB_888: 944 // Single plane, 24bpp. 945 bytesPerPixel = 3; 946 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 947 pData = buffer->data; 948 dataSize = buffer->stride * buffer->height * bytesPerPixel; 949 pStride = bytesPerPixel; 950 rStride = buffer->stride * 3; 951 break; 952 default: 953 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 954 "Pixel format: 0x%x is unsupported", fmt); 955 break; 956 } 957 958 *base = pData; 959 *size = dataSize; 960 *pixelStride = pStride; 961 *rowStride = rStride; 962} 963 964static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz, 965 int numPlanes, int writerFormat) { 966 ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes); 967 int rowStride, pixelStride; 968 uint8_t *pData; 969 uint32_t dataSize; 970 jobject byteBuffer; 971 972 int format = Image_getFormat(env, thiz); 973 if (isFormatOpaque(format) && numPlanes > 0) { 974 String8 msg; 975 msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)" 976 " must be 0", format, numPlanes); 977 jniThrowException(env, "java/lang/IllegalArgumentException", msg.string()); 978 return NULL; 979 } 980 981 jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz, 982 /*initial_element*/NULL); 983 if (surfacePlanes == NULL) { 984 jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays," 985 " probably out of memory"); 986 return NULL; 987 } 988 if (isFormatOpaque(format)) { 989 return surfacePlanes; 990 } 991 992 // Buildup buffer info: rowStride, pixelStride and byteBuffers. 993 LockedImage lockedImg = LockedImage(); 994 Image_getLockedImage(env, thiz, &lockedImg); 995 996 // Create all SurfacePlanes 997 writerFormat = Image_getPixelFormat(env, writerFormat); 998 for (int i = 0; i < numPlanes; i++) { 999 Image_getLockedImageInfo(env, &lockedImg, i, writerFormat, 1000 &pData, &dataSize, &pixelStride, &rowStride); 1001 byteBuffer = env->NewDirectByteBuffer(pData, dataSize); 1002 if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) { 1003 jniThrowException(env, "java/lang/IllegalStateException", 1004 "Failed to allocate ByteBuffer"); 1005 return NULL; 1006 } 1007 1008 // Finally, create this SurfacePlane. 1009 jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz, 1010 gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer); 1011 env->SetObjectArrayElement(surfacePlanes, i, surfacePlane); 1012 } 1013 1014 return surfacePlanes; 1015} 1016 1017// -------------------------------Private convenience methods-------------------- 1018 1019static bool isFormatOpaque(int format) { 1020 // Only treat IMPLEMENTATION_DEFINED as an opaque format for now. 1021 return format == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED; 1022} 1023 1024static bool isPossiblyYUV(PixelFormat format) { 1025 switch (static_cast<int>(format)) { 1026 case HAL_PIXEL_FORMAT_RGBA_8888: 1027 case HAL_PIXEL_FORMAT_RGBX_8888: 1028 case HAL_PIXEL_FORMAT_RGB_888: 1029 case HAL_PIXEL_FORMAT_RGB_565: 1030 case HAL_PIXEL_FORMAT_BGRA_8888: 1031 case HAL_PIXEL_FORMAT_Y8: 1032 case HAL_PIXEL_FORMAT_Y16: 1033 case HAL_PIXEL_FORMAT_RAW16: 1034 case HAL_PIXEL_FORMAT_RAW10: 1035 case HAL_PIXEL_FORMAT_RAW_OPAQUE: 1036 case HAL_PIXEL_FORMAT_BLOB: 1037 case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED: 1038 return false; 1039 1040 case HAL_PIXEL_FORMAT_YV12: 1041 case HAL_PIXEL_FORMAT_YCbCr_420_888: 1042 case HAL_PIXEL_FORMAT_YCbCr_422_SP: 1043 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 1044 case HAL_PIXEL_FORMAT_YCbCr_422_I: 1045 default: 1046 return true; 1047 } 1048} 1049 1050} // extern "C" 1051 1052// ---------------------------------------------------------------------------- 1053 1054static JNINativeMethod gImageWriterMethods[] = { 1055 {"nativeClassInit", "()V", (void*)ImageWriter_classInit }, 1056 {"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;I)J", 1057 (void*)ImageWriter_init }, 1058 {"nativeClose", "(J)V", (void*)ImageWriter_close }, 1059 {"nativeAttachAndQueueImage", "(JJIJIIII)I", (void*)ImageWriter_attachAndQueueImage }, 1060 {"nativeDequeueInputImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_dequeueImage }, 1061 {"nativeQueueInputImage", "(JLandroid/media/Image;JIIII)V", (void*)ImageWriter_queueImage }, 1062 {"cancelImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_cancelImage }, 1063}; 1064 1065static JNINativeMethod gImageMethods[] = { 1066 {"nativeCreatePlanes", "(II)[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;", 1067 (void*)Image_createSurfacePlanes }, 1068 {"nativeGetWidth", "()I", (void*)Image_getWidth }, 1069 {"nativeGetHeight", "()I", (void*)Image_getHeight }, 1070 {"nativeGetFormat", "()I", (void*)Image_getFormat }, 1071}; 1072 1073int register_android_media_ImageWriter(JNIEnv *env) { 1074 1075 int ret1 = AndroidRuntime::registerNativeMethods(env, 1076 "android/media/ImageWriter", gImageWriterMethods, NELEM(gImageWriterMethods)); 1077 1078 int ret2 = AndroidRuntime::registerNativeMethods(env, 1079 "android/media/ImageWriter$WriterSurfaceImage", gImageMethods, NELEM(gImageMethods)); 1080 1081 return (ret1 || ret2); 1082} 1083 1084