android_media_ImageReader.cpp revision 916d8ac650865cd05808d48bad68b69bebbc95ab
1/* 2 * Copyright 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "ImageReader_JNI" 19#include "android_media_Utils.h" 20#include <cutils/atomic.h> 21#include <utils/Log.h> 22#include <utils/misc.h> 23#include <utils/List.h> 24#include <utils/String8.h> 25 26#include <cstdio> 27 28#include <gui/BufferItemConsumer.h> 29#include <gui/Surface.h> 30#include <camera3.h> 31 32#include <android_runtime/AndroidRuntime.h> 33#include <android_runtime/android_view_Surface.h> 34#include <android_runtime/android_hardware_HardwareBuffer.h> 35#include <grallocusage/GrallocUsageConversion.h> 36 37#include <jni.h> 38#include <JNIHelp.h> 39 40#include <stdint.h> 41#include <inttypes.h> 42 43#define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID "mNativeContext" 44#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mNativeBuffer" 45#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp" 46 47#define CONSUMER_BUFFER_USAGE_UNKNOWN 0; 48// ---------------------------------------------------------------------------- 49 50using namespace android; 51 52 53enum { 54 ACQUIRE_SUCCESS = 0, 55 ACQUIRE_NO_BUFFERS = 1, 56 ACQUIRE_MAX_IMAGES = 2, 57}; 58 59static struct { 60 jfieldID mNativeContext; 61 jmethodID postEventFromNative; 62} gImageReaderClassInfo; 63 64static struct { 65 jfieldID mNativeBuffer; 66 jfieldID mTimestamp; 67 jfieldID mPlanes; 68} gSurfaceImageClassInfo; 69 70static struct { 71 jclass clazz; 72 jmethodID ctor; 73} gSurfacePlaneClassInfo; 74 75// Get an ID that's unique within this process. 76static int32_t createProcessUniqueId() { 77 static volatile int32_t globalCounter = 0; 78 return android_atomic_inc(&globalCounter); 79} 80 81// ---------------------------------------------------------------------------- 82 83class JNIImageReaderContext : public ConsumerBase::FrameAvailableListener 84{ 85public: 86 JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages); 87 88 virtual ~JNIImageReaderContext(); 89 90 virtual void onFrameAvailable(const BufferItem& item); 91 92 BufferItem* getBufferItem(); 93 void returnBufferItem(BufferItem* buffer); 94 95 96 void setBufferConsumer(const sp<BufferItemConsumer>& consumer) { mConsumer = consumer; } 97 BufferItemConsumer* getBufferConsumer() { return mConsumer.get(); } 98 99 void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; } 100 IGraphicBufferProducer* getProducer() { return mProducer.get(); } 101 102 void setBufferFormat(int format) { mFormat = format; } 103 int getBufferFormat() { return mFormat; } 104 105 void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; } 106 android_dataspace getBufferDataspace() { return mDataSpace; } 107 108 void setBufferWidth(int width) { mWidth = width; } 109 int getBufferWidth() { return mWidth; } 110 111 void setBufferHeight(int height) { mHeight = height; } 112 int getBufferHeight() { return mHeight; } 113 114private: 115 static JNIEnv* getJNIEnv(bool* needsDetach); 116 static void detachJNI(); 117 118 List<BufferItem*> mBuffers; 119 sp<BufferItemConsumer> mConsumer; 120 sp<IGraphicBufferProducer> mProducer; 121 jobject mWeakThiz; 122 jclass mClazz; 123 int mFormat; 124 android_dataspace mDataSpace; 125 int mWidth; 126 int mHeight; 127}; 128 129JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env, 130 jobject weakThiz, jclass clazz, int maxImages) : 131 mWeakThiz(env->NewGlobalRef(weakThiz)), 132 mClazz((jclass)env->NewGlobalRef(clazz)), 133 mFormat(0), 134 mDataSpace(HAL_DATASPACE_UNKNOWN), 135 mWidth(-1), 136 mHeight(-1) { 137 for (int i = 0; i < maxImages; i++) { 138 BufferItem* buffer = new BufferItem; 139 mBuffers.push_back(buffer); 140 } 141} 142 143JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) { 144 LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!"); 145 *needsDetach = false; 146 JNIEnv* env = AndroidRuntime::getJNIEnv(); 147 if (env == NULL) { 148 JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL}; 149 JavaVM* vm = AndroidRuntime::getJavaVM(); 150 int result = vm->AttachCurrentThread(&env, (void*) &args); 151 if (result != JNI_OK) { 152 ALOGE("thread attach failed: %#x", result); 153 return NULL; 154 } 155 *needsDetach = true; 156 } 157 return env; 158} 159 160void JNIImageReaderContext::detachJNI() { 161 JavaVM* vm = AndroidRuntime::getJavaVM(); 162 int result = vm->DetachCurrentThread(); 163 if (result != JNI_OK) { 164 ALOGE("thread detach failed: %#x", result); 165 } 166} 167 168BufferItem* JNIImageReaderContext::getBufferItem() { 169 if (mBuffers.empty()) { 170 return NULL; 171 } 172 // Return a BufferItem pointer and remove it from the list 173 List<BufferItem*>::iterator it = mBuffers.begin(); 174 BufferItem* buffer = *it; 175 mBuffers.erase(it); 176 return buffer; 177} 178 179void JNIImageReaderContext::returnBufferItem(BufferItem* buffer) { 180 mBuffers.push_back(buffer); 181} 182 183JNIImageReaderContext::~JNIImageReaderContext() { 184 bool needsDetach = false; 185 JNIEnv* env = getJNIEnv(&needsDetach); 186 if (env != NULL) { 187 env->DeleteGlobalRef(mWeakThiz); 188 env->DeleteGlobalRef(mClazz); 189 } else { 190 ALOGW("leaking JNI object references"); 191 } 192 if (needsDetach) { 193 detachJNI(); 194 } 195 196 // Delete buffer items. 197 for (List<BufferItem *>::iterator it = mBuffers.begin(); 198 it != mBuffers.end(); it++) { 199 delete *it; 200 } 201 202 if (mConsumer != 0) { 203 mConsumer.clear(); 204 } 205} 206 207void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/) 208{ 209 ALOGV("%s: frame available", __FUNCTION__); 210 bool needsDetach = false; 211 JNIEnv* env = getJNIEnv(&needsDetach); 212 if (env != NULL) { 213 env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz); 214 } else { 215 ALOGW("onFrameAvailable event will not posted"); 216 } 217 if (needsDetach) { 218 detachJNI(); 219 } 220} 221 222// ---------------------------------------------------------------------------- 223 224extern "C" { 225 226static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz) 227{ 228 JNIImageReaderContext *ctx; 229 ctx = reinterpret_cast<JNIImageReaderContext *> 230 (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext)); 231 return ctx; 232} 233 234static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz) 235{ 236 ALOGV("%s:", __FUNCTION__); 237 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 238 if (ctx == NULL) { 239 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 240 return NULL; 241 } 242 243 return ctx->getProducer(); 244} 245 246static void ImageReader_setNativeContext(JNIEnv* env, 247 jobject thiz, sp<JNIImageReaderContext> ctx) 248{ 249 ALOGV("%s:", __FUNCTION__); 250 JNIImageReaderContext* const p = ImageReader_getContext(env, thiz); 251 if (ctx != 0) { 252 ctx->incStrong((void*)ImageReader_setNativeContext); 253 } 254 if (p) { 255 p->decStrong((void*)ImageReader_setNativeContext); 256 } 257 env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext, 258 reinterpret_cast<jlong>(ctx.get())); 259} 260 261static BufferItemConsumer* ImageReader_getBufferConsumer(JNIEnv* env, jobject thiz) 262{ 263 ALOGV("%s:", __FUNCTION__); 264 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 265 if (ctx == NULL) { 266 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 267 return NULL; 268 } 269 270 return ctx->getBufferConsumer(); 271} 272 273static void Image_setBufferItem(JNIEnv* env, jobject thiz, 274 const BufferItem* buffer) 275{ 276 env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, reinterpret_cast<jlong>(buffer)); 277} 278 279static BufferItem* Image_getBufferItem(JNIEnv* env, jobject image) 280{ 281 return reinterpret_cast<BufferItem*>( 282 env->GetLongField(image, gSurfaceImageClassInfo.mNativeBuffer)); 283} 284 285 286// ---------------------------------------------------------------------------- 287 288static void ImageReader_classInit(JNIEnv* env, jclass clazz) 289{ 290 ALOGV("%s:", __FUNCTION__); 291 292 jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage"); 293 LOG_ALWAYS_FATAL_IF(imageClazz == NULL, 294 "can't find android/graphics/ImageReader$SurfaceImage"); 295 gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID( 296 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J"); 297 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL, 298 "can't find android/graphics/ImageReader.%s", 299 ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID); 300 301 gSurfaceImageClassInfo.mTimestamp = env->GetFieldID( 302 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J"); 303 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL, 304 "can't find android/graphics/ImageReader.%s", 305 ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID); 306 307 gSurfaceImageClassInfo.mPlanes = env->GetFieldID( 308 imageClazz, "mPlanes", "[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;"); 309 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL, 310 "can't find android/media/ImageReader$ReaderSurfaceImage.mPlanes"); 311 312 gImageReaderClassInfo.mNativeContext = env->GetFieldID( 313 clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J"); 314 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL, 315 "can't find android/graphics/ImageReader.%s", 316 ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID); 317 318 gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID( 319 clazz, "postEventFromNative", "(Ljava/lang/Object;)V"); 320 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL, 321 "can't find android/graphics/ImageReader.postEventFromNative"); 322 323 jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane"); 324 LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class"); 325 // FindClass only gives a local reference of jclass object. 326 gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz); 327 gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>", 328 "(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V"); 329 LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL, 330 "Can not find SurfacePlane constructor"); 331} 332 333static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height, 334 jint format, jint maxImages, jlong ndkUsage) 335{ 336 status_t res; 337 int nativeFormat; 338 android_dataspace nativeDataspace; 339 340 ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d", 341 __FUNCTION__, width, height, format, maxImages); 342 343 PublicFormat publicFormat = static_cast<PublicFormat>(format); 344 nativeFormat = android_view_Surface_mapPublicFormatToHalFormat( 345 publicFormat); 346 nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace( 347 publicFormat); 348 349 jclass clazz = env->GetObjectClass(thiz); 350 if (clazz == NULL) { 351 jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader"); 352 return; 353 } 354 sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages)); 355 356 sp<IGraphicBufferProducer> gbProducer; 357 sp<IGraphicBufferConsumer> gbConsumer; 358 BufferQueue::createBufferQueue(&gbProducer, &gbConsumer); 359 sp<BufferItemConsumer> bufferConsumer; 360 String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d", 361 width, height, format, maxImages, getpid(), 362 createProcessUniqueId()); 363 uint32_t consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN; 364 bool needUsageOverride = ndkUsage != CONSUMER_BUFFER_USAGE_UNKNOWN; 365 uint64_t outProducerUsage = 0; 366 uint64_t outConsumerUsage = 0; 367 android_hardware_HardwareBuffer_convertToGrallocUsageBits(&outProducerUsage, &outConsumerUsage, 368 ndkUsage, 0); 369 370 if (isFormatOpaque(nativeFormat)) { 371 // Use the SW_READ_NEVER usage to tell producer that this format is not for preview or video 372 // encoding. The only possibility will be ZSL output. 373 consumerUsage = GRALLOC_USAGE_SW_READ_NEVER; 374 if (needUsageOverride) { 375 consumerUsage = android_convertGralloc1To0Usage(0, outConsumerUsage); 376 } 377 } else if (needUsageOverride) { 378 ALOGW("Consumer usage override for non-opaque format is not implemented yet, " 379 "ignore the provided usage from the application"); 380 } 381 bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages, 382 /*controlledByApp*/true); 383 if (bufferConsumer == nullptr) { 384 jniThrowExceptionFmt(env, "java/lang/RuntimeException", 385 "Failed to allocate native buffer consumer for format 0x%x and usage 0x%x", 386 nativeFormat, consumerUsage); 387 return; 388 } 389 ctx->setBufferConsumer(bufferConsumer); 390 bufferConsumer->setName(consumerName); 391 392 ctx->setProducer(gbProducer); 393 bufferConsumer->setFrameAvailableListener(ctx); 394 ImageReader_setNativeContext(env, thiz, ctx); 395 ctx->setBufferFormat(nativeFormat); 396 ctx->setBufferDataspace(nativeDataspace); 397 ctx->setBufferWidth(width); 398 ctx->setBufferHeight(height); 399 400 // Set the width/height/format/dataspace to the bufferConsumer. 401 res = bufferConsumer->setDefaultBufferSize(width, height); 402 if (res != OK) { 403 jniThrowExceptionFmt(env, "java/lang/IllegalStateException", 404 "Failed to set buffer consumer default size (%dx%d) for format 0x%x", 405 width, height, nativeFormat); 406 return; 407 } 408 res = bufferConsumer->setDefaultBufferFormat(nativeFormat); 409 if (res != OK) { 410 jniThrowExceptionFmt(env, "java/lang/IllegalStateException", 411 "Failed to set buffer consumer default format 0x%x", nativeFormat); 412 } 413 res = bufferConsumer->setDefaultBufferDataSpace(nativeDataspace); 414 if (res != OK) { 415 jniThrowExceptionFmt(env, "java/lang/IllegalStateException", 416 "Failed to set buffer consumer default dataSpace 0x%x", nativeDataspace); 417 } 418} 419 420static void ImageReader_close(JNIEnv* env, jobject thiz) 421{ 422 ALOGV("%s:", __FUNCTION__); 423 424 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 425 if (ctx == NULL) { 426 // ImageReader is already closed. 427 return; 428 } 429 430 BufferItemConsumer* consumer = NULL; 431 consumer = ImageReader_getBufferConsumer(env, thiz); 432 433 if (consumer != NULL) { 434 consumer->abandon(); 435 consumer->setFrameAvailableListener(NULL); 436 } 437 ImageReader_setNativeContext(env, thiz, NULL); 438} 439 440static sp<Fence> Image_unlockIfLocked(JNIEnv* env, jobject image) { 441 ALOGV("%s", __FUNCTION__); 442 BufferItem* buffer = Image_getBufferItem(env, image); 443 if (buffer == NULL) { 444 jniThrowException(env, "java/lang/IllegalStateException", 445 "Image is not initialized"); 446 return Fence::NO_FENCE; 447 } 448 449 // Is locked? 450 bool wasBufferLocked = false; 451 jobject planes = NULL; 452 if (!isFormatOpaque(buffer->mGraphicBuffer->getPixelFormat())) { 453 planes = env->GetObjectField(image, gSurfaceImageClassInfo.mPlanes); 454 } 455 wasBufferLocked = (planes != NULL); 456 if (wasBufferLocked) { 457 status_t res = OK; 458 int fenceFd = -1; 459 if (wasBufferLocked) { 460 res = buffer->mGraphicBuffer->unlockAsync(&fenceFd); 461 if (res != OK) { 462 jniThrowRuntimeException(env, "unlock buffer failed"); 463 return Fence::NO_FENCE; 464 } 465 } 466 sp<Fence> releaseFence = new Fence(fenceFd); 467 return releaseFence; 468 ALOGV("Successfully unlocked the image"); 469 } 470 return Fence::NO_FENCE; 471} 472 473static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image) 474{ 475 ALOGV("%s:", __FUNCTION__); 476 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 477 if (ctx == NULL) { 478 ALOGW("ImageReader#close called before Image#close, consider calling Image#close first"); 479 return; 480 } 481 482 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer(); 483 BufferItem* buffer = Image_getBufferItem(env, image); 484 if (buffer == nullptr) { 485 // Release an already closed image is harmless. 486 return; 487 } 488 489 sp<Fence> releaseFence = Image_unlockIfLocked(env, image); 490 bufferConsumer->releaseBuffer(*buffer, releaseFence); 491 Image_setBufferItem(env, image, NULL); 492 ctx->returnBufferItem(buffer); 493 ALOGV("%s: Image (format: 0x%x) has been released", __FUNCTION__, ctx->getBufferFormat()); 494} 495 496static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image) { 497 ALOGV("%s:", __FUNCTION__); 498 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 499 if (ctx == NULL) { 500 jniThrowException(env, "java/lang/IllegalStateException", 501 "ImageReader is not initialized or was already closed"); 502 return -1; 503 } 504 505 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer(); 506 BufferItem* buffer = ctx->getBufferItem(); 507 if (buffer == NULL) { 508 ALOGW("Unable to acquire a buffer item, very likely client tried to acquire more than" 509 " maxImages buffers"); 510 return ACQUIRE_MAX_IMAGES; 511 } 512 513 status_t res = bufferConsumer->acquireBuffer(buffer, 0); 514 if (res != OK) { 515 ctx->returnBufferItem(buffer); 516 if (res != BufferQueue::NO_BUFFER_AVAILABLE) { 517 if (res == INVALID_OPERATION) { 518 // Max number of images were already acquired. 519 ALOGE("%s: Max number of buffers allowed are already acquired : %s (%d)", 520 __FUNCTION__, strerror(-res), res); 521 return ACQUIRE_MAX_IMAGES; 522 } else { 523 ALOGE("%s: Acquire image failed with some unknown error: %s (%d)", 524 __FUNCTION__, strerror(-res), res); 525 jniThrowExceptionFmt(env, "java/lang/IllegalStateException", 526 "Unknown error (%d) when we tried to acquire an image.", 527 res); 528 return ACQUIRE_NO_BUFFERS; 529 } 530 } 531 // This isn't really an error case, as the application may acquire buffer at any time. 532 return ACQUIRE_NO_BUFFERS; 533 } 534 535 // Add some extra checks for non-opaque formats. 536 if (!isFormatOpaque(ctx->getBufferFormat())) { 537 // Check if the left-top corner of the crop rect is origin, we currently assume this point is 538 // zero, will revisit this once this assumption turns out problematic. 539 Point lt = buffer->mCrop.leftTop(); 540 if (lt.x != 0 || lt.y != 0) { 541 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 542 "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y); 543 return -1; 544 } 545 546 // Check if the producer buffer configurations match what ImageReader configured. 547 int outputWidth = getBufferWidth(buffer); 548 int outputHeight = getBufferHeight(buffer); 549 550 int imgReaderFmt = ctx->getBufferFormat(); 551 int imageReaderWidth = ctx->getBufferWidth(); 552 int imageReaderHeight = ctx->getBufferHeight(); 553 int bufferFormat = buffer->mGraphicBuffer->getPixelFormat(); 554 if ((bufferFormat != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) && 555 (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) { 556 ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d", 557 __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight); 558 } 559 if (imgReaderFmt != bufferFormat) { 560 if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && 561 isPossiblyYUV(bufferFormat)) { 562 // Treat formats that are compatible with flexible YUV 563 // (HAL_PIXEL_FORMAT_YCbCr_420_888) as HAL_PIXEL_FORMAT_YCbCr_420_888. 564 ALOGV("%s: Treat buffer format to 0x%x as HAL_PIXEL_FORMAT_YCbCr_420_888", 565 __FUNCTION__, bufferFormat); 566 } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB && 567 bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888) { 568 // Using HAL_PIXEL_FORMAT_RGBA_8888 Gralloc buffers containing JPEGs to get around 569 // SW write limitations for (b/17379185). 570 ALOGV("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__); 571 } else { 572 // Return the buffer to the queue. No need to provide fence, as this buffer wasn't 573 // used anywhere yet. 574 bufferConsumer->releaseBuffer(*buffer); 575 ctx->returnBufferItem(buffer); 576 577 // Throw exception 578 ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x", 579 bufferFormat, ctx->getBufferFormat()); 580 String8 msg; 581 msg.appendFormat("The producer output buffer format 0x%x doesn't " 582 "match the ImageReader's configured buffer format 0x%x.", 583 bufferFormat, ctx->getBufferFormat()); 584 jniThrowException(env, "java/lang/UnsupportedOperationException", 585 msg.string()); 586 return -1; 587 } 588 } 589 590 } 591 592 // Set SurfaceImage instance member variables 593 Image_setBufferItem(env, image, buffer); 594 env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp, 595 static_cast<jlong>(buffer->mTimestamp)); 596 597 return ACQUIRE_SUCCESS; 598} 599 600static jint ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) { 601 ALOGV("%s:", __FUNCTION__); 602 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 603 if (ctx == NULL) { 604 jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed"); 605 return -1; 606 } 607 608 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer(); 609 BufferItem* buffer = Image_getBufferItem(env, image); 610 if (!buffer) { 611 ALOGE( 612 "Image already released and can not be detached from ImageReader!!!"); 613 jniThrowException(env, "java/lang/IllegalStateException", 614 "Image detach from ImageReader failed: buffer was already released"); 615 return -1; 616 } 617 618 status_t res = OK; 619 Image_unlockIfLocked(env, image); 620 res = bufferConsumer->detachBuffer(buffer->mSlot); 621 if (res != OK) { 622 ALOGE("Image detach failed: %s (%d)!!!", strerror(-res), res); 623 jniThrowRuntimeException(env, 624 "nativeDetachImage failed for image!!!"); 625 return res; 626 } 627 return OK; 628} 629 630static void ImageReader_discardFreeBuffers(JNIEnv* env, jobject thiz) { 631 ALOGV("%s:", __FUNCTION__); 632 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 633 if (ctx == NULL) { 634 jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed"); 635 return; 636 } 637 638 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer(); 639 status_t res = bufferConsumer->discardFreeBuffers(); 640 if (res != OK) { 641 ALOGE("Buffer discard failed: %s (%d)", strerror(-res), res); 642 jniThrowRuntimeException(env, 643 "nativeDicardFreebuffers failed"); 644 } 645} 646 647static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz) 648{ 649 ALOGV("%s: ", __FUNCTION__); 650 651 IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz); 652 if (gbp == NULL) { 653 jniThrowRuntimeException(env, "Buffer consumer is uninitialized"); 654 return NULL; 655 } 656 657 // Wrap the IGBP in a Java-language Surface. 658 return android_view_Surface_createFromIGraphicBufferProducer(env, gbp); 659} 660 661static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) { 662 ALOGV("%s", __FUNCTION__); 663 BufferItem* buffer = Image_getBufferItem(env, thiz); 664 if (buffer == NULL) { 665 jniThrowException(env, "java/lang/IllegalStateException", 666 "Image is not initialized"); 667 return; 668 } 669 670 status_t res = lockImageFromBuffer(buffer, 671 GRALLOC_USAGE_SW_READ_OFTEN, buffer->mFence->dup(), image); 672 if (res != OK) { 673 jniThrowExceptionFmt(env, "java/lang/RuntimeException", 674 "lock buffer failed for format 0x%x", 675 buffer->mGraphicBuffer->getPixelFormat()); 676 return; 677 } 678 679 // Carry over some fields from BufferItem. 680 image->crop = buffer->mCrop; 681 image->transform = buffer->mTransform; 682 image->scalingMode = buffer->mScalingMode; 683 image->timestamp = buffer->mTimestamp; 684 image->dataSpace = buffer->mDataSpace; 685 image->frameNumber = buffer->mFrameNumber; 686 687 ALOGV("%s: Successfully locked the image", __FUNCTION__); 688 // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer, 689 // and we don't set them here. 690} 691 692static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx, 693 int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) { 694 ALOGV("%s", __FUNCTION__); 695 696 status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size, 697 pixelStride, rowStride); 698 if (res != OK) { 699 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 700 "Pixel format: 0x%x is unsupported", buffer->flexFormat); 701 } 702} 703 704static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz, 705 int numPlanes, int readerFormat) 706{ 707 ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes); 708 int rowStride = 0; 709 int pixelStride = 0; 710 uint8_t *pData = NULL; 711 uint32_t dataSize = 0; 712 jobject byteBuffer = NULL; 713 714 PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat); 715 int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat( 716 publicReaderFormat); 717 718 if (isFormatOpaque(halReaderFormat) && numPlanes > 0) { 719 String8 msg; 720 msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)" 721 " must be 0", halReaderFormat, numPlanes); 722 jniThrowException(env, "java/lang/IllegalArgumentException", msg.string()); 723 return NULL; 724 } 725 726 jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz, 727 /*initial_element*/NULL); 728 if (surfacePlanes == NULL) { 729 jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays," 730 " probably out of memory"); 731 return NULL; 732 } 733 if (isFormatOpaque(halReaderFormat)) { 734 // Return 0 element surface array. 735 return surfacePlanes; 736 } 737 738 LockedImage lockedImg = LockedImage(); 739 Image_getLockedImage(env, thiz, &lockedImg); 740 if (env->ExceptionCheck()) { 741 return NULL; 742 } 743 // Create all SurfacePlanes 744 for (int i = 0; i < numPlanes; i++) { 745 Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat, 746 &pData, &dataSize, &pixelStride, &rowStride); 747 byteBuffer = env->NewDirectByteBuffer(pData, dataSize); 748 if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) { 749 jniThrowException(env, "java/lang/IllegalStateException", 750 "Failed to allocate ByteBuffer"); 751 return NULL; 752 } 753 754 // Finally, create this SurfacePlane. 755 jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz, 756 gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer); 757 env->SetObjectArrayElement(surfacePlanes, i, surfacePlane); 758 } 759 760 return surfacePlanes; 761} 762 763static jint Image_getWidth(JNIEnv* env, jobject thiz) 764{ 765 BufferItem* buffer = Image_getBufferItem(env, thiz); 766 return getBufferWidth(buffer); 767} 768 769static jint Image_getHeight(JNIEnv* env, jobject thiz) 770{ 771 BufferItem* buffer = Image_getBufferItem(env, thiz); 772 return getBufferHeight(buffer); 773} 774 775static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat) 776{ 777 if (isFormatOpaque(readerFormat)) { 778 // Assuming opaque reader produce opaque images. 779 return static_cast<jint>(PublicFormat::PRIVATE); 780 } else { 781 BufferItem* buffer = Image_getBufferItem(env, thiz); 782 int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat( 783 static_cast<PublicFormat>(readerFormat)); 784 int32_t fmt = applyFormatOverrides( 785 buffer->mGraphicBuffer->getPixelFormat(), readerHalFormat); 786 // Override the image format to HAL_PIXEL_FORMAT_YCbCr_420_888 if the actual format is 787 // NV21 or YV12. This could only happen when the Gralloc HAL version is v0.1 thus doesn't 788 // support lockycbcr(), the CpuConsumer need to use the lock() method in the 789 // lockNextBuffer() call. For Gralloc HAL v0.2 or newer, this format should already be 790 // overridden to HAL_PIXEL_FORMAT_YCbCr_420_888 for the flexible YUV compatible formats. 791 if (isPossiblyYUV(fmt)) { 792 fmt = HAL_PIXEL_FORMAT_YCbCr_420_888; 793 } 794 PublicFormat publicFmt = android_view_Surface_mapHalFormatDataspaceToPublicFormat( 795 fmt, buffer->mDataSpace); 796 return static_cast<jint>(publicFmt); 797 } 798} 799 800} // extern "C" 801 802// ---------------------------------------------------------------------------- 803 804static const JNINativeMethod gImageReaderMethods[] = { 805 {"nativeClassInit", "()V", (void*)ImageReader_classInit }, 806 {"nativeInit", "(Ljava/lang/Object;IIIIJ)V", (void*)ImageReader_init }, 807 {"nativeClose", "()V", (void*)ImageReader_close }, 808 {"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease }, 809 {"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup }, 810 {"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface }, 811 {"nativeDetachImage", "(Landroid/media/Image;)I", (void*)ImageReader_detachImage }, 812 {"nativeDiscardFreeBuffers", "()V", (void*)ImageReader_discardFreeBuffers } 813}; 814 815static const JNINativeMethod gImageMethods[] = { 816 {"nativeCreatePlanes", "(II)[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;", 817 (void*)Image_createSurfacePlanes }, 818 {"nativeGetWidth", "()I", (void*)Image_getWidth }, 819 {"nativeGetHeight", "()I", (void*)Image_getHeight }, 820 {"nativeGetFormat", "(I)I", (void*)Image_getFormat }, 821}; 822 823int register_android_media_ImageReader(JNIEnv *env) { 824 825 int ret1 = AndroidRuntime::registerNativeMethods(env, 826 "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods)); 827 828 int ret2 = AndroidRuntime::registerNativeMethods(env, 829 "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods)); 830 831 return (ret1 || ret2); 832} 833