android_media_ImageReader.cpp revision 0fd198ad89ec9c600bb1761b10d938146c28bb98
1/* 2 * Copyright 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "ImageReader_JNI" 19#include <utils/Log.h> 20#include <utils/misc.h> 21#include <utils/List.h> 22#include <utils/String8.h> 23 24#include <cstdio> 25 26#include <gui/CpuConsumer.h> 27#include <gui/Surface.h> 28#include <camera3.h> 29 30#include <android_runtime/AndroidRuntime.h> 31#include <android_runtime/android_view_Surface.h> 32 33#include <jni.h> 34#include <JNIHelp.h> 35 36#include <stdint.h> 37#include <inttypes.h> 38 39#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) ) 40 41#define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID "mNativeContext" 42#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mLockedBuffer" 43#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp" 44 45// ---------------------------------------------------------------------------- 46 47using namespace android; 48 49enum { 50 IMAGE_READER_MAX_NUM_PLANES = 3, 51}; 52 53enum { 54 ACQUIRE_SUCCESS = 0, 55 ACQUIRE_NO_BUFFERS = 1, 56 ACQUIRE_MAX_IMAGES = 2, 57}; 58 59static struct { 60 jfieldID mNativeContext; 61 jmethodID postEventFromNative; 62} gImageReaderClassInfo; 63 64static struct { 65 jfieldID mLockedBuffer; 66 jfieldID mTimestamp; 67} gSurfaceImageClassInfo; 68 69static struct { 70 jclass clazz; 71 jmethodID ctor; 72} gSurfacePlaneClassInfo; 73 74// ---------------------------------------------------------------------------- 75 76class JNIImageReaderContext : public CpuConsumer::FrameAvailableListener 77{ 78public: 79 JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages); 80 81 virtual ~JNIImageReaderContext(); 82 83 virtual void onFrameAvailable(); 84 85 CpuConsumer::LockedBuffer* getLockedBuffer(); 86 87 void returnLockedBuffer(CpuConsumer::LockedBuffer* buffer); 88 89 void setCpuConsumer(const sp<CpuConsumer>& consumer) { mConsumer = consumer; } 90 CpuConsumer* getCpuConsumer() { return mConsumer.get(); } 91 92 void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; } 93 IGraphicBufferProducer* getProducer() { return mProducer.get(); } 94 95 void setBufferFormat(int format) { mFormat = format; } 96 int getBufferFormat() { return mFormat; } 97 98 void setBufferWidth(int width) { mWidth = width; } 99 int getBufferWidth() { return mWidth; } 100 101 void setBufferHeight(int height) { mHeight = height; } 102 int getBufferHeight() { return mHeight; } 103 104private: 105 static JNIEnv* getJNIEnv(bool* needsDetach); 106 static void detachJNI(); 107 108 List<CpuConsumer::LockedBuffer*> mBuffers; 109 sp<CpuConsumer> mConsumer; 110 sp<IGraphicBufferProducer> mProducer; 111 jobject mWeakThiz; 112 jclass mClazz; 113 int mFormat; 114 int mWidth; 115 int mHeight; 116}; 117 118JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env, 119 jobject weakThiz, jclass clazz, int maxImages) : 120 mWeakThiz(env->NewGlobalRef(weakThiz)), 121 mClazz((jclass)env->NewGlobalRef(clazz)) { 122 for (int i = 0; i < maxImages; i++) { 123 CpuConsumer::LockedBuffer *buffer = new CpuConsumer::LockedBuffer; 124 mBuffers.push_back(buffer); 125 } 126} 127 128JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) { 129 LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!"); 130 *needsDetach = false; 131 JNIEnv* env = AndroidRuntime::getJNIEnv(); 132 if (env == NULL) { 133 JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL}; 134 JavaVM* vm = AndroidRuntime::getJavaVM(); 135 int result = vm->AttachCurrentThread(&env, (void*) &args); 136 if (result != JNI_OK) { 137 ALOGE("thread attach failed: %#x", result); 138 return NULL; 139 } 140 *needsDetach = true; 141 } 142 return env; 143} 144 145void JNIImageReaderContext::detachJNI() { 146 JavaVM* vm = AndroidRuntime::getJavaVM(); 147 int result = vm->DetachCurrentThread(); 148 if (result != JNI_OK) { 149 ALOGE("thread detach failed: %#x", result); 150 } 151} 152 153CpuConsumer::LockedBuffer* JNIImageReaderContext::getLockedBuffer() { 154 if (mBuffers.empty()) { 155 return NULL; 156 } 157 // Return a LockedBuffer pointer and remove it from the list 158 List<CpuConsumer::LockedBuffer*>::iterator it = mBuffers.begin(); 159 CpuConsumer::LockedBuffer* buffer = *it; 160 mBuffers.erase(it); 161 return buffer; 162} 163 164void JNIImageReaderContext::returnLockedBuffer(CpuConsumer::LockedBuffer* buffer) { 165 mBuffers.push_back(buffer); 166} 167 168JNIImageReaderContext::~JNIImageReaderContext() { 169 bool needsDetach = false; 170 JNIEnv* env = getJNIEnv(&needsDetach); 171 if (env != NULL) { 172 env->DeleteGlobalRef(mWeakThiz); 173 env->DeleteGlobalRef(mClazz); 174 } else { 175 ALOGW("leaking JNI object references"); 176 } 177 if (needsDetach) { 178 detachJNI(); 179 } 180 181 // Delete LockedBuffers 182 for (List<CpuConsumer::LockedBuffer *>::iterator it = mBuffers.begin(); 183 it != mBuffers.end(); it++) { 184 delete *it; 185 } 186 mBuffers.clear(); 187 mConsumer.clear(); 188} 189 190void JNIImageReaderContext::onFrameAvailable() 191{ 192 ALOGV("%s: frame available", __FUNCTION__); 193 bool needsDetach = false; 194 JNIEnv* env = getJNIEnv(&needsDetach); 195 if (env != NULL) { 196 env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz); 197 } else { 198 ALOGW("onFrameAvailable event will not posted"); 199 } 200 if (needsDetach) { 201 detachJNI(); 202 } 203} 204 205// ---------------------------------------------------------------------------- 206 207extern "C" { 208 209static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz) 210{ 211 JNIImageReaderContext *ctx; 212 ctx = reinterpret_cast<JNIImageReaderContext *> 213 (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext)); 214 return ctx; 215} 216 217static CpuConsumer* ImageReader_getCpuConsumer(JNIEnv* env, jobject thiz) 218{ 219 ALOGV("%s:", __FUNCTION__); 220 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 221 if (ctx == NULL) { 222 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 223 return NULL; 224 } 225 return ctx->getCpuConsumer(); 226} 227 228static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz) 229{ 230 ALOGV("%s:", __FUNCTION__); 231 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 232 if (ctx == NULL) { 233 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 234 return NULL; 235 } 236 return ctx->getProducer(); 237} 238 239static void ImageReader_setNativeContext(JNIEnv* env, 240 jobject thiz, sp<JNIImageReaderContext> ctx) 241{ 242 ALOGV("%s:", __FUNCTION__); 243 JNIImageReaderContext* const p = ImageReader_getContext(env, thiz); 244 if (ctx != 0) { 245 ctx->incStrong((void*)ImageReader_setNativeContext); 246 } 247 if (p) { 248 p->decStrong((void*)ImageReader_setNativeContext); 249 } 250 env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext, 251 reinterpret_cast<jlong>(ctx.get())); 252} 253 254static CpuConsumer::LockedBuffer* Image_getLockedBuffer(JNIEnv* env, jobject image) 255{ 256 return reinterpret_cast<CpuConsumer::LockedBuffer*>( 257 env->GetLongField(image, gSurfaceImageClassInfo.mLockedBuffer)); 258} 259 260static void Image_setBuffer(JNIEnv* env, jobject thiz, 261 const CpuConsumer::LockedBuffer* buffer) 262{ 263 env->SetLongField(thiz, gSurfaceImageClassInfo.mLockedBuffer, reinterpret_cast<jlong>(buffer)); 264} 265 266// Some formats like JPEG defined with different values between android.graphics.ImageFormat and 267// graphics.h, need convert to the one defined in graphics.h here. 268static int Image_getPixelFormat(JNIEnv* env, int format) 269{ 270 int jpegFormat; 271 jfieldID fid; 272 273 ALOGV("%s: format = 0x%x", __FUNCTION__, format); 274 275 jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat"); 276 ALOG_ASSERT(imageFormatClazz != NULL); 277 278 fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I"); 279 jpegFormat = env->GetStaticIntField(imageFormatClazz, fid); 280 281 // Translate the JPEG to BLOB for camera purpose. 282 if (format == jpegFormat) { 283 format = HAL_PIXEL_FORMAT_BLOB; 284 } 285 286 return format; 287} 288 289static uint32_t Image_getJpegSize(CpuConsumer::LockedBuffer* buffer) 290{ 291 ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!"); 292 uint32_t size = 0; 293 uint32_t width = buffer->width; 294 uint8_t* jpegBuffer = buffer->data; 295 296 // First check for JPEG transport header at the end of the buffer 297 uint8_t* header = jpegBuffer + (width - sizeof(struct camera3_jpeg_blob)); 298 struct camera3_jpeg_blob *blob = (struct camera3_jpeg_blob*)(header); 299 if (blob->jpeg_blob_id == CAMERA3_JPEG_BLOB_ID) { 300 size = blob->jpeg_size; 301 ALOGV("%s: Jpeg size = %d", __FUNCTION__, size); 302 } 303 304 // failed to find size, default to whole buffer 305 if (size == 0) { 306 /* 307 * This is a problem because not including the JPEG header 308 * means that in certain rare situations a regular JPEG blob 309 * will be misidentified as having a header, in which case 310 * we will get a garbage size value. 311 */ 312 ALOGW("%s: No JPEG header detected, defaulting to size=width=%d", 313 __FUNCTION__, width); 314 size = width; 315 } 316 317 return size; 318} 319 320static int32_t applyFormatOverrides(int32_t bufferFormat, int32_t readerCtxFormat) 321{ 322 // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW 323 // write limitations for some platforms (b/17379185). 324 if (readerCtxFormat == HAL_PIXEL_FORMAT_BLOB && bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888) { 325 return HAL_PIXEL_FORMAT_BLOB; 326 } 327 return bufferFormat; 328} 329 330static void Image_getLockedBufferInfo(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx, 331 uint8_t **base, uint32_t *size, int32_t readerFormat) 332{ 333 ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!"); 334 ALOG_ASSERT(base != NULL, "base is NULL!!!"); 335 ALOG_ASSERT(size != NULL, "size is NULL!!!"); 336 ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0)); 337 338 ALOGV("%s: buffer: %p", __FUNCTION__, buffer); 339 340 uint32_t dataSize, ySize, cSize, cStride; 341 uint8_t *cb, *cr; 342 uint8_t *pData = NULL; 343 int bytesPerPixel = 0; 344 345 dataSize = ySize = cSize = cStride = 0; 346 int32_t fmt = buffer->format; 347 348 fmt = applyFormatOverrides(fmt, readerFormat); 349 switch (fmt) { 350 case HAL_PIXEL_FORMAT_YCbCr_420_888: 351 pData = 352 (idx == 0) ? 353 buffer->data : 354 (idx == 1) ? 355 buffer->dataCb : 356 buffer->dataCr; 357 if (idx == 0) { 358 dataSize = buffer->stride * buffer->height; 359 } else { 360 dataSize = buffer->chromaStride * buffer->height / 2; 361 } 362 break; 363 // NV21 364 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 365 cr = buffer->data + (buffer->stride * buffer->height); 366 cb = cr + 1; 367 ySize = buffer->width * buffer->height; 368 cSize = buffer->width * buffer->height / 2; 369 370 pData = 371 (idx == 0) ? 372 buffer->data : 373 (idx == 1) ? 374 cb: 375 cr; 376 377 dataSize = (idx == 0) ? ySize : cSize; 378 break; 379 case HAL_PIXEL_FORMAT_YV12: 380 // Y and C stride need to be 16 pixel aligned. 381 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 382 "Stride is not 16 pixel aligned %d", buffer->stride); 383 384 ySize = buffer->stride * buffer->height; 385 cStride = ALIGN(buffer->stride / 2, 16); 386 cr = buffer->data + ySize; 387 cSize = cStride * buffer->height / 2; 388 cb = cr + cSize; 389 390 pData = 391 (idx == 0) ? 392 buffer->data : 393 (idx == 1) ? 394 cb : 395 cr; 396 dataSize = (idx == 0) ? ySize : cSize; 397 break; 398 case HAL_PIXEL_FORMAT_Y8: 399 // Single plane, 8bpp. 400 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 401 402 pData = buffer->data; 403 dataSize = buffer->stride * buffer->height; 404 break; 405 case HAL_PIXEL_FORMAT_Y16: 406 bytesPerPixel = 2; 407 // Single plane, 16bpp, strides are specified in pixels, not in bytes 408 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 409 410 pData = buffer->data; 411 dataSize = buffer->stride * buffer->height * bytesPerPixel; 412 break; 413 case HAL_PIXEL_FORMAT_BLOB: 414 // Used for JPEG data, height must be 1, width == size, single plane. 415 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 416 ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height); 417 418 pData = buffer->data; 419 dataSize = Image_getJpegSize(buffer); 420 break; 421 case HAL_PIXEL_FORMAT_RAW_SENSOR: 422 // Single plane 16bpp bayer data. 423 bytesPerPixel = 2; 424 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 425 pData = buffer->data; 426 dataSize = buffer->stride * buffer->height * bytesPerPixel; 427 break; 428 case HAL_PIXEL_FORMAT_RAW10: 429 // Single plane 10bpp bayer data. 430 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 431 LOG_ALWAYS_FATAL_IF(buffer->width % 4, 432 "Width is not multiple of 4 %d", buffer->width); 433 LOG_ALWAYS_FATAL_IF(buffer->height % 2, 434 "Height is not even %d", buffer->height); 435 LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 10 / 8), 436 "stride (%d) should be at least %d", 437 buffer->stride, buffer->width * 10 / 8); 438 pData = buffer->data; 439 dataSize = buffer->stride * buffer->height; 440 break; 441 case HAL_PIXEL_FORMAT_RGBA_8888: 442 case HAL_PIXEL_FORMAT_RGBX_8888: 443 // Single plane, 32bpp. 444 bytesPerPixel = 4; 445 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 446 pData = buffer->data; 447 dataSize = buffer->stride * buffer->height * bytesPerPixel; 448 break; 449 case HAL_PIXEL_FORMAT_RGB_565: 450 // Single plane, 16bpp. 451 bytesPerPixel = 2; 452 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 453 pData = buffer->data; 454 dataSize = buffer->stride * buffer->height * bytesPerPixel; 455 break; 456 case HAL_PIXEL_FORMAT_RGB_888: 457 // Single plane, 24bpp. 458 bytesPerPixel = 3; 459 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 460 pData = buffer->data; 461 dataSize = buffer->stride * buffer->height * bytesPerPixel; 462 break; 463 default: 464 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 465 "Pixel format: 0x%x is unsupported", fmt); 466 break; 467 } 468 469 *base = pData; 470 *size = dataSize; 471} 472 473static jint Image_imageGetPixelStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx, 474 int32_t readerFormat) 475{ 476 ALOGV("%s: buffer index: %d", __FUNCTION__, idx); 477 ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0), "Index is out of range:%d", idx); 478 479 int pixelStride = 0; 480 ALOG_ASSERT(buffer != NULL, "buffer is NULL"); 481 482 int32_t fmt = buffer->format; 483 484 fmt = applyFormatOverrides(fmt, readerFormat); 485 486 switch (fmt) { 487 case HAL_PIXEL_FORMAT_YCbCr_420_888: 488 pixelStride = (idx == 0) ? 1 : buffer->chromaStep; 489 break; 490 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 491 pixelStride = (idx == 0) ? 1 : 2; 492 break; 493 case HAL_PIXEL_FORMAT_Y8: 494 // Single plane 8bpp data. 495 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 496 pixelStride; 497 break; 498 case HAL_PIXEL_FORMAT_YV12: 499 pixelStride = 1; 500 break; 501 case HAL_PIXEL_FORMAT_BLOB: 502 case HAL_PIXEL_FORMAT_RAW10: 503 // Blob is used for JPEG data, RAW10 is used for 10-bit raw data, they are 504 // single plane, row and pixel strides are 0. 505 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 506 pixelStride = 0; 507 break; 508 case HAL_PIXEL_FORMAT_Y16: 509 case HAL_PIXEL_FORMAT_RAW_SENSOR: 510 case HAL_PIXEL_FORMAT_RGB_565: 511 // Single plane 16bpp data. 512 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 513 pixelStride = 2; 514 break; 515 case HAL_PIXEL_FORMAT_RGBA_8888: 516 case HAL_PIXEL_FORMAT_RGBX_8888: 517 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 518 pixelStride = 4; 519 break; 520 case HAL_PIXEL_FORMAT_RGB_888: 521 // Single plane, 24bpp. 522 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 523 pixelStride = 3; 524 break; 525 default: 526 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 527 "Pixel format: 0x%x is unsupported", fmt); 528 break; 529 } 530 531 return pixelStride; 532} 533 534static jint Image_imageGetRowStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx, 535 int32_t readerFormat) 536{ 537 ALOGV("%s: buffer index: %d", __FUNCTION__, idx); 538 ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0)); 539 540 int rowStride = 0; 541 ALOG_ASSERT(buffer != NULL, "buffer is NULL"); 542 543 int32_t fmt = buffer->format; 544 545 fmt = applyFormatOverrides(fmt, readerFormat); 546 547 switch (fmt) { 548 case HAL_PIXEL_FORMAT_YCbCr_420_888: 549 rowStride = (idx == 0) ? buffer->stride : buffer->chromaStride; 550 break; 551 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 552 rowStride = buffer->width; 553 break; 554 case HAL_PIXEL_FORMAT_YV12: 555 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 556 "Stride is not 16 pixel aligned %d", buffer->stride); 557 rowStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16); 558 break; 559 case HAL_PIXEL_FORMAT_BLOB: 560 // Blob is used for JPEG data, RAW10 is used for 10-bit raw data, they are 561 // single plane, row and pixel strides are 0. 562 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 563 rowStride = 0; 564 break; 565 case HAL_PIXEL_FORMAT_RAW10: 566 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 567 rowStride = buffer->stride; 568 break; 569 case HAL_PIXEL_FORMAT_Y8: 570 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 571 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 572 "Stride is not 16 pixel aligned %d", buffer->stride); 573 rowStride = buffer->stride; 574 break; 575 case HAL_PIXEL_FORMAT_Y16: 576 case HAL_PIXEL_FORMAT_RAW_SENSOR: 577 // In native side, strides are specified in pixels, not in bytes. 578 // Single plane 16bpp bayer data. even width/height, 579 // row stride multiple of 16 pixels (32 bytes) 580 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 581 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 582 "Stride is not 16 pixel aligned %d", buffer->stride); 583 rowStride = buffer->stride * 2; 584 break; 585 case HAL_PIXEL_FORMAT_RGB_565: 586 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 587 rowStride = buffer->stride * 2; 588 break; 589 case HAL_PIXEL_FORMAT_RGBA_8888: 590 case HAL_PIXEL_FORMAT_RGBX_8888: 591 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 592 rowStride = buffer->stride * 4; 593 break; 594 case HAL_PIXEL_FORMAT_RGB_888: 595 // Single plane, 24bpp. 596 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 597 rowStride = buffer->stride * 3; 598 break; 599 default: 600 ALOGE("%s Pixel format: 0x%x is unsupported", __FUNCTION__, fmt); 601 jniThrowException(env, "java/lang/UnsupportedOperationException", 602 "unsupported buffer format"); 603 break; 604 } 605 606 return rowStride; 607} 608 609// ---------------------------------------------------------------------------- 610 611static void ImageReader_classInit(JNIEnv* env, jclass clazz) 612{ 613 ALOGV("%s:", __FUNCTION__); 614 615 jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage"); 616 LOG_ALWAYS_FATAL_IF(imageClazz == NULL, 617 "can't find android/graphics/ImageReader$SurfaceImage"); 618 gSurfaceImageClassInfo.mLockedBuffer = env->GetFieldID( 619 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J"); 620 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mLockedBuffer == NULL, 621 "can't find android/graphics/ImageReader.%s", 622 ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID); 623 624 gSurfaceImageClassInfo.mTimestamp = env->GetFieldID( 625 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J"); 626 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL, 627 "can't find android/graphics/ImageReader.%s", 628 ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID); 629 630 gImageReaderClassInfo.mNativeContext = env->GetFieldID( 631 clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J"); 632 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL, 633 "can't find android/graphics/ImageReader.%s", 634 ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID); 635 636 gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID( 637 clazz, "postEventFromNative", "(Ljava/lang/Object;)V"); 638 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL, 639 "can't find android/graphics/ImageReader.postEventFromNative"); 640 641 jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane"); 642 LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class"); 643 // FindClass only gives a local reference of jclass object. 644 gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz); 645 gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>", 646 "(Landroid/media/ImageReader$SurfaceImage;III)V"); 647 LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL, 648 "Can not find SurfacePlane constructor"); 649} 650 651static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, 652 jint width, jint height, jint format, jint maxImages) 653{ 654 status_t res; 655 int nativeFormat; 656 657 ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d", 658 __FUNCTION__, width, height, format, maxImages); 659 660 nativeFormat = Image_getPixelFormat(env, format); 661 662 sp<IGraphicBufferProducer> gbProducer; 663 sp<IGraphicBufferConsumer> gbConsumer; 664 BufferQueue::createBufferQueue(&gbProducer, &gbConsumer); 665 sp<CpuConsumer> consumer = new CpuConsumer(gbConsumer, maxImages, 666 /*controlledByApp*/true); 667 // TODO: throw dvm exOutOfMemoryError? 668 if (consumer == NULL) { 669 jniThrowRuntimeException(env, "Failed to allocate native CpuConsumer"); 670 return; 671 } 672 673 jclass clazz = env->GetObjectClass(thiz); 674 if (clazz == NULL) { 675 jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader"); 676 return; 677 } 678 sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages)); 679 ctx->setCpuConsumer(consumer); 680 ctx->setProducer(gbProducer); 681 consumer->setFrameAvailableListener(ctx); 682 ImageReader_setNativeContext(env, thiz, ctx); 683 ctx->setBufferFormat(nativeFormat); 684 ctx->setBufferWidth(width); 685 ctx->setBufferHeight(height); 686 687 // Set the width/height/format to the CpuConsumer 688 res = consumer->setDefaultBufferSize(width, height); 689 if (res != OK) { 690 jniThrowException(env, "java/lang/IllegalStateException", 691 "Failed to set CpuConsumer buffer size"); 692 return; 693 } 694 res = consumer->setDefaultBufferFormat(nativeFormat); 695 if (res != OK) { 696 jniThrowException(env, "java/lang/IllegalStateException", 697 "Failed to set CpuConsumer buffer format"); 698 } 699} 700 701static void ImageReader_close(JNIEnv* env, jobject thiz) 702{ 703 ALOGV("%s:", __FUNCTION__); 704 705 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 706 if (ctx == NULL) { 707 // ImageReader is already closed. 708 return; 709 } 710 711 CpuConsumer* consumer = ImageReader_getCpuConsumer(env, thiz); 712 if (consumer != NULL) { 713 consumer->abandon(); 714 consumer->setFrameAvailableListener(NULL); 715 } 716 ImageReader_setNativeContext(env, thiz, NULL); 717} 718 719static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image) 720{ 721 ALOGV("%s:", __FUNCTION__); 722 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 723 if (ctx == NULL) { 724 ALOGW("ImageReader#close called before Image#close, consider calling Image#close first"); 725 return; 726 } 727 728 CpuConsumer* consumer = ctx->getCpuConsumer(); 729 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image); 730 if (!buffer) { 731 ALOGW("Image already released!!!"); 732 return; 733 } 734 consumer->unlockBuffer(*buffer); 735 Image_setBuffer(env, image, NULL); 736 ctx->returnLockedBuffer(buffer); 737} 738 739static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, 740 jobject image) 741{ 742 ALOGV("%s:", __FUNCTION__); 743 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 744 if (ctx == NULL) { 745 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 746 return -1; 747 } 748 749 CpuConsumer* consumer = ctx->getCpuConsumer(); 750 CpuConsumer::LockedBuffer* buffer = ctx->getLockedBuffer(); 751 if (buffer == NULL) { 752 ALOGW("Unable to acquire a lockedBuffer, very likely client tries to lock more than" 753 " maxImages buffers"); 754 return ACQUIRE_MAX_IMAGES; 755 } 756 status_t res = consumer->lockNextBuffer(buffer); 757 if (res != NO_ERROR) { 758 ctx->returnLockedBuffer(buffer); 759 if (res != BAD_VALUE /*no buffers*/) { 760 if (res == NOT_ENOUGH_DATA) { 761 return ACQUIRE_MAX_IMAGES; 762 } else { 763 ALOGE("%s Fail to lockNextBuffer with error: %d ", 764 __FUNCTION__, res); 765 jniThrowExceptionFmt(env, "java/lang/AssertionError", 766 "Unknown error (%d) when we tried to lock buffer.", 767 res); 768 } 769 } 770 return ACQUIRE_NO_BUFFERS; 771 } 772 773 if (buffer->format == HAL_PIXEL_FORMAT_YCrCb_420_SP) { 774 jniThrowException(env, "java/lang/UnsupportedOperationException", 775 "NV21 format is not supported by ImageReader"); 776 return -1; 777 } 778 779 // Check if the left-top corner of the crop rect is origin, we currently assume this point is 780 // zero, will revist this once this assumption turns out problematic. 781 Point lt = buffer->crop.leftTop(); 782 if (lt.x != 0 || lt.y != 0) { 783 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 784 "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y); 785 return -1; 786 } 787 788 // Check if the producer buffer configurations match what ImageReader configured. 789 // We want to fail for the very first image because this case is too bad. 790 int outputWidth = buffer->width; 791 int outputHeight = buffer->height; 792 793 // Correct width/height when crop is set. 794 if (!buffer->crop.isEmpty()) { 795 outputWidth = buffer->crop.getWidth(); 796 outputHeight = buffer->crop.getHeight(); 797 } 798 799 int imgReaderFmt = ctx->getBufferFormat(); 800 int imageReaderWidth = ctx->getBufferWidth(); 801 int imageReaderHeight = ctx->getBufferHeight(); 802 if ((buffer->format != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) && 803 (imageReaderWidth != outputWidth || imageReaderHeight > outputHeight)) { 804 /** 805 * For video decoder, the buffer height is actually the vertical stride, 806 * which is always >= actual image height. For future, decoder need provide 807 * right crop rectangle to CpuConsumer to indicate the actual image height, 808 * see bug 9563986. After this bug is fixed, we can enforce the height equal 809 * check. Right now, only make sure buffer height is no less than ImageReader 810 * height. 811 */ 812 jniThrowExceptionFmt(env, "java/lang/IllegalStateException", 813 "Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d", 814 outputWidth, outputHeight, imageReaderWidth, imageReaderHeight); 815 return -1; 816 } 817 818 int bufFmt = buffer->format; 819 if (imgReaderFmt != bufFmt) { 820 821 if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && (bufFmt == 822 HAL_PIXEL_FORMAT_YCrCb_420_SP || bufFmt == HAL_PIXEL_FORMAT_YV12)) { 823 // Special casing for when producer switches to a format compatible with flexible YUV 824 // (HAL_PIXEL_FORMAT_YCbCr_420_888). 825 ctx->setBufferFormat(bufFmt); 826 ALOGD("%s: Overriding buffer format YUV_420_888 to %x.", __FUNCTION__, bufFmt); 827 } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB && bufFmt == HAL_PIXEL_FORMAT_RGBA_8888) { 828 // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW 829 // write limitations for (b/17379185). 830 ALOGD("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__); 831 } else { 832 // Return the buffer to the queue. 833 consumer->unlockBuffer(*buffer); 834 ctx->returnLockedBuffer(buffer); 835 836 // Throw exception 837 ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x", 838 buffer->format, ctx->getBufferFormat()); 839 String8 msg; 840 msg.appendFormat("The producer output buffer format 0x%x doesn't " 841 "match the ImageReader's configured buffer format 0x%x.", 842 buffer->format, ctx->getBufferFormat()); 843 jniThrowException(env, "java/lang/UnsupportedOperationException", 844 msg.string()); 845 return -1; 846 } 847 } 848 // Set SurfaceImage instance member variables 849 Image_setBuffer(env, image, buffer); 850 env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp, 851 static_cast<jlong>(buffer->timestamp)); 852 853 return ACQUIRE_SUCCESS; 854} 855 856static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz) 857{ 858 ALOGV("%s: ", __FUNCTION__); 859 860 IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz); 861 if (gbp == NULL) { 862 jniThrowRuntimeException(env, "CpuConsumer is uninitialized"); 863 return NULL; 864 } 865 866 // Wrap the IGBP in a Java-language Surface. 867 return android_view_Surface_createFromIGraphicBufferProducer(env, gbp); 868} 869 870static jobject Image_createSurfacePlane(JNIEnv* env, jobject thiz, int idx, int readerFormat) 871{ 872 int rowStride, pixelStride; 873 ALOGV("%s: buffer index: %d", __FUNCTION__, idx); 874 875 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz); 876 877 ALOG_ASSERT(buffer != NULL); 878 if (buffer == NULL) { 879 jniThrowException(env, "java/lang/IllegalStateException", "Image was released"); 880 } 881 882 readerFormat = Image_getPixelFormat(env, readerFormat); 883 884 rowStride = Image_imageGetRowStride(env, buffer, idx, readerFormat); 885 pixelStride = Image_imageGetPixelStride(env, buffer, idx, readerFormat); 886 887 jobject surfPlaneObj = env->NewObject(gSurfacePlaneClassInfo.clazz, 888 gSurfacePlaneClassInfo.ctor, thiz, idx, rowStride, pixelStride); 889 890 return surfPlaneObj; 891} 892 893static jobject Image_getByteBuffer(JNIEnv* env, jobject thiz, int idx, int readerFormat) 894{ 895 uint8_t *base = NULL; 896 uint32_t size = 0; 897 jobject byteBuffer; 898 899 ALOGV("%s: buffer index: %d", __FUNCTION__, idx); 900 901 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz); 902 903 if (buffer == NULL) { 904 jniThrowException(env, "java/lang/IllegalStateException", "Image was released"); 905 } 906 907 readerFormat = Image_getPixelFormat(env, readerFormat); 908 909 // Create byteBuffer from native buffer 910 Image_getLockedBufferInfo(env, buffer, idx, &base, &size, readerFormat); 911 912 if (size > static_cast<uint32_t>(INT32_MAX)) { 913 // Byte buffer have 'int capacity', so check the range 914 jniThrowExceptionFmt(env, "java/lang/IllegalStateException", 915 "Size too large for bytebuffer capacity " PRIu32, size); 916 return NULL; 917 } 918 919 byteBuffer = env->NewDirectByteBuffer(base, size); 920 // TODO: throw dvm exOutOfMemoryError? 921 if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) { 922 jniThrowException(env, "java/lang/IllegalStateException", "Failed to allocate ByteBuffer"); 923 } 924 925 return byteBuffer; 926} 927 928} // extern "C" 929 930// ---------------------------------------------------------------------------- 931 932static JNINativeMethod gImageReaderMethods[] = { 933 {"nativeClassInit", "()V", (void*)ImageReader_classInit }, 934 {"nativeInit", "(Ljava/lang/Object;IIII)V", (void*)ImageReader_init }, 935 {"nativeClose", "()V", (void*)ImageReader_close }, 936 {"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease }, 937 {"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup }, 938 {"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface }, 939}; 940 941static JNINativeMethod gImageMethods[] = { 942 {"nativeImageGetBuffer", "(II)Ljava/nio/ByteBuffer;", (void*)Image_getByteBuffer }, 943 {"nativeCreatePlane", "(II)Landroid/media/ImageReader$SurfaceImage$SurfacePlane;", 944 (void*)Image_createSurfacePlane }, 945}; 946 947int register_android_media_ImageReader(JNIEnv *env) { 948 949 int ret1 = AndroidRuntime::registerNativeMethods(env, 950 "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods)); 951 952 int ret2 = AndroidRuntime::registerNativeMethods(env, 953 "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods)); 954 955 return (ret1 || ret2); 956} 957