android_media_ImageReader.cpp revision f6a09e510649ae4701bb5ad4c40d102d59a5608c
1/* 2 * Copyright 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "ImageReader_JNI" 19#include <utils/Log.h> 20#include <utils/misc.h> 21#include <utils/List.h> 22#include <utils/String8.h> 23 24#include <cstdio> 25 26#include <gui/CpuConsumer.h> 27#include <gui/Surface.h> 28#include <camera3.h> 29 30#include <android_runtime/AndroidRuntime.h> 31#include <android_runtime/android_view_Surface.h> 32 33#include <jni.h> 34#include <JNIHelp.h> 35 36#include <stdint.h> 37#include <inttypes.h> 38 39#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) ) 40 41#define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID "mNativeContext" 42#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mLockedBuffer" 43#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp" 44 45// ---------------------------------------------------------------------------- 46 47using namespace android; 48 49enum { 50 IMAGE_READER_MAX_NUM_PLANES = 3, 51}; 52 53enum { 54 ACQUIRE_SUCCESS = 0, 55 ACQUIRE_NO_BUFFERS = 1, 56 ACQUIRE_MAX_IMAGES = 2, 57}; 58 59static struct { 60 jfieldID mNativeContext; 61 jmethodID postEventFromNative; 62} gImageReaderClassInfo; 63 64static struct { 65 jfieldID mLockedBuffer; 66 jfieldID mTimestamp; 67} gSurfaceImageClassInfo; 68 69static struct { 70 jclass clazz; 71 jmethodID ctor; 72} gSurfacePlaneClassInfo; 73 74// ---------------------------------------------------------------------------- 75 76class JNIImageReaderContext : public CpuConsumer::FrameAvailableListener 77{ 78public: 79 JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages); 80 81 virtual ~JNIImageReaderContext(); 82 83 virtual void onFrameAvailable(const BufferItem& item); 84 85 CpuConsumer::LockedBuffer* getLockedBuffer(); 86 87 void returnLockedBuffer(CpuConsumer::LockedBuffer* buffer); 88 89 void setCpuConsumer(const sp<CpuConsumer>& consumer) { mConsumer = consumer; } 90 CpuConsumer* getCpuConsumer() { return mConsumer.get(); } 91 92 void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; } 93 IGraphicBufferProducer* getProducer() { return mProducer.get(); } 94 95 void setBufferFormat(int format) { mFormat = format; } 96 int getBufferFormat() { return mFormat; } 97 98 void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; } 99 android_dataspace getBufferDataspace() { return mDataSpace; } 100 101 void setBufferWidth(int width) { mWidth = width; } 102 int getBufferWidth() { return mWidth; } 103 104 void setBufferHeight(int height) { mHeight = height; } 105 int getBufferHeight() { return mHeight; } 106 107private: 108 static JNIEnv* getJNIEnv(bool* needsDetach); 109 static void detachJNI(); 110 111 List<CpuConsumer::LockedBuffer*> mBuffers; 112 sp<CpuConsumer> mConsumer; 113 sp<IGraphicBufferProducer> mProducer; 114 jobject mWeakThiz; 115 jclass mClazz; 116 int mFormat; 117 android_dataspace mDataSpace; 118 int mWidth; 119 int mHeight; 120}; 121 122JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env, 123 jobject weakThiz, jclass clazz, int maxImages) : 124 mWeakThiz(env->NewGlobalRef(weakThiz)), 125 mClazz((jclass)env->NewGlobalRef(clazz)) { 126 for (int i = 0; i < maxImages; i++) { 127 CpuConsumer::LockedBuffer *buffer = new CpuConsumer::LockedBuffer; 128 mBuffers.push_back(buffer); 129 } 130} 131 132JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) { 133 LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!"); 134 *needsDetach = false; 135 JNIEnv* env = AndroidRuntime::getJNIEnv(); 136 if (env == NULL) { 137 JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL}; 138 JavaVM* vm = AndroidRuntime::getJavaVM(); 139 int result = vm->AttachCurrentThread(&env, (void*) &args); 140 if (result != JNI_OK) { 141 ALOGE("thread attach failed: %#x", result); 142 return NULL; 143 } 144 *needsDetach = true; 145 } 146 return env; 147} 148 149void JNIImageReaderContext::detachJNI() { 150 JavaVM* vm = AndroidRuntime::getJavaVM(); 151 int result = vm->DetachCurrentThread(); 152 if (result != JNI_OK) { 153 ALOGE("thread detach failed: %#x", result); 154 } 155} 156 157CpuConsumer::LockedBuffer* JNIImageReaderContext::getLockedBuffer() { 158 if (mBuffers.empty()) { 159 return NULL; 160 } 161 // Return a LockedBuffer pointer and remove it from the list 162 List<CpuConsumer::LockedBuffer*>::iterator it = mBuffers.begin(); 163 CpuConsumer::LockedBuffer* buffer = *it; 164 mBuffers.erase(it); 165 return buffer; 166} 167 168void JNIImageReaderContext::returnLockedBuffer(CpuConsumer::LockedBuffer* buffer) { 169 mBuffers.push_back(buffer); 170} 171 172JNIImageReaderContext::~JNIImageReaderContext() { 173 bool needsDetach = false; 174 JNIEnv* env = getJNIEnv(&needsDetach); 175 if (env != NULL) { 176 env->DeleteGlobalRef(mWeakThiz); 177 env->DeleteGlobalRef(mClazz); 178 } else { 179 ALOGW("leaking JNI object references"); 180 } 181 if (needsDetach) { 182 detachJNI(); 183 } 184 185 // Delete LockedBuffers 186 for (List<CpuConsumer::LockedBuffer *>::iterator it = mBuffers.begin(); 187 it != mBuffers.end(); it++) { 188 delete *it; 189 } 190 mBuffers.clear(); 191 mConsumer.clear(); 192} 193 194void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/) 195{ 196 ALOGV("%s: frame available", __FUNCTION__); 197 bool needsDetach = false; 198 JNIEnv* env = getJNIEnv(&needsDetach); 199 if (env != NULL) { 200 env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz); 201 } else { 202 ALOGW("onFrameAvailable event will not posted"); 203 } 204 if (needsDetach) { 205 detachJNI(); 206 } 207} 208 209// ---------------------------------------------------------------------------- 210 211extern "C" { 212 213static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz) 214{ 215 JNIImageReaderContext *ctx; 216 ctx = reinterpret_cast<JNIImageReaderContext *> 217 (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext)); 218 return ctx; 219} 220 221static CpuConsumer* ImageReader_getCpuConsumer(JNIEnv* env, jobject thiz) 222{ 223 ALOGV("%s:", __FUNCTION__); 224 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 225 if (ctx == NULL) { 226 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 227 return NULL; 228 } 229 return ctx->getCpuConsumer(); 230} 231 232static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz) 233{ 234 ALOGV("%s:", __FUNCTION__); 235 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 236 if (ctx == NULL) { 237 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 238 return NULL; 239 } 240 return ctx->getProducer(); 241} 242 243static void ImageReader_setNativeContext(JNIEnv* env, 244 jobject thiz, sp<JNIImageReaderContext> ctx) 245{ 246 ALOGV("%s:", __FUNCTION__); 247 JNIImageReaderContext* const p = ImageReader_getContext(env, thiz); 248 if (ctx != 0) { 249 ctx->incStrong((void*)ImageReader_setNativeContext); 250 } 251 if (p) { 252 p->decStrong((void*)ImageReader_setNativeContext); 253 } 254 env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext, 255 reinterpret_cast<jlong>(ctx.get())); 256} 257 258static CpuConsumer::LockedBuffer* Image_getLockedBuffer(JNIEnv* env, jobject image) 259{ 260 return reinterpret_cast<CpuConsumer::LockedBuffer*>( 261 env->GetLongField(image, gSurfaceImageClassInfo.mLockedBuffer)); 262} 263 264static void Image_setBuffer(JNIEnv* env, jobject thiz, 265 const CpuConsumer::LockedBuffer* buffer) 266{ 267 env->SetLongField(thiz, gSurfaceImageClassInfo.mLockedBuffer, reinterpret_cast<jlong>(buffer)); 268} 269 270static uint32_t Image_getJpegSize(CpuConsumer::LockedBuffer* buffer, bool usingRGBAOverride) 271{ 272 ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!"); 273 uint32_t size = 0; 274 uint32_t width = buffer->width; 275 uint8_t* jpegBuffer = buffer->data; 276 277 if (usingRGBAOverride) { 278 width = (buffer->width + buffer->stride * (buffer->height - 1)) * 4; 279 } 280 281 // First check for JPEG transport header at the end of the buffer 282 uint8_t* header = jpegBuffer + (width - sizeof(struct camera3_jpeg_blob)); 283 struct camera3_jpeg_blob *blob = (struct camera3_jpeg_blob*)(header); 284 if (blob->jpeg_blob_id == CAMERA3_JPEG_BLOB_ID) { 285 size = blob->jpeg_size; 286 ALOGV("%s: Jpeg size = %d", __FUNCTION__, size); 287 } 288 289 // failed to find size, default to whole buffer 290 if (size == 0) { 291 /* 292 * This is a problem because not including the JPEG header 293 * means that in certain rare situations a regular JPEG blob 294 * will be misidentified as having a header, in which case 295 * we will get a garbage size value. 296 */ 297 ALOGW("%s: No JPEG header detected, defaulting to size=width=%d", 298 __FUNCTION__, width); 299 size = width; 300 } 301 302 return size; 303} 304 305static bool usingRGBAToJpegOverride(int32_t bufferFormat, int32_t readerCtxFormat) { 306 return readerCtxFormat == HAL_PIXEL_FORMAT_BLOB && bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888; 307} 308 309static int32_t applyFormatOverrides(int32_t bufferFormat, int32_t readerCtxFormat) 310{ 311 // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW 312 // write limitations for some platforms (b/17379185). 313 if (usingRGBAToJpegOverride(bufferFormat, readerCtxFormat)) { 314 return HAL_PIXEL_FORMAT_BLOB; 315 } 316 return bufferFormat; 317} 318 319static void Image_getLockedBufferInfo(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx, 320 uint8_t **base, uint32_t *size, int32_t readerFormat) 321{ 322 ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!"); 323 ALOG_ASSERT(base != NULL, "base is NULL!!!"); 324 ALOG_ASSERT(size != NULL, "size is NULL!!!"); 325 ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0)); 326 327 ALOGV("%s: buffer: %p", __FUNCTION__, buffer); 328 329 uint32_t dataSize, ySize, cSize, cStride; 330 uint8_t *cb, *cr; 331 uint8_t *pData = NULL; 332 int bytesPerPixel = 0; 333 334 dataSize = ySize = cSize = cStride = 0; 335 int32_t fmt = buffer->flexFormat; 336 337 bool usingRGBAOverride = usingRGBAToJpegOverride(fmt, readerFormat); 338 fmt = applyFormatOverrides(fmt, readerFormat); 339 switch (fmt) { 340 case HAL_PIXEL_FORMAT_YCbCr_420_888: 341 pData = 342 (idx == 0) ? 343 buffer->data : 344 (idx == 1) ? 345 buffer->dataCb : 346 buffer->dataCr; 347 // only map until last pixel 348 if (idx == 0) { 349 dataSize = buffer->stride * (buffer->height - 1) + buffer->width; 350 } else { 351 dataSize = buffer->chromaStride * (buffer->height / 2 - 1) + 352 buffer->chromaStep * (buffer->width / 2 - 1) + 1; 353 } 354 break; 355 // NV21 356 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 357 cr = buffer->data + (buffer->stride * buffer->height); 358 cb = cr + 1; 359 // only map until last pixel 360 ySize = buffer->width * (buffer->height - 1) + buffer->width; 361 cSize = buffer->width * (buffer->height / 2 - 1) + buffer->width - 1; 362 363 pData = 364 (idx == 0) ? 365 buffer->data : 366 (idx == 1) ? 367 cb: 368 cr; 369 370 dataSize = (idx == 0) ? ySize : cSize; 371 break; 372 case HAL_PIXEL_FORMAT_YV12: 373 // Y and C stride need to be 16 pixel aligned. 374 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 375 "Stride is not 16 pixel aligned %d", buffer->stride); 376 377 ySize = buffer->stride * buffer->height; 378 cStride = ALIGN(buffer->stride / 2, 16); 379 cr = buffer->data + ySize; 380 cSize = cStride * buffer->height / 2; 381 cb = cr + cSize; 382 383 pData = 384 (idx == 0) ? 385 buffer->data : 386 (idx == 1) ? 387 cb : 388 cr; 389 dataSize = (idx == 0) ? ySize : cSize; 390 break; 391 case HAL_PIXEL_FORMAT_Y8: 392 // Single plane, 8bpp. 393 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 394 395 pData = buffer->data; 396 dataSize = buffer->stride * buffer->height; 397 break; 398 case HAL_PIXEL_FORMAT_Y16: 399 bytesPerPixel = 2; 400 // Single plane, 16bpp, strides are specified in pixels, not in bytes 401 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 402 403 pData = buffer->data; 404 dataSize = buffer->stride * buffer->height * bytesPerPixel; 405 break; 406 case HAL_PIXEL_FORMAT_BLOB: 407 // Used for JPEG data, height must be 1, width == size, single plane. 408 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 409 ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height); 410 411 pData = buffer->data; 412 dataSize = Image_getJpegSize(buffer, usingRGBAOverride); 413 break; 414 case HAL_PIXEL_FORMAT_RAW16: 415 // Single plane 16bpp bayer data. 416 bytesPerPixel = 2; 417 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 418 pData = buffer->data; 419 dataSize = buffer->stride * buffer->height * bytesPerPixel; 420 break; 421 case HAL_PIXEL_FORMAT_RAW10: 422 // Single plane 10bpp bayer data. 423 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 424 LOG_ALWAYS_FATAL_IF(buffer->width % 4, 425 "Width is not multiple of 4 %d", buffer->width); 426 LOG_ALWAYS_FATAL_IF(buffer->height % 2, 427 "Height is not even %d", buffer->height); 428 LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 10 / 8), 429 "stride (%d) should be at least %d", 430 buffer->stride, buffer->width * 10 / 8); 431 pData = buffer->data; 432 dataSize = buffer->stride * buffer->height; 433 break; 434 case HAL_PIXEL_FORMAT_RGBA_8888: 435 case HAL_PIXEL_FORMAT_RGBX_8888: 436 // Single plane, 32bpp. 437 bytesPerPixel = 4; 438 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 439 pData = buffer->data; 440 dataSize = buffer->stride * buffer->height * bytesPerPixel; 441 break; 442 case HAL_PIXEL_FORMAT_RGB_565: 443 // Single plane, 16bpp. 444 bytesPerPixel = 2; 445 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 446 pData = buffer->data; 447 dataSize = buffer->stride * buffer->height * bytesPerPixel; 448 break; 449 case HAL_PIXEL_FORMAT_RGB_888: 450 // Single plane, 24bpp. 451 bytesPerPixel = 3; 452 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 453 pData = buffer->data; 454 dataSize = buffer->stride * buffer->height * bytesPerPixel; 455 break; 456 default: 457 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 458 "Pixel format: 0x%x is unsupported", fmt); 459 break; 460 } 461 462 *base = pData; 463 *size = dataSize; 464} 465 466static jint Image_imageGetPixelStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx, 467 int32_t halReaderFormat) 468{ 469 ALOGV("%s: buffer index: %d", __FUNCTION__, idx); 470 ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0), "Index is out of range:%d", idx); 471 472 int pixelStride = 0; 473 ALOG_ASSERT(buffer != NULL, "buffer is NULL"); 474 475 int32_t fmt = buffer->flexFormat; 476 477 fmt = applyFormatOverrides(fmt, halReaderFormat); 478 479 switch (fmt) { 480 case HAL_PIXEL_FORMAT_YCbCr_420_888: 481 pixelStride = (idx == 0) ? 1 : buffer->chromaStep; 482 break; 483 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 484 pixelStride = (idx == 0) ? 1 : 2; 485 break; 486 case HAL_PIXEL_FORMAT_Y8: 487 // Single plane 8bpp data. 488 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 489 break; 490 case HAL_PIXEL_FORMAT_YV12: 491 pixelStride = 1; 492 break; 493 case HAL_PIXEL_FORMAT_BLOB: 494 case HAL_PIXEL_FORMAT_RAW10: 495 // Blob is used for JPEG data, RAW10 is used for 10-bit raw data, they are 496 // single plane, row and pixel strides are 0. 497 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 498 pixelStride = 0; 499 break; 500 case HAL_PIXEL_FORMAT_Y16: 501 case HAL_PIXEL_FORMAT_RAW16: 502 case HAL_PIXEL_FORMAT_RGB_565: 503 // Single plane 16bpp data. 504 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 505 pixelStride = 2; 506 break; 507 case HAL_PIXEL_FORMAT_RGBA_8888: 508 case HAL_PIXEL_FORMAT_RGBX_8888: 509 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 510 pixelStride = 4; 511 break; 512 case HAL_PIXEL_FORMAT_RGB_888: 513 // Single plane, 24bpp. 514 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 515 pixelStride = 3; 516 break; 517 default: 518 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 519 "Pixel format: 0x%x is unsupported", fmt); 520 break; 521 } 522 523 return pixelStride; 524} 525 526static jint Image_imageGetRowStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx, 527 int32_t halReaderFormat) 528{ 529 ALOGV("%s: buffer index: %d", __FUNCTION__, idx); 530 ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0)); 531 532 int rowStride = 0; 533 ALOG_ASSERT(buffer != NULL, "buffer is NULL"); 534 535 int32_t fmt = buffer->flexFormat; 536 537 fmt = applyFormatOverrides(fmt, halReaderFormat); 538 539 switch (fmt) { 540 case HAL_PIXEL_FORMAT_YCbCr_420_888: 541 rowStride = (idx == 0) ? buffer->stride : buffer->chromaStride; 542 break; 543 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 544 rowStride = buffer->width; 545 break; 546 case HAL_PIXEL_FORMAT_YV12: 547 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 548 "Stride is not 16 pixel aligned %d", buffer->stride); 549 rowStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16); 550 break; 551 case HAL_PIXEL_FORMAT_BLOB: 552 // Blob is used for JPEG data, RAW10 is used for 10-bit raw data, they are 553 // single plane, row and pixel strides are 0. 554 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 555 rowStride = 0; 556 break; 557 case HAL_PIXEL_FORMAT_RAW10: 558 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 559 rowStride = buffer->stride; 560 break; 561 case HAL_PIXEL_FORMAT_Y8: 562 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 563 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 564 "Stride is not 16 pixel aligned %d", buffer->stride); 565 rowStride = buffer->stride; 566 break; 567 case HAL_PIXEL_FORMAT_Y16: 568 case HAL_PIXEL_FORMAT_RAW16: 569 // In native side, strides are specified in pixels, not in bytes. 570 // Single plane 16bpp bayer data. even width/height, 571 // row stride multiple of 16 pixels (32 bytes) 572 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 573 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 574 "Stride is not 16 pixel aligned %d", buffer->stride); 575 rowStride = buffer->stride * 2; 576 break; 577 case HAL_PIXEL_FORMAT_RGB_565: 578 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 579 rowStride = buffer->stride * 2; 580 break; 581 case HAL_PIXEL_FORMAT_RGBA_8888: 582 case HAL_PIXEL_FORMAT_RGBX_8888: 583 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 584 rowStride = buffer->stride * 4; 585 break; 586 case HAL_PIXEL_FORMAT_RGB_888: 587 // Single plane, 24bpp. 588 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 589 rowStride = buffer->stride * 3; 590 break; 591 default: 592 ALOGE("%s Pixel format: 0x%x is unsupported", __FUNCTION__, fmt); 593 jniThrowException(env, "java/lang/UnsupportedOperationException", 594 "unsupported buffer format"); 595 break; 596 } 597 598 return rowStride; 599} 600 601static int Image_getBufferWidth(CpuConsumer::LockedBuffer* buffer) { 602 if (buffer == NULL) return -1; 603 604 if (!buffer->crop.isEmpty()) { 605 return buffer->crop.getWidth(); 606 } 607 return buffer->width; 608} 609 610static int Image_getBufferHeight(CpuConsumer::LockedBuffer* buffer) { 611 if (buffer == NULL) return -1; 612 613 if (!buffer->crop.isEmpty()) { 614 return buffer->crop.getHeight(); 615 } 616 return buffer->height; 617} 618 619// ---------------------------------------------------------------------------- 620 621static void ImageReader_classInit(JNIEnv* env, jclass clazz) 622{ 623 ALOGV("%s:", __FUNCTION__); 624 625 jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage"); 626 LOG_ALWAYS_FATAL_IF(imageClazz == NULL, 627 "can't find android/graphics/ImageReader$SurfaceImage"); 628 gSurfaceImageClassInfo.mLockedBuffer = env->GetFieldID( 629 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J"); 630 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mLockedBuffer == NULL, 631 "can't find android/graphics/ImageReader.%s", 632 ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID); 633 634 gSurfaceImageClassInfo.mTimestamp = env->GetFieldID( 635 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J"); 636 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL, 637 "can't find android/graphics/ImageReader.%s", 638 ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID); 639 640 gImageReaderClassInfo.mNativeContext = env->GetFieldID( 641 clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J"); 642 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL, 643 "can't find android/graphics/ImageReader.%s", 644 ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID); 645 646 gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID( 647 clazz, "postEventFromNative", "(Ljava/lang/Object;)V"); 648 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL, 649 "can't find android/graphics/ImageReader.postEventFromNative"); 650 651 jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane"); 652 LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class"); 653 // FindClass only gives a local reference of jclass object. 654 gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz); 655 gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>", 656 "(Landroid/media/ImageReader$SurfaceImage;III)V"); 657 LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL, 658 "Can not find SurfacePlane constructor"); 659} 660 661static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, 662 jint width, jint height, jint format, jint maxImages) 663{ 664 status_t res; 665 int nativeFormat; 666 android_dataspace nativeDataspace; 667 668 ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d", 669 __FUNCTION__, width, height, format, maxImages); 670 671 PublicFormat publicFormat = static_cast<PublicFormat>(format); 672 nativeFormat = android_view_Surface_mapPublicFormatToHalFormat( 673 publicFormat); 674 nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace( 675 publicFormat); 676 677 sp<IGraphicBufferProducer> gbProducer; 678 sp<IGraphicBufferConsumer> gbConsumer; 679 BufferQueue::createBufferQueue(&gbProducer, &gbConsumer); 680 sp<CpuConsumer> consumer = new CpuConsumer(gbConsumer, maxImages, 681 /*controlledByApp*/true); 682 // TODO: throw dvm exOutOfMemoryError? 683 if (consumer == NULL) { 684 jniThrowRuntimeException(env, "Failed to allocate native CpuConsumer"); 685 return; 686 } 687 688 jclass clazz = env->GetObjectClass(thiz); 689 if (clazz == NULL) { 690 jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader"); 691 return; 692 } 693 sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages)); 694 ctx->setCpuConsumer(consumer); 695 ctx->setProducer(gbProducer); 696 consumer->setFrameAvailableListener(ctx); 697 ImageReader_setNativeContext(env, thiz, ctx); 698 ctx->setBufferFormat(nativeFormat); 699 ctx->setBufferDataspace(nativeDataspace); 700 ctx->setBufferWidth(width); 701 ctx->setBufferHeight(height); 702 703 // Set the width/height/format/dataspace to the CpuConsumer 704 res = consumer->setDefaultBufferSize(width, height); 705 if (res != OK) { 706 jniThrowException(env, "java/lang/IllegalStateException", 707 "Failed to set CpuConsumer buffer size"); 708 return; 709 } 710 res = consumer->setDefaultBufferFormat(nativeFormat); 711 if (res != OK) { 712 jniThrowException(env, "java/lang/IllegalStateException", 713 "Failed to set CpuConsumer buffer format"); 714 } 715 res = consumer->setDefaultBufferDataSpace(nativeDataspace); 716 if (res != OK) { 717 jniThrowException(env, "java/lang/IllegalStateException", 718 "Failed to set CpuConsumer buffer dataSpace"); 719 } 720 721} 722 723static void ImageReader_close(JNIEnv* env, jobject thiz) 724{ 725 ALOGV("%s:", __FUNCTION__); 726 727 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 728 if (ctx == NULL) { 729 // ImageReader is already closed. 730 return; 731 } 732 733 CpuConsumer* consumer = ImageReader_getCpuConsumer(env, thiz); 734 if (consumer != NULL) { 735 consumer->abandon(); 736 consumer->setFrameAvailableListener(NULL); 737 } 738 ImageReader_setNativeContext(env, thiz, NULL); 739} 740 741static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image) 742{ 743 ALOGV("%s:", __FUNCTION__); 744 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 745 if (ctx == NULL) { 746 ALOGW("ImageReader#close called before Image#close, consider calling Image#close first"); 747 return; 748 } 749 750 CpuConsumer* consumer = ctx->getCpuConsumer(); 751 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image); 752 if (!buffer) { 753 ALOGW("Image already released!!!"); 754 return; 755 } 756 consumer->unlockBuffer(*buffer); 757 Image_setBuffer(env, image, NULL); 758 ctx->returnLockedBuffer(buffer); 759} 760 761static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, 762 jobject image) 763{ 764 ALOGV("%s:", __FUNCTION__); 765 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 766 if (ctx == NULL) { 767 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 768 return -1; 769 } 770 771 CpuConsumer* consumer = ctx->getCpuConsumer(); 772 CpuConsumer::LockedBuffer* buffer = ctx->getLockedBuffer(); 773 if (buffer == NULL) { 774 ALOGW("Unable to acquire a lockedBuffer, very likely client tries to lock more than" 775 " maxImages buffers"); 776 return ACQUIRE_MAX_IMAGES; 777 } 778 status_t res = consumer->lockNextBuffer(buffer); 779 if (res != NO_ERROR) { 780 ctx->returnLockedBuffer(buffer); 781 if (res != BAD_VALUE /*no buffers*/) { 782 if (res == NOT_ENOUGH_DATA) { 783 return ACQUIRE_MAX_IMAGES; 784 } else { 785 ALOGE("%s Fail to lockNextBuffer with error: %d ", 786 __FUNCTION__, res); 787 jniThrowExceptionFmt(env, "java/lang/AssertionError", 788 "Unknown error (%d) when we tried to lock buffer.", 789 res); 790 } 791 } 792 return ACQUIRE_NO_BUFFERS; 793 } 794 795 if (buffer->flexFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) { 796 jniThrowException(env, "java/lang/UnsupportedOperationException", 797 "NV21 format is not supported by ImageReader"); 798 return -1; 799 } 800 801 // Check if the left-top corner of the crop rect is origin, we currently assume this point is 802 // zero, will revist this once this assumption turns out problematic. 803 Point lt = buffer->crop.leftTop(); 804 if (lt.x != 0 || lt.y != 0) { 805 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 806 "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y); 807 return -1; 808 } 809 810 // Check if the producer buffer configurations match what ImageReader configured. 811 int outputWidth = Image_getBufferWidth(buffer); 812 int outputHeight = Image_getBufferHeight(buffer); 813 814 int imgReaderFmt = ctx->getBufferFormat(); 815 int imageReaderWidth = ctx->getBufferWidth(); 816 int imageReaderHeight = ctx->getBufferHeight(); 817 if ((buffer->format != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) && 818 (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) { 819 ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d", 820 __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight); 821 } 822 823 int bufFmt = buffer->format; 824 if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888) { 825 bufFmt = buffer->flexFormat; 826 } 827 if (imgReaderFmt != bufFmt) { 828 if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && (bufFmt == 829 HAL_PIXEL_FORMAT_YCrCb_420_SP || bufFmt == HAL_PIXEL_FORMAT_YV12)) { 830 // Special casing for when producer switches to a format compatible with flexible YUV 831 // (HAL_PIXEL_FORMAT_YCbCr_420_888). 832 ctx->setBufferFormat(bufFmt); 833 ALOGD("%s: Overriding buffer format YUV_420_888 to %x.", __FUNCTION__, bufFmt); 834 } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB && bufFmt == HAL_PIXEL_FORMAT_RGBA_8888) { 835 // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW 836 // write limitations for (b/17379185). 837 ALOGD("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__); 838 } else { 839 // Return the buffer to the queue. 840 consumer->unlockBuffer(*buffer); 841 ctx->returnLockedBuffer(buffer); 842 843 // Throw exception 844 ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x", 845 buffer->format, ctx->getBufferFormat()); 846 String8 msg; 847 msg.appendFormat("The producer output buffer format 0x%x doesn't " 848 "match the ImageReader's configured buffer format 0x%x.", 849 bufFmt, ctx->getBufferFormat()); 850 jniThrowException(env, "java/lang/UnsupportedOperationException", 851 msg.string()); 852 return -1; 853 } 854 } 855 // Set SurfaceImage instance member variables 856 Image_setBuffer(env, image, buffer); 857 env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp, 858 static_cast<jlong>(buffer->timestamp)); 859 860 return ACQUIRE_SUCCESS; 861} 862 863static void ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) { 864 ALOGV("%s:", __FUNCTION__); 865 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 866 if (ctx == NULL) { 867 jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed"); 868 return; 869 } 870 871 // CpuConsumer* consumer = ctx->getCpuConsumer(); 872 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image); 873 if (!buffer) { 874 ALOGW("Image already released!!!"); 875 return; 876 } 877 878 // TODO: need implement 879 jniThrowRuntimeException(env, "nativeDetachImage is not implemented yet!!!"); 880} 881 882static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz) 883{ 884 ALOGV("%s: ", __FUNCTION__); 885 886 IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz); 887 if (gbp == NULL) { 888 jniThrowRuntimeException(env, "CpuConsumer is uninitialized"); 889 return NULL; 890 } 891 892 // Wrap the IGBP in a Java-language Surface. 893 return android_view_Surface_createFromIGraphicBufferProducer(env, gbp); 894} 895 896static jobject Image_createSurfacePlane(JNIEnv* env, jobject thiz, int idx, int readerFormat) 897{ 898 int rowStride, pixelStride; 899 PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat); 900 901 ALOGV("%s: buffer index: %d", __FUNCTION__, idx); 902 903 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz); 904 905 ALOG_ASSERT(buffer != NULL); 906 if (buffer == NULL) { 907 jniThrowException(env, "java/lang/IllegalStateException", "Image was released"); 908 } 909 910 int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat( 911 publicReaderFormat); 912 913 rowStride = Image_imageGetRowStride(env, buffer, idx, halReaderFormat); 914 pixelStride = Image_imageGetPixelStride(env, buffer, idx, halReaderFormat); 915 916 jobject surfPlaneObj = env->NewObject(gSurfacePlaneClassInfo.clazz, 917 gSurfacePlaneClassInfo.ctor, thiz, idx, rowStride, pixelStride); 918 919 return surfPlaneObj; 920} 921 922static jobject Image_getByteBuffer(JNIEnv* env, jobject thiz, int idx, int readerFormat) 923{ 924 uint8_t *base = NULL; 925 uint32_t size = 0; 926 jobject byteBuffer; 927 PublicFormat readerPublicFormat = static_cast<PublicFormat>(readerFormat); 928 929 ALOGV("%s: buffer index: %d", __FUNCTION__, idx); 930 931 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz); 932 933 if (buffer == NULL) { 934 jniThrowException(env, "java/lang/IllegalStateException", "Image was released"); 935 } 936 937 int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat( 938 readerPublicFormat); 939 940 // Create byteBuffer from native buffer 941 Image_getLockedBufferInfo(env, buffer, idx, &base, &size, readerHalFormat); 942 943 if (size > static_cast<uint32_t>(INT32_MAX)) { 944 // Byte buffer have 'int capacity', so check the range 945 jniThrowExceptionFmt(env, "java/lang/IllegalStateException", 946 "Size too large for bytebuffer capacity %" PRIu32, size); 947 return NULL; 948 } 949 950 byteBuffer = env->NewDirectByteBuffer(base, size); 951 // TODO: throw dvm exOutOfMemoryError? 952 if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) { 953 jniThrowException(env, "java/lang/IllegalStateException", "Failed to allocate ByteBuffer"); 954 } 955 956 return byteBuffer; 957} 958 959static jint Image_getWidth(JNIEnv* env, jobject thiz) 960{ 961 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz); 962 return Image_getBufferWidth(buffer); 963} 964 965static jint Image_getHeight(JNIEnv* env, jobject thiz) 966{ 967 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz); 968 return Image_getBufferHeight(buffer); 969} 970 971 972} // extern "C" 973 974// ---------------------------------------------------------------------------- 975 976static JNINativeMethod gImageReaderMethods[] = { 977 {"nativeClassInit", "()V", (void*)ImageReader_classInit }, 978 {"nativeInit", "(Ljava/lang/Object;IIII)V", (void*)ImageReader_init }, 979 {"nativeClose", "()V", (void*)ImageReader_close }, 980 {"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease }, 981 {"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup }, 982 {"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface }, 983 {"nativeDetachImage", "(Landroid/media/Image;)V", (void*)ImageReader_detachImage }, 984}; 985 986static JNINativeMethod gImageMethods[] = { 987 {"nativeImageGetBuffer", "(II)Ljava/nio/ByteBuffer;", (void*)Image_getByteBuffer }, 988 {"nativeCreatePlane", "(II)Landroid/media/ImageReader$SurfaceImage$SurfacePlane;", 989 (void*)Image_createSurfacePlane }, 990 {"nativeGetWidth", "()I", (void*)Image_getWidth }, 991 {"nativeGetHeight", "()I", (void*)Image_getHeight }, 992}; 993 994int register_android_media_ImageReader(JNIEnv *env) { 995 996 int ret1 = AndroidRuntime::registerNativeMethods(env, 997 "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods)); 998 999 int ret2 = AndroidRuntime::registerNativeMethods(env, 1000 "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods)); 1001 1002 return (ret1 || ret2); 1003} 1004