ImageReader.java revision e3351f1942bfe86682389b278e7ff128a72ea671
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.media; 18 19import android.graphics.ImageFormat; 20import android.graphics.PixelFormat; 21import android.os.Handler; 22import android.os.Looper; 23import android.view.Surface; 24 25import java.lang.ref.WeakReference; 26import java.nio.ByteBuffer; 27import java.nio.ByteOrder; 28 29/** 30 * <p>The ImageReader class allows direct application access to image data 31 * rendered into a {@link android.view.Surface}</p> 32 * 33 * <p>Several Android media API classes accept Surface objects as targets to 34 * render to, including {@link MediaPlayer}, {@link MediaCodec}, 35 * {@link android.hardware.camera2.CameraDevice}, and 36 * {@link android.renderscript.Allocation RenderScript Allocations}. The image 37 * sizes and formats that can be used with each source vary, and should be 38 * checked in the documentation for the specific API.</p> 39 * 40 * <p>The image data is encapsulated in {@link Image} objects, and multiple such 41 * objects can be accessed at the same time, up to the number specified by the 42 * {@code maxImages} constructor parameter. New images sent to an ImageReader 43 * through its {@link Surface} are queued until accessed through the {@link #acquireLatestImage} 44 * or {@link #acquireNextImage} call. Due to memory limits, an image source will 45 * eventually stall or drop Images in trying to render to the Surface if the 46 * ImageReader does not obtain and release Images at a rate equal to the 47 * production rate.</p> 48 */ 49public class ImageReader implements AutoCloseable { 50 51 /** 52 * Returned by nativeImageSetup when acquiring the image was successful. 53 */ 54 private static final int ACQUIRE_SUCCESS = 0; 55 /** 56 * Returned by nativeImageSetup when we couldn't acquire the buffer, 57 * because there were no buffers available to acquire. 58 */ 59 private static final int ACQUIRE_NO_BUFS = 1; 60 /** 61 * Returned by nativeImageSetup when we couldn't acquire the buffer 62 * because the consumer has already acquired {@maxImages} and cannot 63 * acquire more than that. 64 */ 65 private static final int ACQUIRE_MAX_IMAGES = 2; 66 67 /** 68 * <p>Create a new reader for images of the desired size and format.</p> 69 * 70 * <p>The {@code maxImages} parameter determines the maximum number of {@link Image} 71 * objects that can be be acquired from the {@code ImageReader} 72 * simultaneously. Requesting more buffers will use up more memory, so it is 73 * important to use only the minimum number necessary for the use case.</p> 74 * 75 * <p>The valid sizes and formats depend on the source of the image 76 * data.</p> 77 * 78 * @param width 79 * The width in pixels of the Images that this reader will produce. 80 * @param height 81 * The height in pixels of the Images that this reader will produce. 82 * @param format 83 * The format of the Image that this reader will produce. This 84 * must be one of the {@link android.graphics.ImageFormat} or 85 * {@link android.graphics.PixelFormat} constants. 86 * @param maxImages 87 * The maximum number of images the user will want to 88 * access simultaneously. This should be as small as possible to limit 89 * memory use. Once maxImages Images are obtained by the user, one of them 90 * has to be released before a new Image will become available for access 91 * through {@link #acquireLatestImage()} or {@link #acquireNextImage()}. 92 * Must be greater than 0. 93 * 94 * @see Image 95 */ 96 public static ImageReader newInstance(int width, int height, int format, int maxImages) { 97 return new ImageReader(width, height, format, maxImages); 98 } 99 100 /** 101 * @hide 102 */ 103 protected ImageReader(int width, int height, int format, int maxImages) { 104 mWidth = width; 105 mHeight = height; 106 mFormat = format; 107 mMaxImages = maxImages; 108 109 if (width < 1 || height < 1) { 110 throw new IllegalArgumentException( 111 "The image dimensions must be positive"); 112 } 113 if (mMaxImages < 1) { 114 throw new IllegalArgumentException( 115 "Maximum outstanding image count must be at least 1"); 116 } 117 118 mNumPlanes = getNumPlanesFromFormat(); 119 120 nativeInit(new WeakReference<ImageReader>(this), width, height, format, maxImages); 121 122 mSurface = nativeGetSurface(); 123 } 124 125 /** 126 * The width of each {@link Image}, in pixels. 127 * 128 * <p>ImageReader guarantees that all Images acquired from ImageReader (for example, with 129 * {@link #acquireNextImage}) will have the same dimensions as specified in 130 * {@link #newInstance}.</p> 131 * 132 * @return the width of an Image 133 */ 134 public int getWidth() { 135 return mWidth; 136 } 137 138 /** 139 * The height of each {@link Image}, in pixels. 140 * 141 * <p>ImageReader guarantees that all Images acquired from ImageReader (for example, with 142 * {@link #acquireNextImage}) will have the same dimensions as specified in 143 * {@link #newInstance}.</p> 144 * 145 * @return the height of an Image 146 */ 147 public int getHeight() { 148 return mHeight; 149 } 150 151 /** 152 * The {@link ImageFormat image format} of each Image. 153 * 154 * <p>ImageReader guarantees that all {@link Image Images} acquired from ImageReader 155 * (for example, with {@link #acquireNextImage}) will have the same format as specified in 156 * {@link #newInstance}.</p> 157 * 158 * @return the format of an Image 159 * 160 * @see ImageFormat 161 */ 162 public int getImageFormat() { 163 return mFormat; 164 } 165 166 /** 167 * Maximum number of images that can be acquired from the ImageReader by any time (for example, 168 * with {@link #acquireNextImage}). 169 * 170 * <p>An image is considered acquired after it's returned by a function from ImageReader, and 171 * until the Image is {@link Image#close closed} to release the image back to the ImageReader. 172 * </p> 173 * 174 * <p>Attempting to acquire more than {@code maxImages} concurrently will result in the 175 * acquire function throwing a {@link IllegalStateException}. Furthermore, 176 * while the max number of images have been acquired by the ImageReader user, the producer 177 * enqueueing additional images may stall until at least one image has been released. </p> 178 * 179 * @return Maximum number of images for this ImageReader. 180 * 181 * @see Image#close 182 */ 183 public int getMaxImages() { 184 return mMaxImages; 185 } 186 187 /** 188 * <p>Get a {@link Surface} that can be used to produce {@link Image Images} for this 189 * {@code ImageReader}.</p> 190 * 191 * <p>Until valid image data is rendered into this {@link Surface}, the 192 * {@link #acquireNextImage} method will return {@code null}. Only one source 193 * can be producing data into this Surface at the same time, although the 194 * same {@link Surface} can be reused with a different API once the first source is 195 * disconnected from the {@link Surface}.</p> 196 * 197 * @return A {@link Surface} to use for a drawing target for various APIs. 198 */ 199 public Surface getSurface() { 200 return mSurface; 201 } 202 203 /** 204 * <p> 205 * Acquire the latest {@link Image} from the ImageReader's queue, dropping older 206 * {@link Image images}. Returns {@code null} if no new image is available. 207 * </p> 208 * <p> 209 * This operation will acquire all the images possible from the ImageReader, 210 * but {@link #close} all images that aren't the latest. This function is 211 * recommended to use over {@link #acquireNextImage} for most use-cases, as it's 212 * more suited for real-time processing. 213 * </p> 214 * <p> 215 * Note that {@link #getMaxImages maxImages} should be at least 2 for 216 * {@link #acquireLatestImage} to be any different than {@link #acquireNextImage} - 217 * discarding all-but-the-newest {@link Image} requires temporarily acquiring two 218 * {@link Image Images} at once. Or more generally, calling {@link #acquireLatestImage} 219 * with less than two images of margin, that is 220 * {@code (maxImages - currentAcquiredImages < 2)} will not discard as expected. 221 * </p> 222 * <p> 223 * This operation will fail by throwing an {@link IllegalStateException} if 224 * {@code maxImages} have been acquired with {@link #acquireLatestImage} or 225 * {@link #acquireNextImage}. In particular a sequence of {@link #acquireLatestImage} 226 * calls greater than {@link #getMaxImages} without calling {@link Image#close} in-between 227 * will exhaust the underlying queue. At such a time, {@link IllegalStateException} 228 * will be thrown until more images are 229 * released with {@link Image#close}. 230 * </p> 231 * 232 * @return latest frame of image data, or {@code null} if no image data is available. 233 * @throws IllegalStateException if too many images are currently acquired 234 */ 235 public Image acquireLatestImage() { 236 Image image = acquireNextImage(); 237 if (image == null) { 238 return null; 239 } 240 try { 241 for (;;) { 242 Image next = acquireNextImageNoThrowISE(); 243 if (next == null) { 244 Image result = image; 245 image = null; 246 return result; 247 } 248 image.close(); 249 image = next; 250 } 251 } finally { 252 if (image != null) { 253 image.close(); 254 } 255 } 256 } 257 258 /** 259 * Don't throw IllegalStateException if there are too many images acquired. 260 * 261 * @return Image if acquiring succeeded, or null otherwise. 262 * 263 * @hide 264 */ 265 public Image acquireNextImageNoThrowISE() { 266 SurfaceImage si = new SurfaceImage(); 267 return acquireNextSurfaceImage(si) == ACQUIRE_SUCCESS ? si : null; 268 } 269 270 /** 271 * Attempts to acquire the next image from the underlying native implementation. 272 * 273 * <p> 274 * Note that unexpected failures will throw at the JNI level. 275 * </p> 276 * 277 * @param si A blank SurfaceImage. 278 * @return One of the {@code ACQUIRE_*} codes that determine success or failure. 279 * 280 * @see #ACQUIRE_MAX_IMAGES 281 * @see #ACQUIRE_NO_BUFS 282 * @see #ACQUIRE_SUCCESS 283 */ 284 private int acquireNextSurfaceImage(SurfaceImage si) { 285 286 int status = nativeImageSetup(si); 287 288 switch (status) { 289 case ACQUIRE_SUCCESS: 290 si.createSurfacePlanes(); 291 si.setImageValid(true); 292 case ACQUIRE_NO_BUFS: 293 case ACQUIRE_MAX_IMAGES: 294 break; 295 default: 296 throw new AssertionError("Unknown nativeImageSetup return code " + status); 297 } 298 299 return status; 300 } 301 302 /** 303 * <p> 304 * Acquire the next Image from the ImageReader's queue. Returns {@code null} if 305 * no new image is available. 306 * </p> 307 * 308 * <p><i>Warning:</i> Consider using {@link #acquireLatestImage()} instead, as it will 309 * automatically release older images, and allow slower-running processing routines to catch 310 * up to the newest frame. Usage of {@link #acquireNextImage} is recommended for 311 * batch/background processing. Incorrectly using this function can cause images to appear 312 * with an ever-increasing delay, followed by a complete stall where no new images seem to 313 * appear. 314 * </p> 315 * 316 * <p> 317 * This operation will fail by throwing an {@link IllegalStateException} if 318 * {@code maxImages} have been acquired with {@link #acquireNextImage} or 319 * {@link #acquireLatestImage}. In particular a sequence of {@link #acquireNextImage} or 320 * {@link #acquireLatestImage} calls greater than {@link #getMaxImages maxImages} without 321 * calling {@link Image#close} in-between will exhaust the underlying queue. At such a time, 322 * {@link IllegalStateException} will be thrown until more images are released with 323 * {@link Image#close}. 324 * </p> 325 * 326 * @return a new frame of image data, or {@code null} if no image data is available. 327 * @throws IllegalStateException if {@code maxImages} images are currently acquired 328 * @see #acquireLatestImage 329 */ 330 public Image acquireNextImage() { 331 SurfaceImage si = new SurfaceImage(); 332 int status = acquireNextSurfaceImage(si); 333 334 switch (status) { 335 case ACQUIRE_SUCCESS: 336 return si; 337 case ACQUIRE_NO_BUFS: 338 return null; 339 case ACQUIRE_MAX_IMAGES: 340 throw new IllegalStateException( 341 String.format( 342 "maxImages (%d) has already been acquired, " + 343 "call #close before acquiring more.", mMaxImages)); 344 default: 345 throw new AssertionError("Unknown nativeImageSetup return code " + status); 346 } 347 } 348 349 /** 350 * <p>Return the frame to the ImageReader for reuse.</p> 351 */ 352 private void releaseImage(Image i) { 353 if (! (i instanceof SurfaceImage) ) { 354 throw new IllegalArgumentException( 355 "This image was not produced by an ImageReader"); 356 } 357 SurfaceImage si = (SurfaceImage) i; 358 if (si.getReader() != this) { 359 throw new IllegalArgumentException( 360 "This image was not produced by this ImageReader"); 361 } 362 363 si.clearSurfacePlanes(); 364 nativeReleaseImage(i); 365 si.setImageValid(false); 366 } 367 368 /** 369 * Register a listener to be invoked when a new image becomes available 370 * from the ImageReader. 371 * 372 * @param listener 373 * The listener that will be run. 374 * @param handler 375 * The handler on which the listener should be invoked, or null 376 * if the listener should be invoked on the calling thread's looper. 377 * @throws IllegalArgumentException 378 * If no handler specified and the calling thread has no looper. 379 */ 380 public void setOnImageAvailableListener(OnImageAvailableListener listener, Handler handler) { 381 mImageListener = listener; 382 383 Looper looper; 384 mHandler = handler; 385 if (listener != null && mHandler == null) { 386 if ((looper = Looper.myLooper()) != null) { 387 mHandler = new Handler(); 388 } else { 389 throw new IllegalArgumentException( 390 "Looper doesn't exist in the calling thread"); 391 } 392 } 393 } 394 395 /** 396 * Callback interface for being notified that a new image is available. 397 * 398 * <p> 399 * The onImageAvailable is called per image basis, that is, callback fires for every new frame 400 * available from ImageReader. 401 * </p> 402 */ 403 public interface OnImageAvailableListener { 404 /** 405 * Callback that is called when a new image is available from ImageReader. 406 * 407 * @param reader the ImageReader the callback is associated with. 408 * @see ImageReader 409 * @see Image 410 */ 411 void onImageAvailable(ImageReader reader); 412 } 413 414 /** 415 * Free up all the resources associated with this ImageReader. 416 * 417 * <p> 418 * After calling this method, this ImageReader can not be used. Calling 419 * any methods on this ImageReader and Images previously provided by 420 * {@link #acquireNextImage} or {@link #acquireLatestImage} 421 * will result in an {@link IllegalStateException}, and attempting to read from 422 * {@link ByteBuffer ByteBuffers} returned by an earlier 423 * {@link Image.Plane#getBuffer Plane#getBuffer} call will 424 * have undefined behavior. 425 * </p> 426 */ 427 @Override 428 public void close() { 429 nativeClose(); 430 } 431 432 @Override 433 protected void finalize() throws Throwable { 434 try { 435 close(); 436 } finally { 437 super.finalize(); 438 } 439 } 440 441 /** 442 * Only a subset of the formats defined in 443 * {@link android.graphics.ImageFormat ImageFormat} and 444 * {@link android.graphics.PixelFormat PixelFormat} are supported by 445 * ImageReader. When reading RGB data from a surface, the formats defined in 446 * {@link android.graphics.PixelFormat PixelFormat} can be used, when 447 * reading YUV, JPEG or raw sensor data (for example, from camera or video 448 * decoder), formats from {@link android.graphics.ImageFormat ImageFormat} 449 * are used. 450 */ 451 private int getNumPlanesFromFormat() { 452 switch (mFormat) { 453 case ImageFormat.YV12: 454 case ImageFormat.YUV_420_888: 455 case ImageFormat.NV21: 456 return 3; 457 case ImageFormat.NV16: 458 return 2; 459 case PixelFormat.RGB_565: 460 case PixelFormat.RGBA_8888: 461 case PixelFormat.RGBX_8888: 462 case PixelFormat.RGB_888: 463 case ImageFormat.JPEG: 464 case ImageFormat.YUY2: 465 case ImageFormat.Y8: 466 case ImageFormat.Y16: 467 case ImageFormat.RAW_SENSOR: 468 return 1; 469 default: 470 throw new UnsupportedOperationException( 471 String.format("Invalid format specified %d", mFormat)); 472 } 473 } 474 475 /** 476 * Called from Native code when an Event happens. 477 */ 478 private static void postEventFromNative(Object selfRef) { 479 @SuppressWarnings("unchecked") 480 WeakReference<ImageReader> weakSelf = (WeakReference<ImageReader>)selfRef; 481 final ImageReader ir = weakSelf.get(); 482 if (ir == null) { 483 return; 484 } 485 486 if (ir.mHandler != null && ir.mImageListener != null) { 487 ir.mHandler.post(new Runnable() { 488 @Override 489 public void run() { 490 ir.mImageListener.onImageAvailable(ir); 491 } 492 }); 493 } 494 } 495 496 private final int mWidth; 497 private final int mHeight; 498 private final int mFormat; 499 private final int mMaxImages; 500 private final int mNumPlanes; 501 private final Surface mSurface; 502 503 private Handler mHandler; 504 private OnImageAvailableListener mImageListener; 505 506 /** 507 * This field is used by native code, do not access or modify. 508 */ 509 private long mNativeContext; 510 511 private class SurfaceImage extends android.media.Image { 512 public SurfaceImage() { 513 mIsImageValid = false; 514 } 515 516 @Override 517 public void close() { 518 if (mIsImageValid) { 519 ImageReader.this.releaseImage(this); 520 } 521 } 522 523 public ImageReader getReader() { 524 return ImageReader.this; 525 } 526 527 @Override 528 public int getFormat() { 529 if (mIsImageValid) { 530 return ImageReader.this.mFormat; 531 } else { 532 throw new IllegalStateException("Image is already released"); 533 } 534 } 535 536 @Override 537 public int getWidth() { 538 if (mIsImageValid) { 539 return ImageReader.this.mWidth; 540 } else { 541 throw new IllegalStateException("Image is already released"); 542 } 543 } 544 545 @Override 546 public int getHeight() { 547 if (mIsImageValid) { 548 return ImageReader.this.mHeight; 549 } else { 550 throw new IllegalStateException("Image is already released"); 551 } 552 } 553 554 @Override 555 public long getTimestamp() { 556 if (mIsImageValid) { 557 return mTimestamp; 558 } else { 559 throw new IllegalStateException("Image is already released"); 560 } 561 } 562 563 @Override 564 public Plane[] getPlanes() { 565 if (mIsImageValid) { 566 // Shallow copy is fine. 567 return mPlanes.clone(); 568 } else { 569 throw new IllegalStateException("Image is already released"); 570 } 571 } 572 573 @Override 574 protected final void finalize() throws Throwable { 575 try { 576 close(); 577 } finally { 578 super.finalize(); 579 } 580 } 581 582 private void setImageValid(boolean isValid) { 583 mIsImageValid = isValid; 584 } 585 586 private boolean isImageValid() { 587 return mIsImageValid; 588 } 589 590 private void clearSurfacePlanes() { 591 if (mIsImageValid) { 592 for (int i = 0; i < mPlanes.length; i++) { 593 if (mPlanes[i] != null) { 594 mPlanes[i].clearBuffer(); 595 mPlanes[i] = null; 596 } 597 } 598 } 599 } 600 601 private void createSurfacePlanes() { 602 mPlanes = new SurfacePlane[ImageReader.this.mNumPlanes]; 603 for (int i = 0; i < ImageReader.this.mNumPlanes; i++) { 604 mPlanes[i] = nativeCreatePlane(i); 605 } 606 } 607 private class SurfacePlane extends android.media.Image.Plane { 608 // SurfacePlane instance is created by native code when a new SurfaceImage is created 609 private SurfacePlane(int index, int rowStride, int pixelStride) { 610 mIndex = index; 611 mRowStride = rowStride; 612 mPixelStride = pixelStride; 613 } 614 615 @Override 616 public ByteBuffer getBuffer() { 617 if (SurfaceImage.this.isImageValid() == false) { 618 throw new IllegalStateException("Image is already released"); 619 } 620 if (mBuffer != null) { 621 return mBuffer; 622 } else { 623 mBuffer = SurfaceImage.this.nativeImageGetBuffer(mIndex); 624 // Set the byteBuffer order according to host endianness (native order), 625 // otherwise, the byteBuffer order defaults to ByteOrder.BIG_ENDIAN. 626 return mBuffer.order(ByteOrder.nativeOrder()); 627 } 628 } 629 630 @Override 631 public int getPixelStride() { 632 if (SurfaceImage.this.isImageValid()) { 633 return mPixelStride; 634 } else { 635 throw new IllegalStateException("Image is already released"); 636 } 637 } 638 639 @Override 640 public int getRowStride() { 641 if (SurfaceImage.this.isImageValid()) { 642 return mRowStride; 643 } else { 644 throw new IllegalStateException("Image is already released"); 645 } 646 } 647 648 private void clearBuffer() { 649 mBuffer = null; 650 } 651 652 final private int mIndex; 653 final private int mPixelStride; 654 final private int mRowStride; 655 656 private ByteBuffer mBuffer; 657 } 658 659 /** 660 * This field is used to keep track of native object and used by native code only. 661 * Don't modify. 662 */ 663 private long mLockedBuffer; 664 665 /** 666 * This field is set by native code during nativeImageSetup(). 667 */ 668 private long mTimestamp; 669 670 private SurfacePlane[] mPlanes; 671 private boolean mIsImageValid; 672 673 private synchronized native ByteBuffer nativeImageGetBuffer(int idx); 674 private synchronized native SurfacePlane nativeCreatePlane(int idx); 675 } 676 677 private synchronized native void nativeInit(Object weakSelf, int w, int h, 678 int fmt, int maxImgs); 679 private synchronized native void nativeClose(); 680 private synchronized native void nativeReleaseImage(Image i); 681 private synchronized native Surface nativeGetSurface(); 682 683 /** 684 * @return A return code {@code ACQUIRE_*} 685 * 686 * @see #ACQUIRE_SUCCESS 687 * @see #ACQUIRE_NO_BUFS 688 * @see #ACQUIRE_MAX_IMAGES 689 */ 690 private synchronized native int nativeImageSetup(Image i); 691 692 /** 693 * We use a class initializer to allow the native code to cache some 694 * field offsets. 695 */ 696 private static native void nativeClassInit(); 697 static { 698 System.loadLibrary("media_jni"); 699 nativeClassInit(); 700 } 701} 702