ImageReader.java revision 708e3595031fa15f4ac26c5675a53c1ed495b895
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.media; 18 19import android.graphics.ImageFormat; 20import android.graphics.PixelFormat; 21import android.os.Handler; 22import android.os.Looper; 23import android.os.Message; 24import android.view.Surface; 25 26import java.lang.ref.WeakReference; 27import java.nio.ByteBuffer; 28import java.nio.ByteOrder; 29 30/** 31 * <p>The ImageReader class allows direct application access to image data 32 * rendered into a {@link android.view.Surface}</p> 33 * 34 * <p>Several Android media API classes accept Surface objects as targets to 35 * render to, including {@link MediaPlayer}, {@link MediaCodec}, 36 * {@link android.hardware.camera2.CameraDevice}, and 37 * {@link android.renderscript.Allocation RenderScript Allocations}. The image 38 * sizes and formats that can be used with each source vary, and should be 39 * checked in the documentation for the specific API.</p> 40 * 41 * <p>The image data is encapsulated in {@link Image} objects, and multiple such 42 * objects can be accessed at the same time, up to the number specified by the 43 * {@code maxImages} constructor parameter. New images sent to an ImageReader 44 * through its Surface are queued until accessed through the 45 * {@link #getNextImage} call. Due to memory limits, an image source will 46 * eventually stall or drop Images in trying to render to the Surface if the 47 * ImageReader does not obtain and release Images at a rate equal to the 48 * production rate.</p> 49 */ 50public final class ImageReader implements AutoCloseable { 51 52 /** 53 * <p>Create a new reader for images of the desired size and format.</p> 54 * 55 * <p>The maxImages parameter determines the maximum number of {@link Image} 56 * objects that can be be acquired from the ImageReader 57 * simultaneously. Requesting more buffers will use up more memory, so it is 58 * important to use only the minimum number necessary for the use case.</p> 59 * 60 * <p>The valid sizes and formats depend on the source of the image 61 * data.</p> 62 * 63 * @param width the width in pixels of the Images that this reader will 64 * produce. 65 * @param height the height in pixels of the Images that this reader will 66 * produce. 67 * @param format the format of the Image that this reader will produce. This 68 * must be one of the {@link android.graphics.ImageFormat} or 69 * {@link android.graphics.PixelFormat} constants. 70 * @param maxImages the maximum number of images the user will want to 71 * access simultaneously. This should be as small as possible to limit 72 * memory use. Once maxImages Images are obtained by the user, one of them 73 * has to be released before a new Image will become available for access 74 * through getNextImage(). Must be greater than 0. 75 * 76 * @see Image 77 */ 78 public ImageReader(int width, int height, int format, int maxImages) { 79 mWidth = width; 80 mHeight = height; 81 mFormat = format; 82 mMaxImages = maxImages; 83 84 if (width < 1 || height < 1) { 85 throw new IllegalArgumentException( 86 "The image dimensions must be positive"); 87 } 88 if (mMaxImages < 1) { 89 throw new IllegalArgumentException( 90 "Maximum outstanding image count must be at least 1"); 91 } 92 93 mNumPlanes = getNumPlanesFromFormat(); 94 95 nativeInit(new WeakReference<ImageReader>(this), width, height, format, maxImages); 96 97 mSurface = nativeGetSurface(); 98 } 99 100 public int getWidth() { 101 return mWidth; 102 } 103 104 public int getHeight() { 105 return mHeight; 106 } 107 108 public int getImageFormat() { 109 return mFormat; 110 } 111 112 public int getMaxImages() { 113 return mMaxImages; 114 } 115 116 /** 117 * <p>Get a Surface that can be used to produce Images for this 118 * ImageReader.</p> 119 * 120 * <p>Until valid image data is rendered into this Surface, the 121 * {@link #getNextImage} method will return {@code null}. Only one source 122 * can be producing data into this Surface at the same time, although the 123 * same Surface can be reused with a different API once the first source is 124 * disconnected from the Surface.</p> 125 * 126 * @return A Surface to use for a drawing target for various APIs. 127 */ 128 public Surface getSurface() { 129 return mSurface; 130 } 131 132 /** 133 * <p>Get the next Image from the ImageReader's queue. Returns {@code null} 134 * if no new image is available.</p> 135 * 136 * @return a new frame of image data, or {@code null} if no image data is 137 * available. 138 */ 139 public Image getNextImage() { 140 SurfaceImage si = new SurfaceImage(); 141 if (nativeImageSetup(si)) { 142 // create SurfacePlane objects 143 si.createSurfacePlanes(); 144 si.setImageValid(true); 145 return si; 146 } 147 return null; 148 } 149 150 /** 151 * <p>Return the frame to the ImageReader for reuse.</p> 152 */ 153 public void releaseImage(Image i) { 154 if (! (i instanceof SurfaceImage) ) { 155 throw new IllegalArgumentException( 156 "This image was not produced by an ImageReader"); 157 } 158 SurfaceImage si = (SurfaceImage) i; 159 if (si.getReader() != this) { 160 throw new IllegalArgumentException( 161 "This image was not produced by this ImageReader"); 162 } 163 164 si.clearSurfacePlanes(); 165 nativeReleaseImage(i); 166 si.setImageValid(false); 167 } 168 169 /** 170 * Register a listener to be invoked when a new image becomes available 171 * from the ImageReader. 172 * @param listener the listener that will be run 173 * @param handler The handler on which the listener should be invoked, or null 174 * if the listener should be invoked on the calling thread's looper. 175 */ 176 public void setImageAvailableListener(OnImageAvailableListener listener, Handler handler) { 177 mImageListener = listener; 178 179 Looper looper; 180 mHandler = handler; 181 if (mHandler == null) { 182 if ((looper = Looper.myLooper()) != null) { 183 mHandler = new Handler(); 184 } else { 185 throw new IllegalArgumentException( 186 "Looper doesn't exist in the calling thread"); 187 } 188 } 189 } 190 191 /** 192 * Callback interface for being notified that a new image is available. 193 * The onImageAvailable is called per image basis, that is, callback fires for every new frame 194 * available from ImageReader. 195 */ 196 public interface OnImageAvailableListener { 197 /** 198 * Callback that is called when a new image is available from ImageReader. 199 * @param reader the ImageReader the callback is associated with. 200 * @see ImageReader 201 * @see Image 202 */ 203 void onImageAvailable(ImageReader reader); 204 } 205 206 /** 207 * Free up all the resources associated with this ImageReader. After 208 * Calling this method, this ImageReader can not be used. calling 209 * any methods on this ImageReader and Images previously provided by {@link #getNextImage} 210 * will result in an IllegalStateException, and attempting to read from 211 * ByteBuffers returned by an earlier {@code Plane#getBuffer} call will 212 * have undefined behavior. 213 */ 214 @Override 215 public void close() { 216 nativeClose(); 217 } 218 219 @Override 220 protected void finalize() throws Throwable { 221 try { 222 close(); 223 } finally { 224 super.finalize(); 225 } 226 } 227 228 /** 229 * Only a subset of the formats defined in {@link android.graphics.ImageFormat} and 230 * {@link android.graphics.PixelFormat} are supported by ImageReader. When reading RGB 231 * data from a surface, the formats defined in {@link android.graphics.PixelFormat} 232 * can be used, when reading YUV, JPEG or raw sensor data ( for example, from camera 233 * or video decoder), formats from {@link android.graphics.ImageFormat} are used. 234 */ 235 private int getNumPlanesFromFormat() { 236 switch (mFormat) { 237 case ImageFormat.YV12: 238 case ImageFormat.YUV_420_888: 239 case ImageFormat.NV21: 240 return 3; 241 case ImageFormat.NV16: 242 return 2; 243 case PixelFormat.RGB_565: 244 case PixelFormat.RGBA_8888: 245 case PixelFormat.RGBX_8888: 246 case PixelFormat.RGB_888: 247 case ImageFormat.JPEG: 248 case ImageFormat.YUY2: 249 case ImageFormat.Y8: 250 case ImageFormat.Y16: 251 case ImageFormat.RAW_SENSOR: 252 return 1; 253 default: 254 throw new UnsupportedOperationException( 255 String.format("Invalid format specified %d", mFormat)); 256 } 257 } 258 259 /** 260 * Called from Native code when an Event happens. 261 */ 262 private static void postEventFromNative(Object selfRef) { 263 WeakReference weakSelf = (WeakReference)selfRef; 264 final ImageReader ir = (ImageReader)weakSelf.get(); 265 if (ir == null) { 266 return; 267 } 268 269 if (ir.mHandler != null) { 270 ir.mHandler.post(new Runnable() { 271 @Override 272 public void run() { 273 ir.mImageListener.onImageAvailable(ir); 274 } 275 }); 276 } 277 } 278 279 private final int mWidth; 280 private final int mHeight; 281 private final int mFormat; 282 private final int mMaxImages; 283 private final int mNumPlanes; 284 private final Surface mSurface; 285 286 private Handler mHandler; 287 private OnImageAvailableListener mImageListener; 288 289 /** 290 * This field is used by native code, do not access or modify. 291 */ 292 private long mNativeContext; 293 294 private class SurfaceImage implements android.media.Image { 295 public SurfaceImage() { 296 mIsImageValid = false; 297 } 298 299 @Override 300 public void close() { 301 if (mIsImageValid) { 302 ImageReader.this.releaseImage(this); 303 } 304 } 305 306 public ImageReader getReader() { 307 return ImageReader.this; 308 } 309 310 @Override 311 public int getFormat() { 312 if (mIsImageValid) { 313 return ImageReader.this.mFormat; 314 } else { 315 throw new IllegalStateException("Image is already released"); 316 } 317 } 318 319 @Override 320 public int getWidth() { 321 if (mIsImageValid) { 322 return ImageReader.this.mWidth; 323 } else { 324 throw new IllegalStateException("Image is already released"); 325 } 326 } 327 328 @Override 329 public int getHeight() { 330 if (mIsImageValid) { 331 return ImageReader.this.mHeight; 332 } else { 333 throw new IllegalStateException("Image is already released"); 334 } 335 } 336 337 @Override 338 public long getTimestamp() { 339 if (mIsImageValid) { 340 return mTimestamp; 341 } else { 342 throw new IllegalStateException("Image is already released"); 343 } 344 } 345 346 @Override 347 public Plane[] getPlanes() { 348 if (mIsImageValid) { 349 // Shallow copy is fine. 350 return mPlanes.clone(); 351 } else { 352 throw new IllegalStateException("Image is already released"); 353 } 354 } 355 356 @Override 357 protected final void finalize() throws Throwable { 358 try { 359 close(); 360 } finally { 361 super.finalize(); 362 } 363 } 364 365 private void setImageValid(boolean isValid) { 366 mIsImageValid = isValid; 367 } 368 369 private boolean isImageValid() { 370 return mIsImageValid; 371 } 372 373 private void clearSurfacePlanes() { 374 if (mIsImageValid) { 375 for (int i = 0; i < mPlanes.length; i++) { 376 if (mPlanes[i] != null) { 377 mPlanes[i].clearBuffer(); 378 mPlanes[i] = null; 379 } 380 } 381 } 382 } 383 384 private void createSurfacePlanes() { 385 mPlanes = new SurfacePlane[ImageReader.this.mNumPlanes]; 386 for (int i = 0; i < ImageReader.this.mNumPlanes; i++) { 387 mPlanes[i] = nativeCreatePlane(i); 388 } 389 } 390 private class SurfacePlane implements android.media.Image.Plane { 391 // SurfacePlane instance is created by native code when a new SurfaceImage is created 392 private SurfacePlane(int index, int rowStride, int pixelStride) { 393 mIndex = index; 394 mRowStride = rowStride; 395 mPixelStride = pixelStride; 396 } 397 398 @Override 399 public ByteBuffer getBuffer() { 400 if (SurfaceImage.this.isImageValid() == false) { 401 throw new IllegalStateException("Image is already released"); 402 } 403 if (mBuffer != null) { 404 return mBuffer; 405 } else { 406 mBuffer = SurfaceImage.this.nativeImageGetBuffer(mIndex); 407 // Set the byteBuffer order according to host endianness (native order), 408 // otherwise, the byteBuffer order defaults to ByteOrder.BIG_ENDIAN. 409 return mBuffer.order(ByteOrder.nativeOrder()); 410 } 411 } 412 413 @Override 414 public int getPixelStride() { 415 if (SurfaceImage.this.isImageValid()) { 416 return mPixelStride; 417 } else { 418 throw new IllegalStateException("Image is already released"); 419 } 420 } 421 422 @Override 423 public int getRowStride() { 424 if (SurfaceImage.this.isImageValid()) { 425 return mRowStride; 426 } else { 427 throw new IllegalStateException("Image is already released"); 428 } 429 } 430 431 private void clearBuffer() { 432 mBuffer = null; 433 } 434 435 final private int mIndex; 436 final private int mPixelStride; 437 final private int mRowStride; 438 439 private ByteBuffer mBuffer; 440 } 441 442 /** 443 * This field is used to keep track of native object and used by native code only. 444 * Don't modify. 445 */ 446 private long mLockedBuffer; 447 448 /** 449 * This field is set by native code during nativeImageSetup(). 450 */ 451 private long mTimestamp; 452 453 private SurfacePlane[] mPlanes; 454 private boolean mIsImageValid; 455 456 private synchronized native ByteBuffer nativeImageGetBuffer(int idx); 457 private synchronized native SurfacePlane nativeCreatePlane(int idx); 458 } 459 460 private synchronized native void nativeInit(Object weakSelf, int w, int h, 461 int fmt, int maxImgs); 462 private synchronized native void nativeClose(); 463 private synchronized native void nativeReleaseImage(Image i); 464 private synchronized native Surface nativeGetSurface(); 465 private synchronized native boolean nativeImageSetup(Image i); 466 467 /* 468 * We use a class initializer to allow the native code to cache some 469 * field offsets. 470 */ 471 private static native void nativeClassInit(); 472 static { 473 System.loadLibrary("media_jni"); 474 nativeClassInit(); 475 } 476} 477