MediaCodec.java revision 68b1a6eed8479e16456f018663b1bcfaecc2a3f4
1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.media; 18 19import android.media.MediaCrypto; 20import android.media.MediaFormat; 21import android.view.Surface; 22import java.nio.ByteBuffer; 23import java.util.Map; 24 25/** 26 * MediaCodec class can be used to access low-level media codec, i.e. 27 * encoder/decoder components. 28 * 29 * <p>MediaCodec is generally used like this: 30 * <pre> 31 * MediaCodec codec = MediaCodec.createDecoderByType(type); 32 * codec.configure(format, ...); 33 * codec.start(); 34 * ByteBuffer[] inputBuffers = codec.getInputBuffers(); 35 * ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 36 * Map<String, Object> format = codec.getOutputFormat(); 37 * for (;;) { 38 * int inputBufferIndex = codec.dequeueInputBuffer(timeoutUs); 39 * if (inputBufferIndex >= 0) { 40 * // fill inputBuffers[inputBufferIndex] with valid data 41 * ... 42 * codec.queueInputBuffer(inputBufferIndex, ...); 43 * } 44 * 45 * int outputBufferIndex = codec.dequeueOutputBuffer(timeoutUs); 46 * if (outputBufferIndex >= 0) { 47 * // outputBuffer is ready to be processed or rendered. 48 * ... 49 * codec.releaseOutputBuffer(outputBufferIndex, ...); 50 * } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 51 * outputBuffers = codec.getOutputBuffers(); 52 * } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 53 * // Subsequent data will conform to new format. 54 * format = codec.getOutputFormat(); 55 * ... 56 * } 57 * } 58 * codec.stop(); 59 * codec.release(); 60 * codec = null; 61 * </pre> 62 * 63 * Each codec maintains a number of input and output buffers that are 64 * referred to by index in API calls. 65 * The contents of these buffers is represented by the ByteBuffer[] arrays 66 * accessible through getInputBuffers() and getOutputBuffers(). 67 * 68 * After a successful call to {@link #start} the client "owns" neither 69 * input nor output buffers, subsequent calls to {@link #dequeueInputBuffer} 70 * and {@link #dequeueOutputBuffer} then transfer ownership from the codec 71 * to the client.<p> 72 * The client is not required to resubmit/release buffers immediately 73 * to the codec, the sample code above simply does this for simplicity's sake.<p> 74 * Once the client has an input buffer available it can fill it with data 75 * and submit it it to the codec via a call to {@link #queueInputBuffer}.<p> 76 * The codec in turn will return an output buffer to the client in response 77 * to {@link #dequeueOutputBuffer}. After the output buffer has been processed 78 * a call to {@link #releaseOutputBuffer} will return it to the codec. 79 * If a video surface has been provided in the call to {@link #configure}, 80 * {@link #releaseOutputBuffer} optionally allows rendering of the buffer 81 * to the surface.<p> 82 * 83 * Input buffers (for decoders) and Output buffers (for encoders) contain 84 * encoded data according to the format's type. For video types this data 85 * is all the encoded data representing a single moment in time, for audio 86 * data this is slightly relaxed in that a buffer may contain multiple 87 * encoded frames of audio. In either case, buffers do not start and end on 88 * arbitrary byte boundaries, this is not a stream of bytes, it's a stream 89 * of access units.<p> 90 * 91 * Most formats also require the actual data to be prefixed by a number 92 * of buffers containing setup data, or codec specific data, i.e. the 93 * first few buffers submitted to the codec object after starting it must 94 * be codec specific data marked as such using the flag {@link #BUFFER_FLAG_CODEC_CONFIG} 95 * in a call to {@link #queueInputBuffer}. 96 * 97 * Codec specific data included in the format passed to {@link #configure} 98 * (in ByteBuffer entries with keys "csd-0", "csd-1", ...) is automatically 99 * submitted to the codec, this data MUST NOT be submitted explicitly by the 100 * client. 101 * 102 * Once the client reaches the end of the input data it signals the end of 103 * the input stream by specifying a flag of {@link #BUFFER_FLAG_END_OF_STREAM} in the call to 104 * {@link #queueInputBuffer}. The codec will continue to return output buffers 105 * until it eventually signals the end of the output stream by specifying 106 * the same flag ({@link #BUFFER_FLAG_END_OF_STREAM}) on the BufferInfo returned in 107 * {@link #dequeueOutputBuffer}. 108 * 109 * In order to start decoding data that's not adjacent to previously submitted 110 * data (i.e. after a seek) it is necessary to {@link #flush} the decoder. 111 * Any input or output buffers the client may own at the point of the flush are 112 * immediately revoked, i.e. after a call to {@link #flush} the client does not 113 * own any buffers anymore. 114 * Note that the format of the data submitted after a flush must not change, 115 * flush does not support format discontinuities, 116 * for this a full stop(), configure(), start() cycle is necessary. 117 * 118 */ 119final public class MediaCodec { 120 /** 121 * Per buffer metadata includes an offset and size specifying 122 * the range of valid data in the associated codec buffer. 123 */ 124 public final static class BufferInfo { 125 public void set( 126 int newOffset, int newSize, long newTimeUs, int newFlags) { 127 offset = newOffset; 128 size = newSize; 129 presentationTimeUs = newTimeUs; 130 flags = newFlags; 131 } 132 133 public int offset; 134 public int size; 135 public long presentationTimeUs; 136 public int flags; 137 }; 138 139 // The follow flag constants MUST stay in sync with their equivalents 140 // in MediaCodec.h ! 141 142 /** 143 * This indicates that the buffer marked as such contains the data 144 * for a sync frame. 145 */ 146 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 147 148 /** 149 * This indicated that the buffer marked as such contains codec 150 * initialization / codec specific data instead of media data. 151 */ 152 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 153 154 /** 155 * This signals the end of stream, i.e. no buffers will be available 156 * after this, unless of course, {@link #flush} follows. 157 */ 158 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 159 160 /** 161 * Instantiate a decoder supporting input data of the given mime type. 162 * 163 * The following is a partial list of defined mime types and their semantics: 164 * <ul> 165 * <li>"video/x-vnd.on2.vp8" - VPX video (i.e. video in .webm) 166 * <li>"video/avc" - H.264/AVC video 167 * <li>"video/mp4v-es" - MPEG4 video 168 * <li>"video/3gpp" - H.263 video 169 * <li>"audio/3gpp" - AMR narrowband audio 170 * <li>"audio/amr-wb" - AMR wideband audio 171 * <li>"audio/mpeg" - MPEG1/2 audio layer III 172 * <li>"audio/mp4a-latm" - AAC audio 173 * <li>"audio/vorbis" - vorbis audio 174 * <li>"audio/g711-alaw" - G.711 alaw audio 175 * <li>"audio/g711-mlaw" - G.711 ulaw audio 176 * </ul> 177 * 178 * @param type The mime type of the input data. 179 */ 180 public static MediaCodec createDecoderByType(String type) { 181 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 182 } 183 184 /** 185 * Instantiate an encoder supporting output data of the given mime type. 186 * @param type The desired mime type of the output data. 187 */ 188 public static MediaCodec createEncoderByType(String type) { 189 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 190 } 191 192 /** 193 * If you know the exact name of the component you want to instantiate 194 * use this method to instantiate it. Use with caution. 195 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 196 * @param name The name of the codec to be instantiated. 197 */ 198 public static MediaCodec createByCodecName(String name) { 199 return new MediaCodec( 200 name, false /* nameIsType */, false /* unused */); 201 } 202 203 private MediaCodec( 204 String name, boolean nameIsType, boolean encoder) { 205 native_setup(name, nameIsType, encoder); 206 } 207 208 @Override 209 protected void finalize() { 210 native_finalize(); 211 } 212 213 /** 214 * Make sure you call this when you're done to free up any opened 215 * component instance instead of relying on the garbage collector 216 * to do this for you at some point in the future. 217 */ 218 public native final void release(); 219 220 /** 221 * If this codec is to be used as an encoder, pass this flag. 222 */ 223 public static final int CONFIGURE_FLAG_ENCODE = 1; 224 225 /** 226 * Configures a component. 227 * 228 * @param format The format of the input data (decoder) or the desired 229 * format of the output data (encoder). 230 * @param surface Specify a surface on which to render the output of this 231 * decoder. 232 * @param crypto Specify a crypto object to facilitate secure decryption 233 * of the media data. 234 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 235 * component as an encoder. 236 */ 237 public void configure( 238 MediaFormat format, 239 Surface surface, MediaCrypto crypto, int flags) { 240 Map<String, Object> formatMap = format.getMap(); 241 242 String[] keys = null; 243 Object[] values = null; 244 245 if (format != null) { 246 keys = new String[formatMap.size()]; 247 values = new Object[formatMap.size()]; 248 249 int i = 0; 250 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 251 keys[i] = entry.getKey(); 252 values[i] = entry.getValue(); 253 ++i; 254 } 255 } 256 257 native_configure(keys, values, surface, crypto, flags); 258 } 259 260 private native final void native_configure( 261 String[] keys, Object[] values, 262 Surface surface, MediaCrypto crypto, int flags); 263 264 /** 265 * After successfully configuring the component, call start. On return 266 * you can query the component for its input/output buffers. 267 */ 268 public native final void start(); 269 270 /** 271 * Finish the decode/encode session, note that the codec instance 272 * remains active and ready to be {@link #start}ed again. 273 * To ensure that it is available to other client call {@link #release} 274 * and don't just rely on garbage collection to eventually do this for you. 275 */ 276 public native final void stop(); 277 278 /** 279 * Flush both input and output ports of the component, all indices 280 * previously returned in calls to {@link #dequeueInputBuffer} and 281 * {@link #dequeueOutputBuffer} become invalid. 282 */ 283 public native final void flush(); 284 285 public final static class CryptoException extends RuntimeException { 286 public CryptoException(int errorCode, String detailMessage) { 287 super(detailMessage); 288 mErrorCode = errorCode; 289 } 290 291 public int getErrorCode() { 292 return mErrorCode; 293 } 294 295 private int mErrorCode; 296 } 297 298 /** 299 * After filling a range of the input buffer at the specified index 300 * submit it to the component. 301 * 302 * Many decoders require the actual compressed data stream to be 303 * preceded by "codec specific data", i.e. setup data used to initialize 304 * the codec such as PPS/SPS in the case of AVC video or code tables 305 * in the case of vorbis audio. 306 * The class {@link android.media.MediaExtractor} provides codec 307 * specific data as part of 308 * the returned track format in entries named "csd-0", "csd-1" ... 309 * 310 * These buffers should be submitted using the flag {@link #BUFFER_FLAG_CODEC_CONFIG}. 311 * 312 * To indicate that this is the final piece of input data (or rather that 313 * no more input data follows unless the decoder is subsequently flushed) 314 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 315 * 316 * @param index The index of a client-owned input buffer previously returned 317 * in a call to {@link #dequeueInputBuffer}. 318 * @param offset The byte offset into the input buffer at which the data starts. 319 * @param size The number of bytes of valid input data. 320 * @param presentationTimeUs The time at which this buffer should be rendered. 321 * @param flags A bitmask of flags {@link #BUFFER_FLAG_SYNC_FRAME}, 322 * {@link #BUFFER_FLAG_CODEC_CONFIG} or {@link #BUFFER_FLAG_END_OF_STREAM}. 323 * @throws CryptoException if a crypto object has been specified in 324 * {@link #configure} 325 */ 326 public native final void queueInputBuffer( 327 int index, 328 int offset, int size, long presentationTimeUs, int flags) 329 throws CryptoException; 330 331 // The following mode constants MUST stay in sync with their equivalents 332 // in media/hardware/CryptoAPI.h ! 333 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 334 public static final int CRYPTO_MODE_AES_CTR = 1; 335 336 /** 337 * Metadata describing the structure of a (at least partially) encrypted 338 * input sample. 339 * A buffer's data is considered to be partitioned into "subSamples", 340 * each subSample starts with a (potentially empty) run of plain, 341 * unencrypted bytes followed by a (also potentially empty) run of 342 * encrypted bytes. 343 * numBytesOfClearData can be null to indicate that all data is encrypted. 344 * This information encapsulates per-sample metadata as outlined in 345 * ISO/IEC FDIS 23001-7:2011 "Common encryption in ISO base media file format files". 346 */ 347 public final static class CryptoInfo { 348 public void set( 349 int newNumSubSamples, 350 int[] newNumBytesOfClearData, 351 int[] newNumBytesOfEncryptedData, 352 byte[] newKey, 353 byte[] newIV, 354 int newMode) { 355 numSubSamples = newNumSubSamples; 356 numBytesOfClearData = newNumBytesOfClearData; 357 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 358 key = newKey; 359 iv = newIV; 360 mode = newMode; 361 } 362 363 /** 364 * The number of subSamples that make up the buffer's contents. 365 */ 366 public int numSubSamples; 367 /** 368 * The number of leading unencrypted bytes in each subSample. 369 */ 370 public int[] numBytesOfClearData; 371 /** 372 * The number of trailing encrypted bytes in each subSample. 373 */ 374 public int[] numBytesOfEncryptedData; 375 /** 376 * A 16-byte opaque key 377 */ 378 public byte[] key; 379 /** 380 * A 16-byte initialization vector 381 */ 382 public byte[] iv; 383 /** 384 * The type of encryption that has been applied, 385 * see {@link #CRYPTO_MODE_UNENCRYPTED} and {@link #CRYPTO_MODE_AES_CTR}. 386 */ 387 public int mode; 388 }; 389 390 /** 391 * Similar to {@link #queueInputBuffer} but submits a buffer that is 392 * potentially encrypted. 393 * @param index The index of a client-owned input buffer previously returned 394 * in a call to {@link #dequeueInputBuffer}. 395 * @param offset The byte offset into the input buffer at which the data starts. 396 * @param info Metadata required to facilitate decryption, the object can be 397 * reused immediately after this call returns. 398 * @param presentationTimeUs The time at which this buffer should be rendered. 399 * @param flags A bitmask of flags {@link #BUFFER_FLAG_SYNC_FRAME}, 400 * {@link #BUFFER_FLAG_CODEC_CONFIG} or {@link #BUFFER_FLAG_END_OF_STREAM}. 401 */ 402 public native final void queueSecureInputBuffer( 403 int index, 404 int offset, 405 CryptoInfo info, 406 long presentationTimeUs, 407 int flags) throws CryptoException; 408 409 /** 410 * Returns the index of an input buffer to be filled with valid data 411 * or -1 if no such buffer is currently available. 412 * This method will return immediately if timeoutUs == 0, wait indefinitely 413 * for the availability of an input buffer if timeoutUs < 0 or wait up 414 * to "timeoutUs" microseconds if timeoutUs > 0. 415 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 416 */ 417 public native final int dequeueInputBuffer(long timeoutUs); 418 419 /** 420 * If a non-negative timeout had been specified in the call 421 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 422 */ 423 public static final int INFO_TRY_AGAIN_LATER = -1; 424 425 /** 426 * The output format has changed, subsequent data will follow the new 427 * format. {@link #getOutputFormat} returns the new format. 428 */ 429 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 430 431 /** 432 * The output buffers have changed, the client must refer to the new 433 * set of output buffers returned by {@link #getOutputBuffers} from 434 * this point on. 435 */ 436 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 437 438 /** 439 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 440 * Returns the index of an output buffer that has been successfully 441 * decoded or one of the INFO_* constants below. 442 * @param info Will be filled with buffer meta data. 443 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 444 */ 445 public native final int dequeueOutputBuffer( 446 BufferInfo info, long timeoutUs); 447 448 /** 449 * If you are done with a buffer, use this call to return the buffer to 450 * the codec. If you previously specified a surface when configuring this 451 * video decoder you can optionally render the buffer. 452 * @param index The index of a client-owned output buffer previously returned 453 * in a call to {@link #dequeueOutputBuffer}. 454 * @param render If a valid surface was specified when configuring the codec, 455 * passing true renders this output buffer to the surface. 456 */ 457 public native final void releaseOutputBuffer(int index, boolean render); 458 459 /** 460 * Call this after dequeueOutputBuffer signals a format change by returning 461 * {@link #INFO_OUTPUT_FORMAT_CHANGED} 462 */ 463 public final MediaFormat getOutputFormat() { 464 return new MediaFormat(getOutputFormatNative()); 465 } 466 467 private native final Map<String, Object> getOutputFormatNative(); 468 469 /** 470 * Call this after start() returns. 471 */ 472 public ByteBuffer[] getInputBuffers() { 473 return getBuffers(true /* input */); 474 } 475 476 /** 477 * Call this after start() returns and whenever dequeueOutputBuffer 478 * signals an output buffer change by returning 479 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} 480 */ 481 public ByteBuffer[] getOutputBuffers() { 482 return getBuffers(false /* input */); 483 } 484 485 /** 486 * The content is scaled to the surface dimensions 487 */ 488 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 489 490 /** 491 * The content is scaled, maintaining its aspect ratio, the whole 492 * surface area is used, content may be cropped 493 */ 494 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 495 496 /** 497 * If a surface has been specified in a previous call to {@link #configure} 498 * specifies the scaling mode to use. The default is "scale to fit". 499 */ 500 public native final void setVideoScalingMode(int mode); 501 502 private native final ByteBuffer[] getBuffers(boolean input); 503 504 private static native final void native_init(); 505 506 private native final void native_setup( 507 String name, boolean nameIsType, boolean encoder); 508 509 private native final void native_finalize(); 510 511 static { 512 System.loadLibrary("media_jni"); 513 native_init(); 514 } 515 516 private int mNativeContext; 517} 518