1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.media; 18 19import android.media.MediaCrypto; 20import android.media.MediaFormat; 21import android.view.Surface; 22import java.nio.ByteBuffer; 23import java.util.Map; 24 25/** 26 * MediaCodec class can be used to access low-level media codec, i.e. 27 * encoder/decoder components. 28 * 29 * <p>MediaCodec is generally used like this: 30 * <pre> 31 * MediaCodec codec = MediaCodec.createDecoderByType(type); 32 * codec.configure(format, ...); 33 * codec.start(); 34 * ByteBuffer[] inputBuffers = codec.getInputBuffers(); 35 * ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 36 * for (;;) { 37 * int inputBufferIndex = codec.dequeueInputBuffer(timeoutUs); 38 * if (inputBufferIndex >= 0) { 39 * // fill inputBuffers[inputBufferIndex] with valid data 40 * ... 41 * codec.queueInputBuffer(inputBufferIndex, ...); 42 * } 43 * 44 * int outputBufferIndex = codec.dequeueOutputBuffer(timeoutUs); 45 * if (outputBufferIndex >= 0) { 46 * // outputBuffer is ready to be processed or rendered. 47 * ... 48 * codec.releaseOutputBuffer(outputBufferIndex, ...); 49 * } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 50 * outputBuffers = codec.getOutputBuffers(); 51 * } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 52 * // Subsequent data will conform to new format. 53 * MediaFormat format = codec.getOutputFormat(); 54 * ... 55 * } 56 * } 57 * codec.stop(); 58 * codec.release(); 59 * codec = null; 60 * </pre> 61 * 62 * Each codec maintains a number of input and output buffers that are 63 * referred to by index in API calls. 64 * The contents of these buffers is represented by the ByteBuffer[] arrays 65 * accessible through getInputBuffers() and getOutputBuffers(). 66 * 67 * After a successful call to {@link #start} the client "owns" neither 68 * input nor output buffers, subsequent calls to {@link #dequeueInputBuffer} 69 * and {@link #dequeueOutputBuffer} then transfer ownership from the codec 70 * to the client.<p> 71 * The client is not required to resubmit/release buffers immediately 72 * to the codec, the sample code above simply does this for simplicity's sake.<p> 73 * Once the client has an input buffer available it can fill it with data 74 * and submit it it to the codec via a call to {@link #queueInputBuffer}.<p> 75 * The codec in turn will return an output buffer to the client in response 76 * to {@link #dequeueOutputBuffer}. After the output buffer has been processed 77 * a call to {@link #releaseOutputBuffer} will return it to the codec. 78 * If a video surface has been provided in the call to {@link #configure}, 79 * {@link #releaseOutputBuffer} optionally allows rendering of the buffer 80 * to the surface.<p> 81 * 82 * Input buffers (for decoders) and Output buffers (for encoders) contain 83 * encoded data according to the format's type. For video types this data 84 * is all the encoded data representing a single moment in time, for audio 85 * data this is slightly relaxed in that a buffer may contain multiple 86 * encoded frames of audio. In either case, buffers do not start and end on 87 * arbitrary byte boundaries, this is not a stream of bytes, it's a stream 88 * of access units.<p> 89 * 90 * Most formats also require the actual data to be prefixed by a number 91 * of buffers containing setup data, or codec specific data, i.e. the 92 * first few buffers submitted to the codec object after starting it must 93 * be codec specific data marked as such using the flag {@link #BUFFER_FLAG_CODEC_CONFIG} 94 * in a call to {@link #queueInputBuffer}. 95 * 96 * Codec specific data included in the format passed to {@link #configure} 97 * (in ByteBuffer entries with keys "csd-0", "csd-1", ...) is automatically 98 * submitted to the codec, this data MUST NOT be submitted explicitly by the 99 * client. 100 * 101 * Once the client reaches the end of the input data it signals the end of 102 * the input stream by specifying a flag of {@link #BUFFER_FLAG_END_OF_STREAM} in the call to 103 * {@link #queueInputBuffer}. The codec will continue to return output buffers 104 * until it eventually signals the end of the output stream by specifying 105 * the same flag ({@link #BUFFER_FLAG_END_OF_STREAM}) on the BufferInfo returned in 106 * {@link #dequeueOutputBuffer}. 107 * 108 * In order to start decoding data that's not adjacent to previously submitted 109 * data (i.e. after a seek) it is necessary to {@link #flush} the decoder. 110 * Any input or output buffers the client may own at the point of the flush are 111 * immediately revoked, i.e. after a call to {@link #flush} the client does not 112 * own any buffers anymore. 113 * Note that the format of the data submitted after a flush must not change, 114 * flush does not support format discontinuities, 115 * for this a full stop(), configure(), start() cycle is necessary. 116 * 117 */ 118final public class MediaCodec { 119 /** 120 * Per buffer metadata includes an offset and size specifying 121 * the range of valid data in the associated codec buffer. 122 */ 123 public final static class BufferInfo { 124 public void set( 125 int newOffset, int newSize, long newTimeUs, int newFlags) { 126 offset = newOffset; 127 size = newSize; 128 presentationTimeUs = newTimeUs; 129 flags = newFlags; 130 } 131 132 public int offset; 133 public int size; 134 public long presentationTimeUs; 135 public int flags; 136 }; 137 138 // The follow flag constants MUST stay in sync with their equivalents 139 // in MediaCodec.h ! 140 141 /** 142 * This indicates that the buffer marked as such contains the data 143 * for a sync frame. 144 */ 145 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 146 147 /** 148 * This indicated that the buffer marked as such contains codec 149 * initialization / codec specific data instead of media data. 150 */ 151 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 152 153 /** 154 * This signals the end of stream, i.e. no buffers will be available 155 * after this, unless of course, {@link #flush} follows. 156 */ 157 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 158 159 /** 160 * Instantiate a decoder supporting input data of the given mime type. 161 * 162 * The following is a partial list of defined mime types and their semantics: 163 * <ul> 164 * <li>"video/x-vnd.on2.vp8" - VPX video (i.e. video in .webm) 165 * <li>"video/avc" - H.264/AVC video 166 * <li>"video/mp4v-es" - MPEG4 video 167 * <li>"video/3gpp" - H.263 video 168 * <li>"audio/3gpp" - AMR narrowband audio 169 * <li>"audio/amr-wb" - AMR wideband audio 170 * <li>"audio/mpeg" - MPEG1/2 audio layer III 171 * <li>"audio/mp4a-latm" - AAC audio 172 * <li>"audio/vorbis" - vorbis audio 173 * <li>"audio/g711-alaw" - G.711 alaw audio 174 * <li>"audio/g711-mlaw" - G.711 ulaw audio 175 * </ul> 176 * 177 * @param type The mime type of the input data. 178 */ 179 public static MediaCodec createDecoderByType(String type) { 180 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 181 } 182 183 /** 184 * Instantiate an encoder supporting output data of the given mime type. 185 * @param type The desired mime type of the output data. 186 */ 187 public static MediaCodec createEncoderByType(String type) { 188 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 189 } 190 191 /** 192 * If you know the exact name of the component you want to instantiate 193 * use this method to instantiate it. Use with caution. 194 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 195 * @param name The name of the codec to be instantiated. 196 */ 197 public static MediaCodec createByCodecName(String name) { 198 return new MediaCodec( 199 name, false /* nameIsType */, false /* unused */); 200 } 201 202 private MediaCodec( 203 String name, boolean nameIsType, boolean encoder) { 204 native_setup(name, nameIsType, encoder); 205 } 206 207 @Override 208 protected void finalize() { 209 native_finalize(); 210 } 211 212 /** 213 * Make sure you call this when you're done to free up any opened 214 * component instance instead of relying on the garbage collector 215 * to do this for you at some point in the future. 216 */ 217 public native final void release(); 218 219 /** 220 * If this codec is to be used as an encoder, pass this flag. 221 */ 222 public static final int CONFIGURE_FLAG_ENCODE = 1; 223 224 /** 225 * Configures a component. 226 * 227 * @param format The format of the input data (decoder) or the desired 228 * format of the output data (encoder). 229 * @param surface Specify a surface on which to render the output of this 230 * decoder. 231 * @param crypto Specify a crypto object to facilitate secure decryption 232 * of the media data. 233 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 234 * component as an encoder. 235 */ 236 public void configure( 237 MediaFormat format, 238 Surface surface, MediaCrypto crypto, int flags) { 239 Map<String, Object> formatMap = format.getMap(); 240 241 String[] keys = null; 242 Object[] values = null; 243 244 if (format != null) { 245 keys = new String[formatMap.size()]; 246 values = new Object[formatMap.size()]; 247 248 int i = 0; 249 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 250 keys[i] = entry.getKey(); 251 values[i] = entry.getValue(); 252 ++i; 253 } 254 } 255 256 native_configure(keys, values, surface, crypto, flags); 257 } 258 259 private native final void native_configure( 260 String[] keys, Object[] values, 261 Surface surface, MediaCrypto crypto, int flags); 262 263 /** 264 * After successfully configuring the component, call start. On return 265 * you can query the component for its input/output buffers. 266 */ 267 public native final void start(); 268 269 /** 270 * Finish the decode/encode session, note that the codec instance 271 * remains active and ready to be {@link #start}ed again. 272 * To ensure that it is available to other client call {@link #release} 273 * and don't just rely on garbage collection to eventually do this for you. 274 */ 275 public native final void stop(); 276 277 /** 278 * Flush both input and output ports of the component, all indices 279 * previously returned in calls to {@link #dequeueInputBuffer} and 280 * {@link #dequeueOutputBuffer} become invalid. 281 */ 282 public native final void flush(); 283 284 public final static class CryptoException extends RuntimeException { 285 public CryptoException(int errorCode, String detailMessage) { 286 super(detailMessage); 287 mErrorCode = errorCode; 288 } 289 290 public int getErrorCode() { 291 return mErrorCode; 292 } 293 294 private int mErrorCode; 295 } 296 297 /** 298 * After filling a range of the input buffer at the specified index 299 * submit it to the component. 300 * 301 * Many decoders require the actual compressed data stream to be 302 * preceded by "codec specific data", i.e. setup data used to initialize 303 * the codec such as PPS/SPS in the case of AVC video or code tables 304 * in the case of vorbis audio. 305 * The class {@link android.media.MediaExtractor} provides codec 306 * specific data as part of 307 * the returned track format in entries named "csd-0", "csd-1" ... 308 * 309 * These buffers should be submitted using the flag {@link #BUFFER_FLAG_CODEC_CONFIG}. 310 * 311 * To indicate that this is the final piece of input data (or rather that 312 * no more input data follows unless the decoder is subsequently flushed) 313 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 314 * 315 * @param index The index of a client-owned input buffer previously returned 316 * in a call to {@link #dequeueInputBuffer}. 317 * @param offset The byte offset into the input buffer at which the data starts. 318 * @param size The number of bytes of valid input data. 319 * @param presentationTimeUs The time at which this buffer should be rendered. 320 * @param flags A bitmask of flags {@link #BUFFER_FLAG_SYNC_FRAME}, 321 * {@link #BUFFER_FLAG_CODEC_CONFIG} or {@link #BUFFER_FLAG_END_OF_STREAM}. 322 * @throws CryptoException if a crypto object has been specified in 323 * {@link #configure} 324 */ 325 public native final void queueInputBuffer( 326 int index, 327 int offset, int size, long presentationTimeUs, int flags) 328 throws CryptoException; 329 330 // The following mode constants MUST stay in sync with their equivalents 331 // in media/hardware/CryptoAPI.h ! 332 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 333 public static final int CRYPTO_MODE_AES_CTR = 1; 334 335 /** 336 * Metadata describing the structure of a (at least partially) encrypted 337 * input sample. 338 * A buffer's data is considered to be partitioned into "subSamples", 339 * each subSample starts with a (potentially empty) run of plain, 340 * unencrypted bytes followed by a (also potentially empty) run of 341 * encrypted bytes. 342 * numBytesOfClearData can be null to indicate that all data is encrypted. 343 * This information encapsulates per-sample metadata as outlined in 344 * ISO/IEC FDIS 23001-7:2011 "Common encryption in ISO base media file format files". 345 */ 346 public final static class CryptoInfo { 347 public void set( 348 int newNumSubSamples, 349 int[] newNumBytesOfClearData, 350 int[] newNumBytesOfEncryptedData, 351 byte[] newKey, 352 byte[] newIV, 353 int newMode) { 354 numSubSamples = newNumSubSamples; 355 numBytesOfClearData = newNumBytesOfClearData; 356 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 357 key = newKey; 358 iv = newIV; 359 mode = newMode; 360 } 361 362 /** 363 * The number of subSamples that make up the buffer's contents. 364 */ 365 public int numSubSamples; 366 /** 367 * The number of leading unencrypted bytes in each subSample. 368 */ 369 public int[] numBytesOfClearData; 370 /** 371 * The number of trailing encrypted bytes in each subSample. 372 */ 373 public int[] numBytesOfEncryptedData; 374 /** 375 * A 16-byte opaque key 376 */ 377 public byte[] key; 378 /** 379 * A 16-byte initialization vector 380 */ 381 public byte[] iv; 382 /** 383 * The type of encryption that has been applied, 384 * see {@link #CRYPTO_MODE_UNENCRYPTED} and {@link #CRYPTO_MODE_AES_CTR}. 385 */ 386 public int mode; 387 }; 388 389 /** 390 * Similar to {@link #queueInputBuffer} but submits a buffer that is 391 * potentially encrypted. 392 * @param index The index of a client-owned input buffer previously returned 393 * in a call to {@link #dequeueInputBuffer}. 394 * @param offset The byte offset into the input buffer at which the data starts. 395 * @param info Metadata required to facilitate decryption, the object can be 396 * reused immediately after this call returns. 397 * @param presentationTimeUs The time at which this buffer should be rendered. 398 * @param flags A bitmask of flags {@link #BUFFER_FLAG_SYNC_FRAME}, 399 * {@link #BUFFER_FLAG_CODEC_CONFIG} or {@link #BUFFER_FLAG_END_OF_STREAM}. 400 */ 401 public native final void queueSecureInputBuffer( 402 int index, 403 int offset, 404 CryptoInfo info, 405 long presentationTimeUs, 406 int flags) throws CryptoException; 407 408 /** 409 * Returns the index of an input buffer to be filled with valid data 410 * or -1 if no such buffer is currently available. 411 * This method will return immediately if timeoutUs == 0, wait indefinitely 412 * for the availability of an input buffer if timeoutUs < 0 or wait up 413 * to "timeoutUs" microseconds if timeoutUs > 0. 414 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 415 */ 416 public native final int dequeueInputBuffer(long timeoutUs); 417 418 /** 419 * If a non-negative timeout had been specified in the call 420 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 421 */ 422 public static final int INFO_TRY_AGAIN_LATER = -1; 423 424 /** 425 * The output format has changed, subsequent data will follow the new 426 * format. {@link #getOutputFormat} returns the new format. 427 */ 428 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 429 430 /** 431 * The output buffers have changed, the client must refer to the new 432 * set of output buffers returned by {@link #getOutputBuffers} from 433 * this point on. 434 */ 435 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 436 437 /** 438 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 439 * Returns the index of an output buffer that has been successfully 440 * decoded or one of the INFO_* constants below. 441 * @param info Will be filled with buffer meta data. 442 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 443 */ 444 public native final int dequeueOutputBuffer( 445 BufferInfo info, long timeoutUs); 446 447 /** 448 * If you are done with a buffer, use this call to return the buffer to 449 * the codec. If you previously specified a surface when configuring this 450 * video decoder you can optionally render the buffer. 451 * @param index The index of a client-owned output buffer previously returned 452 * in a call to {@link #dequeueOutputBuffer}. 453 * @param render If a valid surface was specified when configuring the codec, 454 * passing true renders this output buffer to the surface. 455 */ 456 public native final void releaseOutputBuffer(int index, boolean render); 457 458 /** 459 * Call this after dequeueOutputBuffer signals a format change by returning 460 * {@link #INFO_OUTPUT_FORMAT_CHANGED} 461 */ 462 public final MediaFormat getOutputFormat() { 463 return new MediaFormat(getOutputFormatNative()); 464 } 465 466 private native final Map<String, Object> getOutputFormatNative(); 467 468 /** 469 * Call this after start() returns. 470 */ 471 public ByteBuffer[] getInputBuffers() { 472 return getBuffers(true /* input */); 473 } 474 475 /** 476 * Call this after start() returns and whenever dequeueOutputBuffer 477 * signals an output buffer change by returning 478 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} 479 */ 480 public ByteBuffer[] getOutputBuffers() { 481 return getBuffers(false /* input */); 482 } 483 484 /** 485 * The content is scaled to the surface dimensions 486 */ 487 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 488 489 /** 490 * The content is scaled, maintaining its aspect ratio, the whole 491 * surface area is used, content may be cropped 492 */ 493 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 494 495 /** 496 * If a surface has been specified in a previous call to {@link #configure} 497 * specifies the scaling mode to use. The default is "scale to fit". 498 */ 499 public native final void setVideoScalingMode(int mode); 500 501 private native final ByteBuffer[] getBuffers(boolean input); 502 503 private static native final void native_init(); 504 505 private native final void native_setup( 506 String name, boolean nameIsType, boolean encoder); 507 508 private native final void native_finalize(); 509 510 static { 511 System.loadLibrary("media_jni"); 512 native_init(); 513 } 514 515 private int mNativeContext; 516} 517