MediaCodec.java revision 7de28d34f1ca3a727a8325cf3304f2fe03d2ac59
1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.media; 18 19import android.graphics.ImageFormat; 20import android.graphics.Rect; 21import android.media.Image; 22import android.media.Image.Plane; 23import android.media.MediaCodecInfo; 24import android.media.MediaCodecList; 25import android.media.MediaCrypto; 26import android.media.MediaFormat; 27import android.os.Bundle; 28import android.os.Handler; 29import android.os.Looper; 30import android.os.Message; 31import android.view.Surface; 32 33import java.io.IOException; 34import java.nio.ByteBuffer; 35import java.nio.ReadOnlyBufferException; 36import java.util.Arrays; 37import java.util.Map; 38 39/** 40 * MediaCodec class can be used to access low-level media codec, i.e. 41 * encoder/decoder components. 42 * 43 * <p>MediaCodec is generally used like this: 44 * <pre> 45 * MediaCodec codec = MediaCodec.createDecoderByType(type); 46 * codec.configure(format, ...); 47 * codec.start(); 48 * 49 * // if API level <= 20, get input and output buffer arrays here 50 * ByteBuffer[] inputBuffers = codec.getInputBuffers(); 51 * ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 52 * for (;;) { 53 * int inputBufferIndex = codec.dequeueInputBuffer(timeoutUs); 54 * if (inputBufferIndex >= 0) { 55 * // if API level >= 21, get input buffer here 56 * ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferIndex); 57 * // fill inputBuffers[inputBufferIndex] with valid data 58 * ... 59 * codec.queueInputBuffer(inputBufferIndex, ...); 60 * } 61 * 62 * int outputBufferIndex = codec.dequeueOutputBuffer(timeoutUs); 63 * if (outputBufferIndex >= 0) { 64 * // if API level >= 21, get output buffer here 65 * ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferIndex); 66 * // outputBuffer is ready to be processed or rendered. 67 * ... 68 * codec.releaseOutputBuffer(outputBufferIndex, ...); 69 * } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 70 * // no needed to handle if API level >= 21 and using getOutputBuffer(int) 71 * outputBuffers = codec.getOutputBuffers(); 72 * } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 73 * // Subsequent data will conform to new format. 74 * // can ignore if API level >= 21 and using getOutputFormat(outputBufferIndex) 75 * MediaFormat format = codec.getOutputFormat(); 76 * ... 77 * } 78 * } 79 * codec.stop(); 80 * codec.release(); 81 * codec = null; 82 * </pre> 83 * 84 * Each codec maintains a number of input and output buffers that are 85 * referred to by index in API calls. 86 * <p> 87 * For API levels 20 and below: 88 * The contents of these buffers are represented by the ByteBuffer[] arrays 89 * accessible through {@link #getInputBuffers} and {@link #getOutputBuffers}. 90 * <p> 91 * After a successful call to {@link #start} the client "owns" neither 92 * input nor output buffers, subsequent calls to {@link #dequeueInputBuffer} 93 * and {@link #dequeueOutputBuffer} then transfer ownership from the codec 94 * to the client.<p> 95 * The client is not required to resubmit/release buffers immediately 96 * to the codec, the sample code above simply does this for simplicity's sake.<p> 97 * Once the client has an input buffer available it can fill it with data 98 * and submit it it to the codec via a call to {@link #queueInputBuffer}.<p> 99 * The codec in turn will return an output buffer to the client in response 100 * to {@link #dequeueOutputBuffer}. After the output buffer has been processed 101 * a call to {@link #releaseOutputBuffer} will return it to the codec. 102 * If a video surface has been provided in the call to {@link #configure}, 103 * {@link #releaseOutputBuffer} optionally allows rendering of the buffer 104 * to the surface.<p> 105 * 106 * Input buffers (for decoders) and Output buffers (for encoders) contain 107 * encoded data according to the format's type. For video types this data 108 * is all the encoded data representing a single moment in time, for audio 109 * data this is slightly relaxed in that a buffer may contain multiple 110 * encoded frames of audio. In either case, buffers do not start and end on 111 * arbitrary byte boundaries, this is not a stream of bytes, it's a stream 112 * of access units.<p> 113 * 114 * Most formats also require the actual data to be prefixed by a number 115 * of buffers containing setup data, or codec specific data, i.e. the 116 * first few buffers submitted to the codec object after starting it must 117 * be codec specific data marked as such using the flag {@link #BUFFER_FLAG_CODEC_CONFIG} 118 * in a call to {@link #queueInputBuffer}. 119 * <p> 120 * Codec specific data included in the format passed to {@link #configure} 121 * (in ByteBuffer entries with keys "csd-0", "csd-1", ...) is automatically 122 * submitted to the codec, this data MUST NOT be submitted explicitly by the 123 * client. 124 * <p> 125 * Once the client reaches the end of the input data it signals the end of 126 * the input stream by specifying a flag of {@link #BUFFER_FLAG_END_OF_STREAM} in the call to 127 * {@link #queueInputBuffer}. The codec will continue to return output buffers 128 * until it eventually signals the end of the output stream by specifying 129 * the same flag ({@link #BUFFER_FLAG_END_OF_STREAM}) on the BufferInfo returned in 130 * {@link #dequeueOutputBuffer}. 131 * <p> 132 * In order to start decoding data that's not adjacent to previously submitted 133 * data (i.e. after a seek) it is necessary to {@link #flush} the decoder. 134 * Any input or output buffers the client may own at the point of the flush are 135 * immediately revoked, i.e. after a call to {@link #flush} the client does not 136 * own any buffers anymore. 137 * Note that the format of the data submitted after a flush must not change, 138 * flush does not support format discontinuities, 139 * for this a full {@link #stop}, {@link #configure}, {@link #start} 140 * cycle is necessary. 141 * 142 * <p> During its life, a codec conceptually exists in one of the following states: 143 * Initialized, Configured, Executing, Error, Uninitialized, (omitting transitory states 144 * between them). When created by one of the factory methods, 145 * the codec is in the Initialized state; {@link #configure} brings it to the 146 * Configured state; {@link #start} brings it to the Executing state. 147 * In the Executing state, decoding or encoding occurs through the buffer queue 148 * manipulation described above. The method {@link #stop} 149 * returns the codec to the Initialized state, whereupon it may be configured again, 150 * and {@link #release} brings the codec to the terminal Uninitialized state. When 151 * a codec error occurs, the codec moves to the Error state. Use {@link #reset} to 152 * bring the codec back to the Initialized state, or {@link #release} to move it 153 * to the Uninitialized state. 154 * 155 * <p> The factory methods 156 * {@link #createByCodecName}, 157 * {@link #createDecoderByType}, 158 * and {@link #createEncoderByType} 159 * throw {@link java.io.IOException} on failure which 160 * the caller must catch or declare to pass up. 161 * MediaCodec methods throw {@link java.lang.IllegalStateException} 162 * when the method is called from a codec state that does not allow it; 163 * this is typically due to incorrect application API usage. 164 * Methods involving secure buffers may throw 165 * {@link MediaCodec.CryptoException#MediaCodec.CryptoException}, which 166 * has further error information obtainable from {@link MediaCodec.CryptoException#getErrorCode}. 167 * 168 * <p> Internal codec errors result in a {@link MediaCodec.CodecException}, 169 * which may be due to media content corruption, hardware failure, resource exhaustion, 170 * and so forth, even when the application is correctly using the API. 171 * The recommended action when receiving a {@link MediaCodec.CodecException} can be determined by 172 * calling {@link MediaCodec.CodecException#isRecoverable} and 173 * {@link MediaCodec.CodecException#isTransient}. 174 * If {@link MediaCodec.CodecException#isRecoverable} returns true, 175 * then a {@link #stop}, {@link #configure}, and {@link #start} can be performed to recover. 176 * If {@link MediaCodec.CodecException#isTransient} returns true, 177 * then resources are temporarily unavailable and the method may be retried at a later time. 178 * If both {@link MediaCodec.CodecException#isRecoverable} 179 * and {@link MediaCodec.CodecException#isTransient} return false, 180 * then the {@link MediaCodec.CodecException} is fatal and the codec must be 181 * {@link #reset reset} or {@link #release released}. 182 * Both {@link MediaCodec.CodecException#isRecoverable} and 183 * {@link MediaCodec.CodecException#isTransient} do not return true at the same time. 184 */ 185final public class MediaCodec { 186 /** 187 * Per buffer metadata includes an offset and size specifying 188 * the range of valid data in the associated codec (output) buffer. 189 */ 190 public final static class BufferInfo { 191 /** 192 * Update the buffer metadata information. 193 * 194 * @param newOffset the start-offset of the data in the buffer. 195 * @param newSize the amount of data (in bytes) in the buffer. 196 * @param newTimeUs the presentation timestamp in microseconds. 197 * @param newFlags buffer flags associated with the buffer. This 198 * should be a combination of {@link #BUFFER_FLAG_KEY_FRAME} and 199 * {@link #BUFFER_FLAG_END_OF_STREAM}. 200 */ 201 public void set( 202 int newOffset, int newSize, long newTimeUs, int newFlags) { 203 offset = newOffset; 204 size = newSize; 205 presentationTimeUs = newTimeUs; 206 flags = newFlags; 207 } 208 209 /** 210 * The start-offset of the data in the buffer. 211 */ 212 public int offset; 213 214 /** 215 * The amount of data (in bytes) in the buffer. If this is {@code 0}, 216 * the buffer has no data in it and can be discarded. The only 217 * use of a 0-size buffer is to carry the end-of-stream marker. 218 */ 219 public int size; 220 221 /** 222 * The presentation timestamp in microseconds for the buffer. 223 * This is derived from the presentation timestamp passed in 224 * with the corresponding input buffer. This should be ignored for 225 * a 0-sized buffer. 226 */ 227 public long presentationTimeUs; 228 229 /** 230 * Buffer flags associated with the buffer. A combination of 231 * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}. 232 * 233 * <p>Encoded buffers that are key frames are marked with 234 * {@link #BUFFER_FLAG_KEY_FRAME}. 235 * 236 * <p>The last output buffer corresponding to the input buffer 237 * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked 238 * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could 239 * be an empty buffer, whose sole purpose is to carry the end-of-stream 240 * marker. 241 */ 242 public int flags; 243 }; 244 245 // The follow flag constants MUST stay in sync with their equivalents 246 // in MediaCodec.h ! 247 248 /** 249 * This indicates that the (encoded) buffer marked as such contains 250 * the data for a key frame. 251 * 252 * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead. 253 */ 254 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 255 256 /** 257 * This indicates that the (encoded) buffer marked as such contains 258 * the data for a key frame. 259 */ 260 public static final int BUFFER_FLAG_KEY_FRAME = 1; 261 262 /** 263 * This indicated that the buffer marked as such contains codec 264 * initialization / codec specific data instead of media data. 265 */ 266 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 267 268 /** 269 * This signals the end of stream, i.e. no buffers will be available 270 * after this, unless of course, {@link #flush} follows. 271 */ 272 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 273 274 private EventHandler mEventHandler; 275 private Callback mCallback; 276 277 private static final int EVENT_CALLBACK = 1; 278 private static final int EVENT_SET_CALLBACK = 2; 279 280 private static final int CB_INPUT_AVAILABLE = 1; 281 private static final int CB_OUTPUT_AVAILABLE = 2; 282 private static final int CB_ERROR = 3; 283 private static final int CB_OUTPUT_FORMAT_CHANGE = 4; 284 285 private class EventHandler extends Handler { 286 private MediaCodec mCodec; 287 288 public EventHandler(MediaCodec codec, Looper looper) { 289 super(looper); 290 mCodec = codec; 291 } 292 293 @Override 294 public void handleMessage(Message msg) { 295 switch (msg.what) { 296 case EVENT_CALLBACK: 297 { 298 handleCallback(msg); 299 break; 300 } 301 case EVENT_SET_CALLBACK: 302 { 303 mCallback = (MediaCodec.Callback) msg.obj; 304 break; 305 } 306 default: 307 { 308 break; 309 } 310 } 311 } 312 313 private void handleCallback(Message msg) { 314 if (mCallback == null) { 315 return; 316 } 317 318 switch (msg.arg1) { 319 case CB_INPUT_AVAILABLE: 320 { 321 int index = msg.arg2; 322 synchronized(mBufferLock) { 323 validateInputByteBuffer(mCachedInputBuffers, index); 324 } 325 mCallback.onInputBufferAvailable(mCodec, index); 326 break; 327 } 328 329 case CB_OUTPUT_AVAILABLE: 330 { 331 int index = msg.arg2; 332 BufferInfo info = (MediaCodec.BufferInfo) msg.obj; 333 synchronized(mBufferLock) { 334 validateOutputByteBuffer(mCachedOutputBuffers, index, info); 335 } 336 mCallback.onOutputBufferAvailable( 337 mCodec, index, info); 338 break; 339 } 340 341 case CB_ERROR: 342 { 343 mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj); 344 break; 345 } 346 347 case CB_OUTPUT_FORMAT_CHANGE: 348 { 349 mCallback.onOutputFormatChanged(mCodec, 350 new MediaFormat((Map<String, Object>) msg.obj)); 351 break; 352 } 353 354 default: 355 { 356 break; 357 } 358 } 359 } 360 } 361 362 /** 363 * Instantiate a decoder supporting input data of the given mime type. 364 * 365 * The following is a partial list of defined mime types and their semantics: 366 * <ul> 367 * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) 368 * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) 369 * <li>"video/avc" - H.264/AVC video 370 * <li>"video/hevc" - H.265/HEVC video 371 * <li>"video/mp4v-es" - MPEG4 video 372 * <li>"video/3gpp" - H.263 video 373 * <li>"audio/3gpp" - AMR narrowband audio 374 * <li>"audio/amr-wb" - AMR wideband audio 375 * <li>"audio/mpeg" - MPEG1/2 audio layer III 376 * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!) 377 * <li>"audio/vorbis" - vorbis audio 378 * <li>"audio/g711-alaw" - G.711 alaw audio 379 * <li>"audio/g711-mlaw" - G.711 ulaw audio 380 * </ul> 381 * 382 * @param type The mime type of the input data. 383 * @throws IOException if the codec cannot be created. 384 * @throws IllegalArgumentException if type is not a valid mime type. 385 * @throws NullPointerException if type is null. 386 */ 387 public static MediaCodec createDecoderByType(String type) 388 throws IOException { 389 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 390 } 391 392 /** 393 * Instantiate an encoder supporting output data of the given mime type. 394 * @param type The desired mime type of the output data. 395 * @throws IOException if the codec cannot be created. 396 * @throws IllegalArgumentException if type is not a valid mime type. 397 * @throws NullPointerException if type is null. 398 */ 399 public static MediaCodec createEncoderByType(String type) 400 throws IOException { 401 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 402 } 403 404 /** 405 * If you know the exact name of the component you want to instantiate 406 * use this method to instantiate it. Use with caution. 407 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 408 * @param name The name of the codec to be instantiated. 409 * @throws IOException if the codec cannot be created. 410 * @throws IllegalArgumentException if name is not valid. 411 * @throws NullPointerException if name is null. 412 */ 413 public static MediaCodec createByCodecName(String name) 414 throws IOException { 415 return new MediaCodec( 416 name, false /* nameIsType */, false /* unused */); 417 } 418 419 private MediaCodec( 420 String name, boolean nameIsType, boolean encoder) { 421 Looper looper; 422 if ((looper = Looper.myLooper()) != null) { 423 mEventHandler = new EventHandler(this, looper); 424 } else if ((looper = Looper.getMainLooper()) != null) { 425 mEventHandler = new EventHandler(this, looper); 426 } else { 427 mEventHandler = null; 428 } 429 mBufferLock = new Object(); 430 431 native_setup(name, nameIsType, encoder); 432 } 433 434 @Override 435 protected void finalize() { 436 native_finalize(); 437 } 438 439 /** 440 * Returns the codec to its initial (Initialized) state. 441 * 442 * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable} 443 * error has occured to reset the codec to its initial state after creation. 444 * 445 * @throws CodecException if an unrecoverable error has occured and the codec 446 * could not be reset. 447 * @throws IllegalStateException if in the Uninitialized state. 448 */ 449 public final void reset() { 450 freeAllTrackedBuffers(); // free buffers first 451 native_reset(); 452 } 453 454 private native final void native_reset(); 455 456 /** 457 * Make sure you call this when you're done to free up any opened 458 * component instance instead of relying on the garbage collector 459 * to do this for you at some point in the future. 460 */ 461 public final void release() { 462 freeAllTrackedBuffers(); // free buffers first 463 native_release(); 464 } 465 466 private native final void native_release(); 467 468 /** 469 * If this codec is to be used as an encoder, pass this flag. 470 */ 471 public static final int CONFIGURE_FLAG_ENCODE = 1; 472 473 /** 474 * Configures a component. 475 * 476 * @param format The format of the input data (decoder) or the desired 477 * format of the output data (encoder). 478 * @param surface Specify a surface on which to render the output of this 479 * decoder. 480 * @param crypto Specify a crypto object to facilitate secure decryption 481 * of the media data. 482 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 483 * component as an encoder. 484 * @throws IllegalArgumentException if the surface has been released (or is invalid), 485 * or the format is unacceptable (e.g. missing a mandatory key), 486 * or the flags are not set properly 487 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 488 * @throws IllegalStateException if not in the Initialized state. 489 */ 490 public void configure( 491 MediaFormat format, 492 Surface surface, MediaCrypto crypto, int flags) { 493 Map<String, Object> formatMap = format.getMap(); 494 495 String[] keys = null; 496 Object[] values = null; 497 498 if (format != null) { 499 keys = new String[formatMap.size()]; 500 values = new Object[formatMap.size()]; 501 502 int i = 0; 503 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 504 keys[i] = entry.getKey(); 505 values[i] = entry.getValue(); 506 ++i; 507 } 508 } 509 510 native_configure(keys, values, surface, crypto, flags); 511 } 512 513 private native final void native_setCallback(Callback cb); 514 515 private native final void native_configure( 516 String[] keys, Object[] values, 517 Surface surface, MediaCrypto crypto, int flags); 518 519 /** 520 * Requests a Surface to use as the input to an encoder, in place of input buffers. This 521 * may only be called after {@link #configure} and before {@link #start}. 522 * <p> 523 * The application is responsible for calling release() on the Surface when 524 * done. 525 * <p> 526 * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES. 527 * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce 528 * unexpected results. 529 * @throws IllegalStateException if not in the Configured state. 530 */ 531 public native final Surface createInputSurface(); 532 533 /** 534 * After successfully configuring the component, call start. On return 535 * you can query the component for its input/output buffers. 536 * @throws IllegalStateException if not in the Configured state. 537 * @throws MediaCodec.CodecException upon codec error. Note that some codec errors 538 * for start may be attributed to future method calls. 539 */ 540 public final void start() { 541 native_start(); 542 synchronized(mBufferLock) { 543 cacheBuffers(true /* input */); 544 cacheBuffers(false /* input */); 545 } 546 } 547 private native final void native_start(); 548 549 /** 550 * Finish the decode/encode session, note that the codec instance 551 * remains active and ready to be {@link #start}ed again. 552 * To ensure that it is available to other client call {@link #release} 553 * and don't just rely on garbage collection to eventually do this for you. 554 * @throws IllegalStateException if in the Uninitialized state. 555 */ 556 public final void stop() { 557 native_stop(); 558 freeAllTrackedBuffers(); 559 560 if (mEventHandler != null) { 561 mEventHandler.removeMessages(EVENT_CALLBACK); 562 mEventHandler.removeMessages(EVENT_SET_CALLBACK); 563 } 564 } 565 566 private native final void native_stop(); 567 568 /** 569 * Flush both input and output ports of the component, all indices 570 * previously returned in calls to {@link #dequeueInputBuffer} and 571 * {@link #dequeueOutputBuffer} become invalid. 572 * @throws IllegalStateException if not in the Executing state. 573 * @throws MediaCodec.CodecException upon codec error. 574 */ 575 public final void flush() { 576 synchronized(mBufferLock) { 577 invalidateByteBuffers(mCachedInputBuffers); 578 invalidateByteBuffers(mCachedOutputBuffers); 579 invalidateByteBuffers(mDequeuedInputBuffers); 580 invalidateByteBuffers(mDequeuedOutputBuffers); 581 freeImages(mDequeuedInputImages); 582 freeImages(mDequeuedOutputImages); 583 } 584 native_flush(); 585 } 586 587 private native final void native_flush(); 588 589 /** 590 * Thrown when an internal codec error occurs. 591 */ 592 public final static class CodecException extends IllegalStateException { 593 public CodecException(int errorCode, int actionCode, String detailMessage) { 594 super(detailMessage); 595 mErrorCode = errorCode; 596 mActionCode = actionCode; 597 } 598 599 /** 600 * Returns true if the codec exception is a transient issue, 601 * perhaps due to resource constraints, and that the method 602 * (or encoding/decoding) may be retried at a later time. 603 */ 604 public boolean isTransient() { 605 return mActionCode == ACTION_TRANSIENT; 606 } 607 608 /** 609 * Returns true if the codec cannot proceed further, 610 * but can be recovered by stopping, configuring, 611 * and starting again. 612 */ 613 public boolean isRecoverable() { 614 return mActionCode == ACTION_RECOVERABLE; 615 } 616 617 /** 618 * Retrieve the error code associated with a CodecException. 619 * This is opaque diagnostic information and may depend on 620 * hardware or API level. 621 */ 622 public int getErrorCode() { 623 return mErrorCode; 624 } 625 626 /* Must be in sync with android_media_MediaCodec.cpp */ 627 private final static int ACTION_TRANSIENT = 1; 628 private final static int ACTION_RECOVERABLE = 2; 629 630 private final int mErrorCode; 631 private final int mActionCode; 632 } 633 634 /** 635 * Thrown when a crypto error occurs while queueing a secure input buffer. 636 */ 637 public final static class CryptoException extends RuntimeException { 638 public CryptoException(int errorCode, String detailMessage) { 639 super(detailMessage); 640 mErrorCode = errorCode; 641 } 642 643 /** 644 * This indicates that no key has been set to perform the requested 645 * decrypt operation. 646 */ 647 public static final int ERROR_NO_KEY = 1; 648 649 /** 650 * This indicates that the key used for decryption is no longer 651 * valid due to license term expiration. 652 */ 653 public static final int ERROR_KEY_EXPIRED = 2; 654 655 /** 656 * This indicates that a required crypto resource was not able to be 657 * allocated while attempting the requested operation. 658 */ 659 public static final int ERROR_RESOURCE_BUSY = 3; 660 661 /** 662 * Retrieve the error code associated with a CryptoException 663 */ 664 public int getErrorCode() { 665 return mErrorCode; 666 } 667 668 private int mErrorCode; 669 } 670 671 /** 672 * After filling a range of the input buffer at the specified index 673 * submit it to the component. Once an input buffer is queued to 674 * the codec, it MUST NOT be used until it is later retrieved by 675 * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer} 676 * return value or a {@link Callback#onInputBufferAvailable} 677 * callback. 678 * <p> 679 * Many decoders require the actual compressed data stream to be 680 * preceded by "codec specific data", i.e. setup data used to initialize 681 * the codec such as PPS/SPS in the case of AVC video or code tables 682 * in the case of vorbis audio. 683 * The class {@link android.media.MediaExtractor} provides codec 684 * specific data as part of 685 * the returned track format in entries named "csd-0", "csd-1" ... 686 * <p> 687 * These buffers can be submitted directly after {@link #start} or 688 * {@link #flush} by specifying the flag {@link 689 * #BUFFER_FLAG_CODEC_CONFIG}. However, if you configure the 690 * codec with a {@link MediaFormat} containing these keys, they 691 * will be automatically submitted by MediaCodec directly after 692 * start. Therefore, the use of {@link 693 * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is 694 * recommended only for advanced users. 695 * <p> 696 * To indicate that this is the final piece of input data (or rather that 697 * no more input data follows unless the decoder is subsequently flushed) 698 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 699 * 700 * @param index The index of a client-owned input buffer previously returned 701 * in a call to {@link #dequeueInputBuffer}. 702 * @param offset The byte offset into the input buffer at which the data starts. 703 * @param size The number of bytes of valid input data. 704 * @param presentationTimeUs The presentation timestamp in microseconds for this 705 * buffer. This is normally the media time at which this 706 * buffer should be presented (rendered). 707 * @param flags A bitmask of flags 708 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 709 * While not prohibited, most codecs do not use the 710 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 711 * @throws IllegalStateException if not in the Executing state. 712 * @throws MediaCodec.CodecException upon codec error. 713 * @throws CryptoException if a crypto object has been specified in 714 * {@link #configure} 715 */ 716 public final void queueInputBuffer( 717 int index, 718 int offset, int size, long presentationTimeUs, int flags) 719 throws CryptoException { 720 synchronized(mBufferLock) { 721 invalidateByteBuffer(mCachedInputBuffers, index); 722 updateDequeuedByteBuffer(mDequeuedInputBuffers, index, null); 723 } 724 native_queueInputBuffer( 725 index, offset, size, presentationTimeUs, flags); 726 } 727 728 private native final void native_queueInputBuffer( 729 int index, 730 int offset, int size, long presentationTimeUs, int flags) 731 throws CryptoException; 732 733 // The following mode constants MUST stay in sync with their equivalents 734 // in media/hardware/CryptoAPI.h ! 735 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 736 public static final int CRYPTO_MODE_AES_CTR = 1; 737 738 /** 739 * Metadata describing the structure of a (at least partially) encrypted 740 * input sample. 741 * A buffer's data is considered to be partitioned into "subSamples", 742 * each subSample starts with a (potentially empty) run of plain, 743 * unencrypted bytes followed by a (also potentially empty) run of 744 * encrypted bytes. 745 * numBytesOfClearData can be null to indicate that all data is encrypted. 746 * This information encapsulates per-sample metadata as outlined in 747 * ISO/IEC FDIS 23001-7:2011 "Common encryption in ISO base media file format files". 748 */ 749 public final static class CryptoInfo { 750 public void set( 751 int newNumSubSamples, 752 int[] newNumBytesOfClearData, 753 int[] newNumBytesOfEncryptedData, 754 byte[] newKey, 755 byte[] newIV, 756 int newMode) { 757 numSubSamples = newNumSubSamples; 758 numBytesOfClearData = newNumBytesOfClearData; 759 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 760 key = newKey; 761 iv = newIV; 762 mode = newMode; 763 } 764 765 /** 766 * The number of subSamples that make up the buffer's contents. 767 */ 768 public int numSubSamples; 769 /** 770 * The number of leading unencrypted bytes in each subSample. 771 */ 772 public int[] numBytesOfClearData; 773 /** 774 * The number of trailing encrypted bytes in each subSample. 775 */ 776 public int[] numBytesOfEncryptedData; 777 /** 778 * A 16-byte opaque key 779 */ 780 public byte[] key; 781 /** 782 * A 16-byte initialization vector 783 */ 784 public byte[] iv; 785 /** 786 * The type of encryption that has been applied, 787 * see {@link #CRYPTO_MODE_UNENCRYPTED} and {@link #CRYPTO_MODE_AES_CTR}. 788 */ 789 public int mode; 790 791 @Override 792 public String toString() { 793 StringBuilder builder = new StringBuilder(); 794 builder.append(numSubSamples + " subsamples, key ["); 795 String hexdigits = "0123456789abcdef"; 796 for (int i = 0; i < key.length; i++) { 797 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4)); 798 builder.append(hexdigits.charAt(key[i] & 0x0f)); 799 } 800 builder.append("], iv ["); 801 for (int i = 0; i < key.length; i++) { 802 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4)); 803 builder.append(hexdigits.charAt(iv[i] & 0x0f)); 804 } 805 builder.append("], clear "); 806 builder.append(Arrays.toString(numBytesOfClearData)); 807 builder.append(", encrypted "); 808 builder.append(Arrays.toString(numBytesOfEncryptedData)); 809 return builder.toString(); 810 } 811 }; 812 813 /** 814 * Similar to {@link #queueInputBuffer} but submits a buffer that is 815 * potentially encrypted. 816 * @param index The index of a client-owned input buffer previously returned 817 * in a call to {@link #dequeueInputBuffer}. 818 * @param offset The byte offset into the input buffer at which the data starts. 819 * @param info Metadata required to facilitate decryption, the object can be 820 * reused immediately after this call returns. 821 * @param presentationTimeUs The presentation timestamp in microseconds for this 822 * buffer. This is normally the media time at which this 823 * buffer should be presented (rendered). 824 * @param flags A bitmask of flags 825 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 826 * While not prohibited, most codecs do not use the 827 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 828 * @throws IllegalStateException if not in the Executing state. 829 * @throws MediaCodec.CodecException upon codec error. 830 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 831 * An error code associated with the exception helps identify the 832 * reason for the failure. 833 */ 834 public final void queueSecureInputBuffer( 835 int index, 836 int offset, 837 CryptoInfo info, 838 long presentationTimeUs, 839 int flags) throws CryptoException { 840 synchronized(mBufferLock) { 841 invalidateByteBuffer(mCachedInputBuffers, index); 842 updateDequeuedByteBuffer(mDequeuedInputBuffers, index, null); 843 } 844 native_queueSecureInputBuffer( 845 index, offset, info, presentationTimeUs, flags); 846 } 847 848 private native final void native_queueSecureInputBuffer( 849 int index, 850 int offset, 851 CryptoInfo info, 852 long presentationTimeUs, 853 int flags) throws CryptoException; 854 855 /** 856 * Returns the index of an input buffer to be filled with valid data 857 * or -1 if no such buffer is currently available. 858 * This method will return immediately if timeoutUs == 0, wait indefinitely 859 * for the availability of an input buffer if timeoutUs < 0 or wait up 860 * to "timeoutUs" microseconds if timeoutUs > 0. 861 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 862 * @throws IllegalStateException if not in the Executing state. 863 * @throws MediaCodec.CodecException upon codec error. 864 */ 865 public final int dequeueInputBuffer(long timeoutUs) { 866 int res = native_dequeueInputBuffer(timeoutUs); 867 if (res >= 0) { 868 synchronized(mBufferLock) { 869 validateInputByteBuffer(mCachedInputBuffers, res); 870 } 871 } 872 return res; 873 } 874 875 private native final int native_dequeueInputBuffer(long timeoutUs); 876 877 /** 878 * If a non-negative timeout had been specified in the call 879 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 880 */ 881 public static final int INFO_TRY_AGAIN_LATER = -1; 882 883 /** 884 * The output format has changed, subsequent data will follow the new 885 * format. {@link #getOutputFormat()} returns the new format. Note, that 886 * you can also use the new {@link #getOutputFormat(int)} method to 887 * get the format for a specific output buffer. This frees you from 888 * having to track output format changes. 889 */ 890 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 891 892 /** 893 * The output buffers have changed, the client must refer to the new 894 * set of output buffers returned by {@link #getOutputBuffers} from 895 * this point on. 896 * 897 * @deprecated This return value can be ignored as {@link 898 * #getOutputBuffers} has been deprecated. Client should 899 * request a current buffer using on of the get-buffer or 900 * get-image methods each time one has been dequeued. 901 */ 902 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 903 904 /** 905 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 906 * Returns the index of an output buffer that has been successfully 907 * decoded or one of the INFO_* constants below. 908 * @param info Will be filled with buffer meta data. 909 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 910 * @throws IllegalStateException if not in the Executing state. 911 * @throws MediaCodec.CodecException upon codec error. 912 */ 913 public final int dequeueOutputBuffer( 914 BufferInfo info, long timeoutUs) { 915 int res = native_dequeueOutputBuffer(info, timeoutUs); 916 synchronized(mBufferLock) { 917 if (res == INFO_OUTPUT_BUFFERS_CHANGED) { 918 cacheBuffers(false /* input */); 919 } else if (res >= 0) { 920 validateOutputByteBuffer(mCachedOutputBuffers, res, info); 921 } 922 } 923 return res; 924 } 925 926 private native final int native_dequeueOutputBuffer( 927 BufferInfo info, long timeoutUs); 928 929 /** 930 * If you are done with a buffer, use this call to return the buffer to 931 * the codec. If you previously specified a surface when configuring this 932 * video decoder you can optionally render the buffer. 933 * 934 * Once an output buffer is released to the codec, it MUST NOT 935 * be used until it is later retrieved by {@link #getOutputBuffer} in response 936 * to a {@link #dequeueOutputBuffer} return value or a 937 * {@link Callback#onOutputBufferAvailable} callback. 938 * 939 * @param index The index of a client-owned output buffer previously returned 940 * from a call to {@link #dequeueOutputBuffer}. 941 * @param render If a valid surface was specified when configuring the codec, 942 * passing true renders this output buffer to the surface. 943 * @throws IllegalStateException if not in the Executing state. 944 * @throws MediaCodec.CodecException upon codec error. 945 */ 946 public final void releaseOutputBuffer(int index, boolean render) { 947 synchronized(mBufferLock) { 948 invalidateByteBuffer(mCachedOutputBuffers, index); 949 updateDequeuedByteBuffer(mDequeuedOutputBuffers, index, null); 950 } 951 releaseOutputBuffer(index, render, false /* updatePTS */, 0 /* dummy */); 952 } 953 954 /** 955 * If you are done with a buffer, use this call to update its surface timestamp 956 * and return it to the codec to render it on the output surface. If you 957 * have not specified an output surface when configuring this video codec, 958 * this call will simply return the buffer to the codec.<p> 959 * 960 * The timestamp may have special meaning depending on the destination surface. 961 * 962 * <table> 963 * <tr><th>SurfaceView specifics</th></tr> 964 * <tr><td> 965 * If you render your buffer on a {@link android.view.SurfaceView}, 966 * you can use the timestamp to render the buffer at a specific time (at the 967 * VSYNC at or after the buffer timestamp). For this to work, the timestamp 968 * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}. 969 * Currently, this is set as within one (1) second. A few notes: 970 * 971 * <ul> 972 * <li>the buffer will not be returned to the codec until the timestamp 973 * has passed and the buffer is no longer used by the {@link android.view.Surface}. 974 * <li>buffers are processed sequentially, so you may block subsequent buffers to 975 * be displayed on the {@link android.view.Surface}. This is important if you 976 * want to react to user action, e.g. stop the video or seek. 977 * <li>if multiple buffers are sent to the {@link android.view.Surface} to be 978 * rendered at the same VSYNC, the last one will be shown, and the other ones 979 * will be dropped. 980 * <li>if the timestamp is <em>not</em> "reasonably close" to the current system 981 * time, the {@link android.view.Surface} will ignore the timestamp, and 982 * display the buffer at the earliest feasible time. In this mode it will not 983 * drop frames. 984 * <li>for best performance and quality, call this method when you are about 985 * two VSYNCs' time before the desired render time. For 60Hz displays, this is 986 * about 33 msec. 987 * </ul> 988 * </td></tr> 989 * </table> 990 * 991 * Once an output buffer is released to the codec, it MUST NOT 992 * be used until it is later retrieved by {@link #getOutputBuffer} in response 993 * to a {@link #dequeueOutputBuffer} return value or a 994 * {@link Callback#onOutputBufferAvailable} callback. 995 * 996 * @param index The index of a client-owned output buffer previously returned 997 * from a call to {@link #dequeueOutputBuffer}. 998 * @param renderTimestampNs The timestamp to associate with this buffer when 999 * it is sent to the Surface. 1000 * @throws IllegalStateException if not in the Executing state. 1001 * @throws MediaCodec.CodecException upon codec error. 1002 */ 1003 public final void releaseOutputBuffer(int index, long renderTimestampNs) { 1004 synchronized(mBufferLock) { 1005 invalidateByteBuffer(mCachedOutputBuffers, index); 1006 updateDequeuedByteBuffer(mDequeuedOutputBuffers, index, null); 1007 } 1008 releaseOutputBuffer( 1009 index, true /* render */, true /* updatePTS */, renderTimestampNs); 1010 } 1011 1012 private native final void releaseOutputBuffer( 1013 int index, boolean render, boolean updatePTS, long timeNs); 1014 1015 /** 1016 * Signals end-of-stream on input. Equivalent to submitting an empty buffer with 1017 * {@link #BUFFER_FLAG_END_OF_STREAM} set. This may only be used with 1018 * encoders receiving input from a Surface created by {@link #createInputSurface}. 1019 * @throws IllegalStateException if not in the Executing state. 1020 * @throws MediaCodec.CodecException upon codec error. 1021 */ 1022 public native final void signalEndOfInputStream(); 1023 1024 /** 1025 * Call this after dequeueOutputBuffer signals a format change by returning 1026 * {@link #INFO_OUTPUT_FORMAT_CHANGED}. 1027 * You can also call this after {@link #configure} returns 1028 * successfully to get the output format initially configured 1029 * for the codec. Do this to determine what optional 1030 * configuration parameters were supported by the codec. 1031 * 1032 * @throws IllegalStateException if not in the Executing or 1033 * Configured state. 1034 * @throws MediaCodec.CodecException upon codec error. 1035 */ 1036 public final MediaFormat getOutputFormat() { 1037 return new MediaFormat(getFormatNative(false /* input */)); 1038 } 1039 1040 /** 1041 * Call this after {@link #configure} returns successfully to 1042 * get the input format accepted by the codec. Do this to 1043 * determine what optional configuration parameters were 1044 * supported by the codec. 1045 * 1046 * @throws IllegalStateException if not in the Executing or 1047 * Configured state. 1048 * @throws MediaCodec.CodecException upon codec error. 1049 */ 1050 public final MediaFormat getInputFormat() { 1051 return new MediaFormat(getFormatNative(true /* input */)); 1052 } 1053 1054 /** 1055 * Returns the output format for a specific output buffer. 1056 * 1057 * @param index The index of a client-owned input buffer previously 1058 * returned from a call to {@link #dequeueInputBuffer}. 1059 * 1060 * @return the format for the output buffer, or null if the index 1061 * is not a dequeued output buffer. 1062 */ 1063 public final MediaFormat getOutputFormat(int index) { 1064 return new MediaFormat(getOutputFormatNative(index)); 1065 } 1066 1067 private native final Map<String, Object> getFormatNative(boolean input); 1068 1069 private native final Map<String, Object> getOutputFormatNative(int index); 1070 1071 private ByteBuffer[] mCachedInputBuffers; 1072 private ByteBuffer[] mCachedOutputBuffers; 1073 private ByteBuffer[] mDequeuedInputBuffers; 1074 private ByteBuffer[] mDequeuedOutputBuffers; 1075 private Image[] mDequeuedInputImages; 1076 private Image[] mDequeuedOutputImages; 1077 final private Object mBufferLock; 1078 1079 private final void invalidateByteBuffer( 1080 ByteBuffer[] buffers, int index) { 1081 if (index >= 0 && index < buffers.length) { 1082 ByteBuffer buffer = buffers[index]; 1083 if (buffer != null) { 1084 buffer.setAccessible(false); 1085 } 1086 } 1087 } 1088 1089 private final void validateInputByteBuffer( 1090 ByteBuffer[] buffers, int index) { 1091 if (index >= 0 && index < buffers.length) { 1092 ByteBuffer buffer = buffers[index]; 1093 if (buffer != null) { 1094 buffer.setAccessible(true); 1095 buffer.clear(); 1096 } 1097 } 1098 } 1099 1100 private final void validateOutputByteBuffer( 1101 ByteBuffer[] buffers, int index, BufferInfo info) { 1102 if (index >= 0 && index < buffers.length) { 1103 ByteBuffer buffer = buffers[index]; 1104 if (buffer != null) { 1105 buffer.setAccessible(true); 1106 buffer.limit(info.offset + info.size).position(info.offset); 1107 } 1108 } 1109 } 1110 1111 private final void invalidateByteBuffers(ByteBuffer[] buffers) { 1112 if (buffers != null) { 1113 for (ByteBuffer buffer: buffers) { 1114 if (buffer != null) { 1115 buffer.setAccessible(false); 1116 } 1117 } 1118 } 1119 } 1120 1121 private final void freeByteBuffer(ByteBuffer buffer) { 1122 if (buffer != null /* && buffer.isDirect() */) { 1123 // all of our ByteBuffers are direct 1124 java.nio.NioUtils.freeDirectBuffer(buffer); 1125 } 1126 } 1127 1128 private final void freeByteBuffers(ByteBuffer[] buffers) { 1129 if (buffers != null) { 1130 for (ByteBuffer buffer: buffers) { 1131 freeByteBuffer(buffer); 1132 } 1133 } 1134 } 1135 1136 private final void freeImage(Image image) { 1137 if (image != null) { 1138 image.close(); 1139 } 1140 } 1141 1142 private final void freeImages(Image[] images) { 1143 if (images != null) { 1144 for (Image image: images) { 1145 freeImage(image); 1146 } 1147 } 1148 } 1149 1150 private final void freeAllTrackedBuffers() { 1151 freeByteBuffers(mCachedInputBuffers); 1152 freeByteBuffers(mCachedOutputBuffers); 1153 freeImages(mDequeuedInputImages); 1154 freeImages(mDequeuedOutputImages); 1155 freeByteBuffers(mDequeuedInputBuffers); 1156 freeByteBuffers(mDequeuedOutputBuffers); 1157 mCachedInputBuffers = null; 1158 mCachedOutputBuffers = null; 1159 mDequeuedInputImages = null; 1160 mDequeuedOutputImages = null; 1161 mDequeuedInputBuffers = null; 1162 mDequeuedOutputBuffers = null; 1163 } 1164 1165 private final void cacheBuffers(boolean input) { 1166 ByteBuffer[] buffers = getBuffers(input); 1167 invalidateByteBuffers(buffers); 1168 if (input) { 1169 mCachedInputBuffers = buffers; 1170 mDequeuedInputImages = new Image[buffers.length]; 1171 mDequeuedInputBuffers = new ByteBuffer[buffers.length]; 1172 } else { 1173 mCachedOutputBuffers = buffers; 1174 mDequeuedOutputImages = new Image[buffers.length]; 1175 mDequeuedOutputBuffers = new ByteBuffer[buffers.length]; 1176 } 1177 } 1178 1179 /** 1180 * Retrieve the set of input buffers. Call this after start() 1181 * returns. After calling this method, any ByteBuffers 1182 * previously returned by an earlier call to this method MUST no 1183 * longer be used. 1184 * 1185 * @deprecated Use the new {@link #getInputBuffer} method instead 1186 * each time an input buffer is dequeued. 1187 * 1188 * <b>Note:</b>As of API 21, dequeued input buffers are 1189 * automatically {@link java.nio.Buffer#clear cleared}. 1190 * 1191 * @throws IllegalStateException if not in the Executing state. 1192 * @throws MediaCodec.CodecException upon codec error. 1193 */ 1194 public ByteBuffer[] getInputBuffers() { 1195 if (mCachedInputBuffers == null) { 1196 throw new IllegalStateException(); 1197 } 1198 // FIXME: check codec status 1199 return mCachedInputBuffers; 1200 } 1201 1202 /** 1203 * Retrieve the set of output buffers. Call this after start() 1204 * returns and whenever dequeueOutputBuffer signals an output 1205 * buffer change by returning {@link 1206 * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any 1207 * ByteBuffers previously returned by an earlier call to this 1208 * method MUST no longer be used. 1209 * 1210 * @deprecated Use the new {@link #getOutputBuffer} method instead 1211 * each time an output buffer is dequeued. This method is not 1212 * supported if codec is configured in asynchronous mode. 1213 * 1214 * <b>Note:</b>As of API 21, the position and limit of output 1215 * buffers that are dequeued will be set to the valid data 1216 * range. 1217 * 1218 * @throws IllegalStateException if not in the Executing state, 1219 * or codec is configured in asynchronous mode. 1220 * @throws MediaCodec.CodecException upon codec error. 1221 */ 1222 public ByteBuffer[] getOutputBuffers() { 1223 if (mCachedOutputBuffers == null) { 1224 throw new IllegalStateException(); 1225 } 1226 // FIXME: check codec status 1227 return mCachedOutputBuffers; 1228 } 1229 1230 private boolean updateDequeuedByteBuffer( 1231 ByteBuffer[] buffers, int index, ByteBuffer newBuffer) { 1232 if (index < 0 || index >= buffers.length) { 1233 return false; 1234 } 1235 freeByteBuffer(buffers[index]); 1236 buffers[index] = newBuffer; 1237 return newBuffer != null; 1238 } 1239 1240 private boolean updateDequeuedImage( 1241 Image[] images, int index, Image newImage) { 1242 if (index < 0 || index >= images.length) { 1243 return false; 1244 } 1245 freeImage(images[index]); 1246 images[index] = newImage; 1247 return newImage != null; 1248 } 1249 1250 /** 1251 * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer 1252 * object for a dequeued input buffer index to contain the input data. 1253 * 1254 * After calling this method any ByteBuffer or Image object 1255 * previously returned for the same input index MUST no longer 1256 * be used. 1257 * 1258 * @param index The index of a client-owned input buffer previously 1259 * returned from a call to {@link #dequeueInputBuffer}, 1260 * or received via an onInputBufferAvailable callback. 1261 * 1262 * @return the input buffer, or null if the index is not a dequeued 1263 * input buffer, or if the codec is configured for surface input. 1264 * 1265 * @throws IllegalStateException if not in the Executing state. 1266 * @throws MediaCodec.CodecException upon codec error. 1267 */ 1268 public ByteBuffer getInputBuffer(int index) { 1269 ByteBuffer newBuffer = getBuffer(true /* input */, index); 1270 synchronized(mBufferLock) { 1271 if (updateDequeuedByteBuffer(mDequeuedInputBuffers, index, newBuffer)) { 1272 updateDequeuedImage(mDequeuedInputImages, index, null); 1273 invalidateByteBuffer(mCachedInputBuffers, index); 1274 return newBuffer; 1275 } 1276 } 1277 return null; 1278 } 1279 1280 /** 1281 * Returns a writable Image object for a dequeued input buffer 1282 * index to contain the raw input video frame. 1283 * 1284 * After calling this method any ByteBuffer or Image object 1285 * previously returned for the same input index MUST no longer 1286 * be used. 1287 * 1288 * @param index The index of a client-owned input buffer previously 1289 * returned from a call to {@link #dequeueInputBuffer}, 1290 * or received via an onInputBufferAvailable callback. 1291 * 1292 * @return the input image, or null if the index is not a 1293 * dequeued input buffer, or not a ByteBuffer that contains a 1294 * raw image. 1295 * 1296 * @throws IllegalStateException if not in the Executing state. 1297 * @throws MediaCodec.CodecException upon codec error. 1298 */ 1299 public Image getInputImage(int index) { 1300 Image newImage = getImage(true /* input */, index); 1301 if (updateDequeuedImage(mDequeuedInputImages, index, newImage)) { 1302 updateDequeuedByteBuffer(mDequeuedInputBuffers, index, null); 1303 invalidateByteBuffer(mCachedInputBuffers, index); 1304 return newImage; 1305 } 1306 return null; 1307 } 1308 1309 /** 1310 * Returns a read-only ByteBuffer for a dequeued output buffer 1311 * index. The position and limit of the returned buffer are set 1312 * to the valid output data. 1313 * 1314 * After calling this method, any ByteBuffer or Image object 1315 * previously returned for the same output index MUST no longer 1316 * be used. 1317 * 1318 * @param index The index of a client-owned output buffer previously 1319 * returned from a call to {@link #dequeueOutputBuffer}, 1320 * or received via an onOutputBufferAvailable callback. 1321 * 1322 * @return the output buffer, or null if the index is not a dequeued 1323 * output buffer, or the codec is configured with an output surface. 1324 * 1325 * @throws IllegalStateException if not in the Executing state. 1326 * @throws MediaCodec.CodecException upon codec error. 1327 */ 1328 public ByteBuffer getOutputBuffer(int index) { 1329 ByteBuffer newBuffer = getBuffer(false /* input */, index); 1330 synchronized(mBufferLock) { 1331 if (updateDequeuedByteBuffer(mDequeuedOutputBuffers, index, newBuffer)) { 1332 updateDequeuedImage(mDequeuedOutputImages, index, null); 1333 invalidateByteBuffer(mCachedOutputBuffers, index); 1334 return newBuffer; 1335 } 1336 } 1337 return null; 1338 } 1339 1340 /** 1341 * Returns a read-only Image object for a dequeued output buffer 1342 * index that contains the raw video frame. 1343 * 1344 * After calling this method, any ByteBuffer or Image object previously 1345 * returned for the same output index MUST no longer be used. 1346 * 1347 * @param index The index of a client-owned output buffer previously 1348 * returned from a call to {@link #dequeueOutputBuffer}, 1349 * or received via an onOutputBufferAvailable callback. 1350 * 1351 * @return the output image, or null if the index is not a 1352 * dequeued output buffer, not a raw video frame, or if the codec 1353 * was configured with an output surface. 1354 * 1355 * @throws IllegalStateException if not in the Executing state. 1356 * @throws MediaCodec.CodecException upon codec error. 1357 */ 1358 public Image getOutputImage(int index) { 1359 Image newImage = getImage(false /* input */, index); 1360 synchronized(mBufferLock) { 1361 if (updateDequeuedImage(mDequeuedOutputImages, index, newImage)) { 1362 updateDequeuedByteBuffer(mDequeuedOutputBuffers, index, null); 1363 invalidateByteBuffer(mCachedOutputBuffers, index); 1364 return newImage; 1365 } 1366 } 1367 return null; 1368 } 1369 1370 /** 1371 * The content is scaled to the surface dimensions 1372 */ 1373 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 1374 1375 /** 1376 * The content is scaled, maintaining its aspect ratio, the whole 1377 * surface area is used, content may be cropped 1378 */ 1379 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 1380 1381 /** 1382 * If a surface has been specified in a previous call to {@link #configure} 1383 * specifies the scaling mode to use. The default is "scale to fit". 1384 * @throws IllegalArgumentException if mode is not recognized. 1385 * @throws IllegalStateException if in the Uninitialized state. 1386 */ 1387 public native final void setVideoScalingMode(int mode); 1388 1389 /** 1390 * Get the component name. If the codec was created by createDecoderByType 1391 * or createEncoderByType, what component is chosen is not known beforehand. 1392 * @throws IllegalStateException if in the Uninitialized state. 1393 */ 1394 public native final String getName(); 1395 1396 /** 1397 * Change a video encoder's target bitrate on the fly. The value is an 1398 * Integer object containing the new bitrate in bps. 1399 */ 1400 public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate"; 1401 1402 /** 1403 * Temporarily suspend/resume encoding of input data. While suspended 1404 * input data is effectively discarded instead of being fed into the 1405 * encoder. This parameter really only makes sense to use with an encoder 1406 * in "surface-input" mode, as the client code has no control over the 1407 * input-side of the encoder in that case. 1408 * The value is an Integer object containing the value 1 to suspend 1409 * or the value 0 to resume. 1410 */ 1411 public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames"; 1412 1413 /** 1414 * Request that the encoder produce a sync frame "soon". 1415 * Provide an Integer with the value 0. 1416 */ 1417 public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync"; 1418 1419 /** 1420 * Communicate additional parameter changes to the component instance. 1421 * @throws IllegalStateException if in the Uninitialized state. 1422 */ 1423 public final void setParameters(Bundle params) { 1424 if (params == null) { 1425 return; 1426 } 1427 1428 String[] keys = new String[params.size()]; 1429 Object[] values = new Object[params.size()]; 1430 1431 int i = 0; 1432 for (final String key: params.keySet()) { 1433 keys[i] = key; 1434 values[i] = params.get(key); 1435 ++i; 1436 } 1437 1438 setParameters(keys, values); 1439 } 1440 1441 /** 1442 * Sets an asynchronous callback for actionable MediaCodec events. 1443 * 1444 * If the client intends to use the component in asynchronous mode, 1445 * a valid callback should be provided before {@link #configure} is called. 1446 * 1447 * When asynchronous callback is enabled, the client should not call 1448 * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)} 1449 * 1450 * @param cb The callback that will run. 1451 */ 1452 public void setCallback(/* MediaCodec. */ Callback cb) { 1453 if (mEventHandler != null) { 1454 // set java callback on handler 1455 Message msg = mEventHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb); 1456 mEventHandler.sendMessage(msg); 1457 1458 // set native handler here, don't post to handler because 1459 // it may cause the callback to be delayed and set in a wrong state, 1460 // and MediaCodec is already doing it on looper. 1461 native_setCallback(cb); 1462 } 1463 } 1464 1465 /** 1466 * MediaCodec callback interface. Used to notify the user asynchronously 1467 * of various MediaCodec events. 1468 */ 1469 public static abstract class Callback { 1470 /** 1471 * Called when an input buffer becomes available. 1472 * 1473 * @param codec The MediaCodec object. 1474 * @param index The index of the available input buffer. 1475 */ 1476 public abstract void onInputBufferAvailable(MediaCodec codec, int index); 1477 1478 /** 1479 * Called when an output buffer becomes available. 1480 * 1481 * @param codec The MediaCodec object. 1482 * @param index The index of the available output buffer. 1483 * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}. 1484 */ 1485 public abstract void onOutputBufferAvailable(MediaCodec codec, int index, BufferInfo info); 1486 1487 /** 1488 * Called when the MediaCodec encountered an error 1489 * 1490 * @param codec The MediaCodec object. 1491 * @param e The {@link MediaCodec.CodecException} object describing the error. 1492 */ 1493 public abstract void onError(MediaCodec codec, CodecException e); 1494 1495 /** 1496 * Called when the output format has changed 1497 * 1498 * @param codec The MediaCodec object. 1499 * @param format The new output format. 1500 */ 1501 public abstract void onOutputFormatChanged(MediaCodec codec, MediaFormat format); 1502 } 1503 1504 private void postEventFromNative( 1505 int what, int arg1, int arg2, Object obj) { 1506 if (mEventHandler != null) { 1507 Message msg = mEventHandler.obtainMessage(what, arg1, arg2, obj); 1508 mEventHandler.sendMessage(msg); 1509 } 1510 } 1511 1512 private native final void setParameters(String[] keys, Object[] values); 1513 1514 /** 1515 * Get the codec info. If the codec was created by createDecoderByType 1516 * or createEncoderByType, what component is chosen is not known beforehand, 1517 * and thus the caller does not have the MediaCodecInfo. 1518 * @throws IllegalStateException if in the Uninitialized state. 1519 */ 1520 public MediaCodecInfo getCodecInfo() { 1521 return MediaCodecList.getCodecInfoAt( 1522 MediaCodecList.findCodecByName(getName())); 1523 } 1524 1525 private native final ByteBuffer[] getBuffers(boolean input); 1526 1527 private native final ByteBuffer getBuffer(boolean input, int index); 1528 1529 private native final Image getImage(boolean input, int index); 1530 1531 private static native final void native_init(); 1532 1533 private native final void native_setup( 1534 String name, boolean nameIsType, boolean encoder); 1535 1536 private native final void native_finalize(); 1537 1538 static { 1539 System.loadLibrary("media_jni"); 1540 native_init(); 1541 } 1542 1543 private long mNativeContext; 1544 1545 /** @hide */ 1546 public static class MediaImage extends Image { 1547 private final boolean mIsReadOnly; 1548 private boolean mIsValid; 1549 private final int mWidth; 1550 private final int mHeight; 1551 private final int mFormat; 1552 private long mTimestamp; 1553 private final Plane[] mPlanes; 1554 private final ByteBuffer mBuffer; 1555 private final ByteBuffer mInfo; 1556 private final int mXOffset; 1557 private final int mYOffset; 1558 1559 private final static int TYPE_YUV = 1; 1560 1561 public int getFormat() { 1562 checkValid(); 1563 return mFormat; 1564 } 1565 1566 public int getHeight() { 1567 checkValid(); 1568 return mHeight; 1569 } 1570 1571 public int getWidth() { 1572 checkValid(); 1573 return mWidth; 1574 } 1575 1576 public long getTimestamp() { 1577 checkValid(); 1578 return mTimestamp; 1579 } 1580 1581 public Plane[] getPlanes() { 1582 checkValid(); 1583 return Arrays.copyOf(mPlanes, mPlanes.length); 1584 } 1585 1586 public void close() { 1587 if (mIsValid) { 1588 java.nio.NioUtils.freeDirectBuffer(mBuffer); 1589 mIsValid = false; 1590 } 1591 } 1592 1593 /** 1594 * Set the crop rectangle associated with this frame. 1595 * <p> 1596 * The crop rectangle specifies the region of valid pixels in the image, 1597 * using coordinates in the largest-resolution plane. 1598 */ 1599 public void setCropRect(Rect cropRect) { 1600 if (mIsReadOnly) { 1601 throw new ReadOnlyBufferException(); 1602 } 1603 super.setCropRect(cropRect); 1604 } 1605 1606 private void checkValid() { 1607 if (!mIsValid) { 1608 throw new IllegalStateException("Image is already released"); 1609 } 1610 } 1611 1612 private int readInt(ByteBuffer buffer, boolean asLong) { 1613 if (asLong) { 1614 return (int)buffer.getLong(); 1615 } else { 1616 return buffer.getInt(); 1617 } 1618 } 1619 1620 public MediaImage( 1621 ByteBuffer buffer, ByteBuffer info, boolean readOnly, 1622 long timestamp, int xOffset, int yOffset, Rect cropRect) { 1623 mFormat = ImageFormat.YUV_420_888; 1624 mTimestamp = timestamp; 1625 mIsValid = true; 1626 mIsReadOnly = buffer.isReadOnly(); 1627 mBuffer = buffer.duplicate(); 1628 if (cropRect != null) { 1629 cropRect.offset(-xOffset, -yOffset); 1630 } 1631 mCropRect = cropRect; 1632 1633 // save offsets and info 1634 mXOffset = xOffset; 1635 mYOffset = yOffset; 1636 mInfo = info; 1637 1638 // read media-info. the size of media info can be 80 or 156 depending on 1639 // whether it was created on a 32- or 64-bit process. See MediaImage 1640 if (info.remaining() == 80 || info.remaining() == 156) { 1641 boolean sizeIsLong = info.remaining() == 156; 1642 int type = info.getInt(); 1643 if (type != TYPE_YUV) { 1644 throw new UnsupportedOperationException("unsupported type: " + type); 1645 } 1646 int numPlanes = readInt(info, sizeIsLong); 1647 if (numPlanes != 3) { 1648 throw new RuntimeException("unexpected number of planes: " + numPlanes); 1649 } 1650 mWidth = readInt(info, sizeIsLong); 1651 mHeight = readInt(info, sizeIsLong); 1652 if (mWidth < 1 || mHeight < 1) { 1653 throw new UnsupportedOperationException( 1654 "unsupported size: " + mWidth + "x" + mHeight); 1655 } 1656 int bitDepth = readInt(info, sizeIsLong); 1657 if (bitDepth != 8) { 1658 throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth); 1659 } 1660 mPlanes = new MediaPlane[numPlanes]; 1661 for (int ix = 0; ix < numPlanes; ix++) { 1662 int planeOffset = readInt(info, sizeIsLong); 1663 int colInc = readInt(info, sizeIsLong); 1664 int rowInc = readInt(info, sizeIsLong); 1665 int horiz = readInt(info, sizeIsLong); 1666 int vert = readInt(info, sizeIsLong); 1667 if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) { 1668 throw new UnsupportedOperationException("unexpected subsampling: " 1669 + horiz + "x" + vert + " on plane " + ix); 1670 } 1671 1672 buffer.clear(); 1673 buffer.position(mBuffer.position() + planeOffset 1674 + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); 1675 buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8) 1676 + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc); 1677 mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc); 1678 } 1679 } else { 1680 throw new UnsupportedOperationException( 1681 "unsupported info length: " + info.remaining()); 1682 } 1683 } 1684 1685 private class MediaPlane extends Plane { 1686 public MediaPlane(ByteBuffer buffer, int rowInc, int colInc) { 1687 mData = buffer; 1688 mRowInc = rowInc; 1689 mColInc = colInc; 1690 } 1691 1692 @Override 1693 public int getRowStride() { 1694 checkValid(); 1695 return mRowInc; 1696 } 1697 1698 @Override 1699 public int getPixelStride() { 1700 checkValid(); 1701 return mColInc; 1702 } 1703 1704 @Override 1705 public ByteBuffer getBuffer() { 1706 checkValid(); 1707 return mData; 1708 } 1709 1710 private final int mRowInc; 1711 private final int mColInc; 1712 private final ByteBuffer mData; 1713 } 1714 } 1715} 1716