MediaSync.java revision 25b802d47249702b9e5d175b3e7144934b67553d
1/* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.media; 18 19import android.annotation.IntDef; 20import android.annotation.NonNull; 21import android.annotation.Nullable; 22import android.media.AudioTrack; 23import android.media.PlaybackSettings; 24import android.os.Handler; 25import android.os.Looper; 26import android.os.Message; 27import android.view.Surface; 28 29import java.lang.annotation.Retention; 30import java.lang.annotation.RetentionPolicy; 31import java.nio.ByteBuffer; 32import java.util.concurrent.TimeUnit; 33import java.util.LinkedList; 34import java.util.List; 35 36/** 37 * MediaSync class can be used to synchronously playback audio and video streams. 38 * It can be used to play audio-only or video-only stream, too. 39 * 40 * <p>MediaSync is generally used like this: 41 * <pre> 42 * MediaSync sync = new MediaSync(); 43 * sync.setSurface(surface); 44 * Surface inputSurface = sync.createInputSurface(); 45 * ... 46 * // MediaCodec videoDecoder = ...; 47 * videoDecoder.configure(format, inputSurface, ...); 48 * ... 49 * sync.setAudioTrack(audioTrack); 50 * sync.setCallback(new MediaSync.Callback() { 51 * {@literal @Override} 52 * public void onReturnAudioBuffer(MediaSync sync, ByteBuffer audioBuffer, int bufferIndex) { 53 * ... 54 * } 55 * }, null); 56 * // This needs to be done since sync is paused on creation. 57 * sync.setPlaybackRate(1.0f, MediaSync.PLAYBACK_RATE_AUDIO_MODE_RESAMPLE); 58 * 59 * for (;;) { 60 * ... 61 * // send video frames to surface for rendering, e.g., call 62 * // videoDecoder.releaseOutputBuffer(videoOutputBufferIx, videoPresentationTimeNs); 63 * // More details are available as below. 64 * ... 65 * sync.queueAudio(audioByteBuffer, bufferIndex, size, audioPresentationTimeUs); // non-blocking. 66 * // The audioByteBuffer and bufferIndex will be returned via callback. 67 * // More details are available as below. 68 * ... 69 * ... 70 * } 71 * sync.setPlaybackRate(0.0f, MediaSync.PLAYBACK_RATE_AUDIO_MODE_RESAMPLE); 72 * sync.release(); 73 * sync = null; 74 * 75 * // The following code snippet illustrates how video/audio raw frames are created by 76 * // MediaCodec's, how they are fed to MediaSync and how they are returned by MediaSync. 77 * // This is the callback from MediaCodec. 78 * onOutputBufferAvailable(MediaCodec codec, int bufferIndex, BufferInfo info) { 79 * // ... 80 * if (codec == videoDecoder) { 81 * // surface timestamp must contain media presentation time in nanoseconds. 82 * codec.releaseOutputBuffer(bufferIndex, 1000 * info.presentationTime); 83 * } else { 84 * ByteBuffer audioByteBuffer = codec.getOutputBuffer(bufferIndex); 85 * sync.queueByteBuffer(audioByteBuffer, bufferIndex, info.size, info.presentationTime); 86 * } 87 * // ... 88 * } 89 * 90 * // This is the callback from MediaSync. 91 * onReturnAudioBuffer(MediaSync sync, ByteBuffer buffer, int bufferIndex) { 92 * // ... 93 * audioDecoder.releaseBuffer(bufferIndex, false); 94 * // ... 95 * } 96 * 97 * </pre> 98 * 99 * The client needs to configure corresponding sink by setting the Surface and/or AudioTrack 100 * based on the stream type it will play. 101 * <p> 102 * For video, the client needs to call {@link #createInputSurface} to obtain a surface on 103 * which it will render video frames. 104 * <p> 105 * For audio, the client needs to set up audio track correctly, e.g., using {@link 106 * AudioTrack#MODE_STREAM}. The audio buffers are sent to MediaSync directly via {@link 107 * #queueAudio}, and are returned to the client via {@link Callback#onReturnAudioBuffer} 108 * asynchronously. The client should not modify an audio buffer till it's returned. 109 * <p> 110 * The client can optionally pre-fill audio/video buffers by setting playback rate to 0.0, 111 * and then feed audio/video buffers to corresponding components. This can reduce possible 112 * initial underrun. 113 * <p> 114 */ 115final public class MediaSync { 116 /** 117 * MediaSync callback interface. Used to notify the user asynchronously 118 * of various MediaSync events. 119 */ 120 public static abstract class Callback { 121 /** 122 * Called when returning an audio buffer which has been consumed. 123 * 124 * @param sync The MediaSync object. 125 * @param audioBuffer The returned audio buffer. 126 * @param bufferIndex The index associated with the audio buffer 127 */ 128 public abstract void onReturnAudioBuffer( 129 @NonNull MediaSync sync, @NonNull ByteBuffer audioBuffer, int bufferIndex); 130 } 131 132 private static final String TAG = "MediaSync"; 133 134 private static final int EVENT_CALLBACK = 1; 135 private static final int EVENT_SET_CALLBACK = 2; 136 137 private static final int CB_RETURN_AUDIO_BUFFER = 1; 138 139 private static class AudioBuffer { 140 public ByteBuffer mByteBuffer; 141 public int mBufferIndex; 142 public int mSizeInBytes; 143 long mPresentationTimeUs; 144 145 public AudioBuffer(@NonNull ByteBuffer byteBuffer, int bufferIndex, 146 int sizeInBytes, long presentationTimeUs) { 147 mByteBuffer = byteBuffer; 148 mBufferIndex = bufferIndex; 149 mSizeInBytes = sizeInBytes; 150 mPresentationTimeUs = presentationTimeUs; 151 } 152 } 153 154 private final Object mCallbackLock = new Object(); 155 private Handler mCallbackHandler = null; 156 private MediaSync.Callback mCallback = null; 157 158 private Thread mAudioThread = null; 159 // Created on mAudioThread when mAudioThread is started. When used on user thread, they should 160 // be guarded by checking mAudioThread. 161 private Handler mAudioHandler = null; 162 private Looper mAudioLooper = null; 163 164 private final Object mAudioLock = new Object(); 165 private AudioTrack mAudioTrack = null; 166 private List<AudioBuffer> mAudioBuffers = new LinkedList<AudioBuffer>(); 167 private float mPlaybackRate = 0.0f; 168 169 private long mNativeContext; 170 171 /** 172 * Class constructor. On creation, MediaSync is paused, i.e., playback rate is 0.0f. 173 */ 174 public MediaSync() { 175 native_setup(); 176 } 177 178 private native final void native_setup(); 179 180 @Override 181 protected void finalize() { 182 native_finalize(); 183 } 184 185 private native final void native_finalize(); 186 187 /** 188 * Make sure you call this when you're done to free up any opened 189 * component instance instead of relying on the garbage collector 190 * to do this for you at some point in the future. 191 */ 192 public final void release() { 193 returnAudioBuffers(); 194 if (mAudioThread != null) { 195 if (mAudioLooper != null) { 196 mAudioLooper.quit(); 197 } 198 } 199 setCallback(null, null); 200 native_release(); 201 } 202 203 private native final void native_release(); 204 205 /** 206 * Sets an asynchronous callback for actionable MediaSync events. 207 * <p> 208 * This method can be called multiple times to update a previously set callback. If the 209 * handler is changed, undelivered notifications scheduled for the old handler may be dropped. 210 * <p> 211 * <b>Do not call this inside callback.</b> 212 * 213 * @param cb The callback that will run. Use {@code null} to stop receiving callbacks. 214 * @param handler The Handler that will run the callback. Use {@code null} to use MediaSync's 215 * internal handler if it exists. 216 */ 217 public void setCallback(@Nullable /* MediaSync. */ Callback cb, @Nullable Handler handler) { 218 synchronized(mCallbackLock) { 219 if (handler != null) { 220 mCallbackHandler = handler; 221 } else { 222 Looper looper; 223 if ((looper = Looper.myLooper()) == null) { 224 looper = Looper.getMainLooper(); 225 } 226 if (looper == null) { 227 mCallbackHandler = null; 228 } else { 229 mCallbackHandler = new Handler(looper); 230 } 231 } 232 233 mCallback = cb; 234 } 235 } 236 237 /** 238 * Sets the output surface for MediaSync. 239 * <p> 240 * Currently, this is only supported in the Initialized state. 241 * 242 * @param surface Specify a surface on which to render the video data. 243 * @throws IllegalArgumentException if the surface has been released, is invalid, 244 * or can not be connected. 245 * @throws IllegalStateException if setting the surface is not supported, e.g. 246 * not in the Initialized state, or another surface has already been configured. 247 */ 248 public void setSurface(@Nullable Surface surface) { 249 native_configureSurface(surface); 250 } 251 252 private native final void native_configureSurface(@Nullable Surface surface); 253 254 /** 255 * Sets the audio track for MediaSync. 256 * <p> 257 * Currently, this is only supported in the Initialized state. 258 * 259 * @param audioTrack Specify an AudioTrack through which to render the audio data. 260 * @throws IllegalArgumentException if the audioTrack has been released, or is invalid. 261 * @throws IllegalStateException if setting the audio track is not supported, e.g. 262 * not in the Initialized state, or another audio track has already been configured. 263 */ 264 public void setAudioTrack(@Nullable AudioTrack audioTrack) { 265 // AudioTrack has sanity check for configured sample rate. 266 int nativeSampleRateInHz = (audioTrack == null ? 0 : audioTrack.getSampleRate()); 267 268 native_configureAudioTrack(audioTrack, nativeSampleRateInHz); 269 mAudioTrack = audioTrack; 270 if (audioTrack != null && mAudioThread == null) { 271 createAudioThread(); 272 } 273 } 274 275 private native final void native_configureAudioTrack( 276 @Nullable AudioTrack audioTrack, int nativeSampleRateInHz); 277 278 /** 279 * Requests a Surface to use as the input. This may only be called after 280 * {@link #setSurface}. 281 * <p> 282 * The application is responsible for calling release() on the Surface when 283 * done. 284 * @throws IllegalStateException if not configured, or another input surface has 285 * already been created. 286 */ 287 @NonNull 288 public native final Surface createInputSurface(); 289 290 /** 291 * Resample audio data when changing playback speed. 292 * <p> 293 * Resample the waveform based on the requested playback rate to get 294 * a new waveform, and play back the new waveform at the original sampling 295 * frequency. 296 * <p><ul> 297 * <li>When rate is larger than 1.0, pitch becomes higher. 298 * <li>When rate is smaller than 1.0, pitch becomes lower. 299 * </ul> 300 */ 301 public static final int PLAYBACK_RATE_AUDIO_MODE_RESAMPLE = 2; 302 303 /** 304 * Time stretch audio when changing playback speed. 305 * <p> 306 * Time stretching changes the duration of the audio samples without 307 * affecting their pitch. This is only supported for a limited range 308 * of playback speeds, e.g. from 1/2x to 2x. If the rate is adjusted 309 * beyond this limit, the rate change will fail. 310 */ 311 public static final int PLAYBACK_RATE_AUDIO_MODE_STRETCH = 1; 312 313 /** 314 * Time stretch audio when changing playback speed, and may mute if 315 * stretching is no longer supported. 316 * <p> 317 * Time stretching changes the duration of the audio samples without 318 * affecting their pitch. This is only supported for a limited range 319 * of playback speeds, e.g. from 1/2x to 2x. When it is no longer 320 * supported, the audio may be muted. Using this mode will not fail 321 * for non-negative playback rates. 322 */ 323 public static final int PLAYBACK_RATE_AUDIO_MODE_DEFAULT = 0; 324 325 /** @hide */ 326 @IntDef( 327 value = { 328 PLAYBACK_RATE_AUDIO_MODE_DEFAULT, 329 PLAYBACK_RATE_AUDIO_MODE_STRETCH, 330 PLAYBACK_RATE_AUDIO_MODE_RESAMPLE, 331 }) 332 @Retention(RetentionPolicy.SOURCE) 333 public @interface PlaybackRateAudioMode {} 334 335 /** 336 * Sets playback rate and audio mode. 337 * 338 * @param rate the ratio between desired playback rate and normal one. 1.0 means normal 339 * playback speed. 0.0 means pause. Value larger than 1.0 means faster playback, 340 * while value between 0.0 and 1.0 for slower playback. <b>Note:</b> the normal rate 341 * does not change as a result of this call. To restore the original rate at any time, 342 * use 1.0. 343 * @param audioMode audio playback mode. Must be one of the supported 344 * audio modes. 345 * 346 * @throws IllegalStateException if the internal sync engine or the audio track has not 347 * been initialized. 348 * @throws IllegalArgumentException if audioMode is not supported. 349 */ 350 public void setPlaybackRate(float rate, @PlaybackRateAudioMode int audioMode) { 351 PlaybackSettings rateSettings = new PlaybackSettings(); 352 rateSettings.allowDefaults(); 353 switch (audioMode) { 354 case PLAYBACK_RATE_AUDIO_MODE_DEFAULT: 355 rateSettings.setSpeed(rate).setPitch(1.0f); 356 break; 357 case PLAYBACK_RATE_AUDIO_MODE_STRETCH: 358 rateSettings.setSpeed(rate).setPitch(1.0f) 359 .setAudioFallbackMode(rateSettings.AUDIO_FALLBACK_MODE_FAIL); 360 break; 361 case PLAYBACK_RATE_AUDIO_MODE_RESAMPLE: 362 rateSettings.setSpeed(rate).setPitch(rate); 363 break; 364 default: 365 { 366 final String msg = "Audio playback mode " + audioMode + " is not supported"; 367 throw new IllegalArgumentException(msg); 368 } 369 } 370 setPlaybackSettings(rateSettings); 371 } 372 373 /** 374 * Sets playback rate using {@link PlaybackSettings}. 375 * <p> 376 * When using MediaSync with {@link AudioTrack}, set playback settings using this 377 * call instead of calling it directly on the track, so that the sync is aware of 378 * the settings change. 379 * <p> 380 * This call also works if there is no audio track. 381 * 382 * @param settings the playback settings to use. {@link PlaybackSettings#getSpeed 383 * Speed} is the ratio between desired playback rate and normal one. 1.0 means 384 * normal playback speed. 0.0 means pause. Value larger than 1.0 means faster playback, 385 * while value between 0.0 and 1.0 for slower playback. <b>Note:</b> the normal rate 386 * does not change as a result of this call. To restore the original rate at any time, 387 * use speed of 1.0. 388 * 389 * @throws IllegalStateException if the internal sync engine or the audio track has not 390 * been initialized. 391 * @throws IllegalArgumentException if the settings are not supported. 392 */ 393 public void setPlaybackSettings(@NonNull PlaybackSettings settings) { 394 float rate; 395 try { 396 rate = settings.getSpeed(); 397 398 // rate is specified 399 if (mAudioTrack != null) { 400 try { 401 if (rate == 0.0) { 402 mAudioTrack.pause(); 403 } else { 404 mAudioTrack.setPlaybackSettings(settings); 405 mAudioTrack.play(); 406 } 407 } catch (IllegalStateException e) { 408 throw e; 409 } 410 } 411 412 synchronized(mAudioLock) { 413 mPlaybackRate = rate; 414 } 415 if (mPlaybackRate != 0.0 && mAudioThread != null) { 416 postRenderAudio(0); 417 } 418 native_setPlaybackRate(mPlaybackRate); 419 } catch (IllegalStateException e) { 420 // rate is not specified; still, propagate settings to audio track 421 if (mAudioTrack != null) { 422 mAudioTrack.setPlaybackSettings(settings); 423 } 424 } 425 } 426 427 /** 428 * Gets the playback rate using {@link PlaybackSettings}. 429 * 430 * @return the playback rate being used. 431 * 432 * @throws IllegalStateException if the internal sync engine or the audio track has not 433 * been initialized. 434 */ 435 @NonNull 436 public PlaybackSettings getPlaybackSettings() { 437 if (mAudioTrack != null) { 438 return mAudioTrack.getPlaybackSettings(); 439 } else { 440 PlaybackSettings settings = new PlaybackSettings(); 441 settings.allowDefaults(); 442 settings.setSpeed(mPlaybackRate); 443 return settings; 444 } 445 } 446 447 private native final void native_setPlaybackRate(float rate); 448 449 /** 450 * Sets A/V sync mode. 451 * 452 * @param settings the A/V sync settings to apply 453 * 454 * @throws IllegalStateException if the internal player engine has not been 455 * initialized. 456 * @throws IllegalArgumentException if settings are not supported. 457 */ 458 public native void setSyncSettings(@NonNull SyncSettings settings); 459 460 /** 461 * Gets the A/V sync mode. 462 * 463 * @return the A/V sync settings 464 * 465 * @throws IllegalStateException if the internal player engine has not been 466 * initialized. 467 */ 468 @NonNull 469 public native SyncSettings getSyncSettings(); 470 471 /** 472 * Flushes all buffers from the sync object. 473 * <p> 474 * No callbacks are received for the flushed buffers. 475 * 476 * @throws IllegalStateException if the internal player engine has not been 477 * initialized. 478 */ 479 public void flush() { 480 synchronized(mAudioLock) { 481 mAudioBuffers.clear(); 482 mCallbackHandler.removeCallbacksAndMessages(null); 483 } 484 // TODO implement this for surface buffers. 485 } 486 487 /** 488 * Get current playback position. 489 * <p> 490 * The MediaTimestamp represents how the media time correlates to the system time in 491 * a linear fashion. It contains the media time and system timestamp of an anchor frame 492 * ({@link MediaTimestamp#mediaTimeUs} and {@link MediaTimestamp#nanoTime}) 493 * and the speed of the media clock ({@link MediaTimestamp#clockRate}). 494 * <p> 495 * During regular playback, the media time moves fairly constantly (though the 496 * anchor frame may be rebased to a current system time, the linear correlation stays 497 * steady). Therefore, this method does not need to be called often. 498 * <p> 499 * To help users to get current playback position, this method always returns the timestamp of 500 * just-rendered frame, i.e., {@link System#nanoTime} and its corresponding media time. They 501 * can be used as current playback position. 502 * 503 * @return a MediaTimestamp object if a timestamp is available, or {@code null} if no timestamp 504 * is available, e.g. because the media sync has not been initialized. 505 */ 506 @Nullable 507 public MediaTimestamp getTimestamp() 508 { 509 try { 510 // TODO: create the timestamp in native 511 MediaTimestamp timestamp = new MediaTimestamp(); 512 if (native_getTimestamp(timestamp)) { 513 return timestamp; 514 } else { 515 return null; 516 } 517 } catch (IllegalStateException e) { 518 return null; 519 } 520 } 521 522 private native final boolean native_getTimestamp(@NonNull MediaTimestamp timestamp); 523 524 /** 525 * Queues the audio data asynchronously for playback (AudioTrack must be in streaming mode). 526 * @param audioData the buffer that holds the data to play. This buffer will be returned 527 * to the client via registered callback. 528 * @param bufferIndex the buffer index used to identify audioData. It will be returned to 529 * the client along with audioData. This helps applications to keep track of audioData. 530 * @param sizeInBytes number of bytes to queue. 531 * @param presentationTimeUs the presentation timestamp in microseconds for the first frame 532 * in the buffer. 533 * @throws IllegalStateException if audio track is not configured or internal configureation 534 * has not been done correctly. 535 */ 536 public void queueAudio( 537 @NonNull ByteBuffer audioData, int bufferIndex, int sizeInBytes, 538 long presentationTimeUs) { 539 if (mAudioTrack == null || mAudioThread == null) { 540 throw new IllegalStateException( 541 "AudioTrack is NOT configured or audio thread is not created"); 542 } 543 544 synchronized(mAudioLock) { 545 mAudioBuffers.add(new AudioBuffer( 546 audioData, bufferIndex, sizeInBytes, presentationTimeUs)); 547 } 548 549 if (mPlaybackRate != 0.0) { 550 postRenderAudio(0); 551 } 552 } 553 554 // When called on user thread, make sure to check mAudioThread != null. 555 private void postRenderAudio(long delayMillis) { 556 mAudioHandler.postDelayed(new Runnable() { 557 public void run() { 558 synchronized(mAudioLock) { 559 if (mPlaybackRate == 0.0) { 560 return; 561 } 562 563 if (mAudioBuffers.isEmpty()) { 564 return; 565 } 566 567 AudioBuffer audioBuffer = mAudioBuffers.get(0); 568 int sizeWritten = mAudioTrack.write( 569 audioBuffer.mByteBuffer, 570 audioBuffer.mSizeInBytes, 571 AudioTrack.WRITE_NON_BLOCKING); 572 if (sizeWritten > 0) { 573 if (audioBuffer.mPresentationTimeUs != -1) { 574 native_updateQueuedAudioData( 575 audioBuffer.mSizeInBytes, audioBuffer.mPresentationTimeUs); 576 audioBuffer.mPresentationTimeUs = -1; 577 } 578 579 if (sizeWritten == audioBuffer.mSizeInBytes) { 580 postReturnByteBuffer(audioBuffer); 581 mAudioBuffers.remove(0); 582 if (!mAudioBuffers.isEmpty()) { 583 postRenderAudio(0); 584 } 585 return; 586 } 587 588 audioBuffer.mSizeInBytes -= sizeWritten; 589 } 590 long pendingTimeMs = TimeUnit.MICROSECONDS.toMillis( 591 native_getPlayTimeForPendingAudioFrames()); 592 postRenderAudio(pendingTimeMs / 2); 593 } 594 } 595 }, delayMillis); 596 } 597 598 private native final void native_updateQueuedAudioData( 599 int sizeInBytes, long presentationTimeUs); 600 601 private native final long native_getPlayTimeForPendingAudioFrames(); 602 603 private final void postReturnByteBuffer(@NonNull final AudioBuffer audioBuffer) { 604 synchronized(mCallbackLock) { 605 if (mCallbackHandler != null) { 606 final MediaSync sync = this; 607 mCallbackHandler.post(new Runnable() { 608 public void run() { 609 synchronized(mCallbackLock) { 610 if (mCallbackHandler == null 611 || mCallbackHandler.getLooper().getThread() 612 != Thread.currentThread()) { 613 // callback handler has been changed. 614 return; 615 } 616 if (mCallback != null) { 617 mCallback.onReturnAudioBuffer(sync, audioBuffer.mByteBuffer, 618 audioBuffer.mBufferIndex); 619 } 620 } 621 } 622 }); 623 } 624 } 625 } 626 627 private final void returnAudioBuffers() { 628 synchronized(mAudioLock) { 629 for (AudioBuffer audioBuffer: mAudioBuffers) { 630 postReturnByteBuffer(audioBuffer); 631 } 632 mAudioBuffers.clear(); 633 } 634 } 635 636 private void createAudioThread() { 637 mAudioThread = new Thread() { 638 @Override 639 public void run() { 640 Looper.prepare(); 641 synchronized(mAudioLock) { 642 mAudioLooper = Looper.myLooper(); 643 mAudioHandler = new Handler(); 644 mAudioLock.notify(); 645 } 646 Looper.loop(); 647 } 648 }; 649 mAudioThread.start(); 650 651 synchronized(mAudioLock) { 652 try { 653 mAudioLock.wait(); 654 } catch(InterruptedException e) { 655 } 656 } 657 } 658 659 static { 660 System.loadLibrary("media_jni"); 661 native_init(); 662 } 663 664 private static native final void native_init(); 665} 666