MediaSync.java revision 217ec0adfc35302a6cc6b04bc78bf8fd82ffc8a5
1/* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.media; 18 19import android.annotation.IntDef; 20import android.annotation.NonNull; 21import android.media.AudioTrack; 22import android.os.Handler; 23import android.os.Looper; 24import android.os.Message; 25import android.view.Surface; 26 27import java.lang.annotation.Retention; 28import java.lang.annotation.RetentionPolicy; 29import java.nio.ByteBuffer; 30import java.util.LinkedList; 31import java.util.List; 32 33/** 34 * MediaSync class can be used to synchronously playback audio and video streams. 35 * It can be used to play audio-only or video-only stream, too. 36 * 37 * <p>MediaSync is generally used like this: 38 * <pre> 39 * MediaSync sync = new MediaSync(); 40 * sync.configureSurface(surface); 41 * Surface inputSurface = sync.createInputSurface(); 42 * ... 43 * // MediaCodec videoDecoder = ...; 44 * videoDecoder.configure(format, inputSurface, ...); 45 * ... 46 * sync.configureAudioTrack(audioTrack, nativeSampleRateInHz); 47 * sync.setCallback(new MediaSync.Callback() { 48 * \@Override 49 * public void onReturnAudioBuffer(MediaSync sync, ByteBuffer audioBuffer, int bufferIndex) { 50 * ... 51 * } 52 * }); 53 * // This needs to be done since sync is paused on creation. 54 * sync.setPlaybackRate(1.0f, MediaSync.PLAYBACK_RATE_AUDIO_MODE_RESAMPLE); 55 * 56 * for (;;) { 57 * ... 58 * // send video frames to surface for rendering, e.g., call 59 * // videoDecoder.releaseOutputBuffer(videoOutputBufferIx, videoPresentationTimeNs); 60 * // More details are available as below. 61 * ... 62 * sync.queueAudio(audioByteBuffer, bufferIndex, size, audioPresentationTimeUs); // non-blocking. 63 * // The audioByteBuffer and bufferIndex will be returned via callback. 64 * // More details are available as below. 65 * ... 66 * ... 67 * } 68 * sync.setPlaybackRate(0.0f, MediaSync.PLAYBACK_RATE_AUDIO_MODE_RESAMPLE); 69 * sync.release(); 70 * sync = null; 71 * 72 * // The following code snippet illustrates how video/audio raw frames are created by 73 * // MediaCodec's, how they are fed to MediaSync and how they are returned by MediaSync. 74 * // This is the callback from MediaCodec. 75 * onOutputBufferAvailable(MediaCodec codec, int bufferIndex, BufferInfo info) { 76 * // ... 77 * if (codec == videoDecoder) { 78 * // surface timestamp must contain media presentation time in nanoseconds. 79 * codec.releaseOutputBuffer(bufferIndex, 1000 * info.presentationTime); 80 * } else { 81 * ByteBuffer audioByteBuffer = codec.getOutputBuffer(bufferIndex); 82 * sync.queueByteBuffer(audioByteBuffer, bufferIndex, info.size, info.presentationTime); 83 * } 84 * // ... 85 * } 86 * 87 * // This is the callback from MediaSync. 88 * onReturnAudioBuffer(MediaSync sync, ByteBuffer buffer, int bufferIndex) { 89 * // ... 90 * audioDecoder.releaseBuffer(bufferIndex, false); 91 * // ... 92 * } 93 * 94 * </pre> 95 * 96 * The client needs to configure corresponding sink (i.e., Surface and AudioTrack) based on 97 * the stream type it will play. 98 * <p> 99 * For video, the client needs to call {@link #createInputSurface} to obtain a surface on 100 * which it will render video frames. 101 * <p> 102 * For audio, the client needs to set up audio track correctly, e.g., using {@link 103 * AudioTrack#MODE_STREAM}. The audio buffers are sent to MediaSync directly via {@link 104 * #queueAudio}, and are returned to the client via {@link Callback#onReturnAudioBuffer} 105 * asynchronously. The client should not modify an audio buffer till it's returned. 106 * <p> 107 * The client can optionally pre-fill audio/video buffers by setting playback rate to 0.0, 108 * and then feed audio/video buffers to corresponding components. This can reduce possible 109 * initial underrun. 110 * <p> 111 */ 112final public class MediaSync { 113 /** 114 * MediaSync callback interface. Used to notify the user asynchronously 115 * of various MediaSync events. 116 */ 117 public static abstract class Callback { 118 /** 119 * Called when returning an audio buffer which has been consumed. 120 * 121 * @param sync The MediaSync object. 122 * @param audioBuffer The returned audio buffer. 123 */ 124 public abstract void onReturnAudioBuffer( 125 MediaSync sync, ByteBuffer audioBuffer, int bufferIndex); 126 } 127 128 private static final String TAG = "MediaSync"; 129 130 private static final int EVENT_CALLBACK = 1; 131 private static final int EVENT_SET_CALLBACK = 2; 132 133 private static final int CB_RETURN_AUDIO_BUFFER = 1; 134 135 private static class AudioBuffer { 136 public ByteBuffer mByteBuffer; 137 public int mBufferIndex; 138 public int mSizeInBytes; 139 long mPresentationTimeUs; 140 141 public AudioBuffer(ByteBuffer byteBuffer, int bufferIndex, 142 int sizeInBytes, long presentationTimeUs) { 143 mByteBuffer = byteBuffer; 144 mBufferIndex = bufferIndex; 145 mSizeInBytes = sizeInBytes; 146 mPresentationTimeUs = presentationTimeUs; 147 } 148 } 149 150 private final Object mCallbackLock = new Object(); 151 private Handler mCallbackHandler = null; 152 private MediaSync.Callback mCallback = null; 153 154 private int mNativeSampleRateInHz = 0; 155 156 private Thread mAudioThread = null; 157 // Created on mAudioThread when mAudioThread is started. When used on user thread, they should 158 // be guarded by checking mAudioThread. 159 private Handler mAudioHandler = null; 160 private Looper mAudioLooper = null; 161 162 private final Object mAudioLock = new Object(); 163 private AudioTrack mAudioTrack = null; 164 private List<AudioBuffer> mAudioBuffers = new LinkedList<AudioBuffer>(); 165 private float mPlaybackRate = 0.0f; 166 167 private long mNativeContext; 168 169 /** 170 * Class constructor. On creation, MediaSync is paused, i.e., playback rate is 0.0f. 171 */ 172 public MediaSync() { 173 native_setup(); 174 } 175 176 private native final void native_setup(); 177 178 @Override 179 protected void finalize() { 180 native_finalize(); 181 } 182 183 private native final void native_finalize(); 184 185 /** 186 * Make sure you call this when you're done to free up any opened 187 * component instance instead of relying on the garbage collector 188 * to do this for you at some point in the future. 189 */ 190 public final void release() { 191 returnAudioBuffers(); 192 if (mAudioThread != null) { 193 if (mAudioLooper != null) { 194 mAudioLooper.quit(); 195 } 196 } 197 setCallback(null, null); 198 native_release(); 199 } 200 201 private native final void native_release(); 202 203 /** 204 * Sets an asynchronous callback for actionable MediaSync events. 205 * It shouldn't be called inside callback. 206 * 207 * @param cb The callback that will run. 208 * @param handler The Handler that will run the callback. Using null means to use MediaSync's 209 * internal handler if it exists. 210 */ 211 public void setCallback(/* MediaSync. */ Callback cb, Handler handler) { 212 synchronized(mCallbackLock) { 213 if (handler != null) { 214 mCallbackHandler = handler; 215 } else { 216 Looper looper; 217 if ((looper = Looper.myLooper()) == null) { 218 looper = Looper.getMainLooper(); 219 } 220 if (looper == null) { 221 mCallbackHandler = null; 222 } else { 223 mCallbackHandler = new Handler(looper); 224 } 225 } 226 227 mCallback = cb; 228 } 229 } 230 231 /** 232 * Configures the output surface for MediaSync. 233 * 234 * @param surface Specify a surface on which to render the video data. 235 * @throws IllegalArgumentException if the surface has been released, or is invalid. 236 * or can not be connected. 237 * @throws IllegalStateException if not in the Initialized state, or another surface 238 * has already been configured. 239 */ 240 public void configureSurface(Surface surface) { 241 native_configureSurface(surface); 242 } 243 244 private native final void native_configureSurface(Surface surface); 245 246 /** 247 * Configures the audio track for MediaSync. 248 * 249 * @param audioTrack Specify an AudioTrack through which to render the audio data. 250 * @throws IllegalArgumentException if the audioTrack has been released, or is invalid, 251 * or nativeSampleRateInHz is invalid. 252 * @throws IllegalStateException if not in the Initialized state, or another audio track 253 * has already been configured. 254 */ 255 public void configureAudioTrack(AudioTrack audioTrack, int nativeSampleRateInHz) { 256 if (audioTrack != null && nativeSampleRateInHz <= 0) { 257 final String msg = "Native sample rate " + nativeSampleRateInHz + " is invalid"; 258 throw new IllegalArgumentException(msg); 259 } 260 native_configureAudioTrack(audioTrack, nativeSampleRateInHz); 261 mAudioTrack = audioTrack; 262 mNativeSampleRateInHz = nativeSampleRateInHz; 263 if (mAudioThread == null) { 264 createAudioThread(); 265 } 266 } 267 268 private native final void native_configureAudioTrack( 269 AudioTrack audioTrack, int nativeSampleRateInHz); 270 271 /** 272 * Requests a Surface to use as the input. This may only be called after 273 * {@link #configureSurface}. 274 * <p> 275 * The application is responsible for calling release() on the Surface when 276 * done. 277 * @throws IllegalStateException if not configured, or another input surface has 278 * already been created. 279 */ 280 public native final Surface createInputSurface(); 281 282 /** 283 * Specifies resampling as audio mode for variable rate playback, i.e., 284 * resample the waveform based on the requested playback rate to get 285 * a new waveform, and play back the new waveform at the original sampling 286 * frequency. 287 * When rate is larger than 1.0, pitch becomes higher. 288 * When rate is smaller than 1.0, pitch becomes lower. 289 */ 290 public static final int PLAYBACK_RATE_AUDIO_MODE_RESAMPLE = 0; 291 292 /** 293 * Specifies time stretching as audio mode for variable rate playback. 294 * Time stretching changes the duration of the audio samples without 295 * affecting its pitch. 296 * FIXME: implement time strectching. 297 * @hide 298 */ 299 public static final int PLAYBACK_RATE_AUDIO_MODE_STRETCH = 1; 300 301 /** @hide */ 302 @IntDef( 303 value = { 304 PLAYBACK_RATE_AUDIO_MODE_RESAMPLE, 305 PLAYBACK_RATE_AUDIO_MODE_STRETCH }) 306 @Retention(RetentionPolicy.SOURCE) 307 public @interface PlaybackRateAudioMode {} 308 309 /** 310 * Sets playback rate. It does same as {@link #setPlaybackRate(float, int)}, 311 * except that it always uses {@link #PLAYBACK_RATE_AUDIO_MODE_STRETCH} for audioMode. 312 * 313 * @param rate the ratio between desired playback rate and normal one. 1.0 means normal 314 * playback speed. 0.0 means stop or pause. Value larger than 1.0 means faster playback, 315 * while value between 0.0 and 1.0 for slower playback. 316 * 317 * @throws IllegalStateException if the internal sync engine or the audio track has not 318 * been initialized. 319 * TODO: unhide when PLAYBACK_RATE_AUDIO_MODE_STRETCH is supported. 320 * @hide 321 */ 322 public void setPlaybackRate(float rate) { 323 setPlaybackRate(rate, PLAYBACK_RATE_AUDIO_MODE_STRETCH); 324 } 325 326 /** 327 * Sets playback rate and audio mode. 328 * 329 * <p> The supported audio modes are: 330 * <ul> 331 * <li> {@link #PLAYBACK_RATE_AUDIO_MODE_RESAMPLE} 332 * </ul> 333 * 334 * @param rate the ratio between desired playback rate and normal one. 1.0 means normal 335 * playback speed. 0.0 means stop or pause. Value larger than 1.0 means faster playback, 336 * while value between 0.0 and 1.0 for slower playback. 337 * @param audioMode audio playback mode. Must be one of the supported 338 * audio modes. 339 * 340 * @throws IllegalStateException if the internal sync engine or the audio track has not 341 * been initialized. 342 * @throws IllegalArgumentException if audioMode is not supported. 343 */ 344 public void setPlaybackRate(float rate, @PlaybackRateAudioMode int audioMode) { 345 if (!isAudioPlaybackModeSupported(audioMode)) { 346 final String msg = "Audio playback mode " + audioMode + " is not supported"; 347 throw new IllegalArgumentException(msg); 348 } 349 350 int status = AudioTrack.SUCCESS; 351 if (mAudioTrack != null) { 352 int playbackSampleRate = (int)(rate * mNativeSampleRateInHz + 0.5); 353 rate = playbackSampleRate / (float)mNativeSampleRateInHz; 354 355 try { 356 if (rate == 0.0) { 357 mAudioTrack.pause(); 358 } else { 359 status = mAudioTrack.setPlaybackRate(playbackSampleRate); 360 mAudioTrack.play(); 361 } 362 } catch (IllegalStateException e) { 363 throw e; 364 } 365 } 366 367 if (status != AudioTrack.SUCCESS) { 368 throw new IllegalArgumentException("Fail to set playback rate in audio track"); 369 } 370 371 synchronized(mAudioLock) { 372 mPlaybackRate = rate; 373 } 374 if (mPlaybackRate != 0.0 && mAudioThread != null) { 375 postRenderAudio(0); 376 } 377 native_setPlaybackRate(mPlaybackRate); 378 } 379 380 private native final void native_setPlaybackRate(float rate); 381 382 /* 383 * Test whether a given audio playback mode is supported. 384 * TODO query supported AudioPlaybackMode from audio track. 385 */ 386 private boolean isAudioPlaybackModeSupported(int mode) { 387 return (mode == PLAYBACK_RATE_AUDIO_MODE_RESAMPLE); 388 } 389 390 /** 391 * Get current playback position. 392 * <p> 393 * The MediaTimestamp represents a clock ticking during media playback. It's represented 394 * by an anchor frame ({@link MediaTimestamp#mediaTimeUs} and {@link MediaTimestamp#nanoTime}) 395 * and clock speed ({@link MediaTimestamp#clockRate}). For continous playback with 396 * constant speed, its anchor frame doesn't change that often. Thereafter, it's recommended 397 * to not call this method often. 398 * <p> 399 * To help users to get current playback position, this method always returns the timestamp of 400 * just-rendered frame, i.e., {@link System#nanoTime} and its corresponding media time. They 401 * can be used as current playback position. 402 * 403 * @param timestamp a reference to a non-null MediaTimestamp instance allocated 404 * and owned by caller. 405 * @return true if a timestamp is available, or false if no timestamp is available. 406 * If a timestamp if available, the MediaTimestamp instance is filled in with 407 * playback rate, together with the current media timestamp and the system nanoTime 408 * corresponding to the measured media timestamp. 409 * In the case that no timestamp is available, any supplied instance is left unaltered. 410 */ 411 public boolean getTimestamp(@NonNull MediaTimestamp timestamp) 412 { 413 if (timestamp == null) { 414 throw new IllegalArgumentException(); 415 } 416 return native_getTimestamp(timestamp); 417 } 418 419 private native final boolean native_getTimestamp(MediaTimestamp timestamp); 420 421 /** 422 * Queues the audio data asynchronously for playback (AudioTrack must be in streaming mode). 423 * @param audioData the buffer that holds the data to play. This buffer will be returned 424 * to the client via registered callback. 425 * @param bufferIndex the buffer index used to identify audioData. It will be returned to 426 * the client along with audioData. This helps applications to keep track of audioData. 427 * @param sizeInBytes number of bytes to queue. 428 * @param presentationTimeUs the presentation timestamp in microseconds for the first frame 429 * in the buffer. 430 * @throws IllegalStateException if audio track is not configured or internal configureation 431 * has not been done correctly. 432 */ 433 public void queueAudio( 434 ByteBuffer audioData, int bufferIndex, int sizeInBytes, long presentationTimeUs) { 435 if (mAudioTrack == null || mAudioThread == null) { 436 throw new IllegalStateException( 437 "AudioTrack is NOT configured or audio thread is not created"); 438 } 439 440 synchronized(mAudioLock) { 441 mAudioBuffers.add(new AudioBuffer( 442 audioData, bufferIndex, sizeInBytes, presentationTimeUs)); 443 } 444 445 if (mPlaybackRate != 0.0) { 446 postRenderAudio(0); 447 } 448 } 449 450 // When called on user thread, make sure to check mAudioThread != null. 451 private void postRenderAudio(long delayMillis) { 452 mAudioHandler.postDelayed(new Runnable() { 453 public void run() { 454 synchronized(mAudioLock) { 455 if (mPlaybackRate == 0.0) { 456 return; 457 } 458 459 if (mAudioBuffers.isEmpty()) { 460 return; 461 } 462 463 AudioBuffer audioBuffer = mAudioBuffers.get(0); 464 int sizeWritten = mAudioTrack.write( 465 audioBuffer.mByteBuffer, 466 audioBuffer.mSizeInBytes, 467 AudioTrack.WRITE_NON_BLOCKING); 468 if (sizeWritten > 0) { 469 if (audioBuffer.mPresentationTimeUs != -1) { 470 native_updateQueuedAudioData( 471 audioBuffer.mSizeInBytes, audioBuffer.mPresentationTimeUs); 472 audioBuffer.mPresentationTimeUs = -1; 473 } 474 475 if (sizeWritten == audioBuffer.mSizeInBytes) { 476 postReturnByteBuffer(audioBuffer); 477 mAudioBuffers.remove(0); 478 if (!mAudioBuffers.isEmpty()) { 479 postRenderAudio(0); 480 } 481 return; 482 } 483 484 audioBuffer.mSizeInBytes -= sizeWritten; 485 } 486 // TODO: wait time depends on fullness of audio track. 487 postRenderAudio(10); 488 } 489 } 490 }, delayMillis); 491 } 492 493 private native final void native_updateQueuedAudioData( 494 int sizeInBytes, long presentationTimeUs); 495 496 private final void postReturnByteBuffer(final AudioBuffer audioBuffer) { 497 synchronized(mCallbackLock) { 498 if (mCallbackHandler != null) { 499 final MediaSync sync = this; 500 mCallbackHandler.post(new Runnable() { 501 public void run() { 502 synchronized(mCallbackLock) { 503 if (mCallbackHandler == null 504 || mCallbackHandler.getLooper().getThread() 505 != Thread.currentThread()) { 506 // callback handler has been changed. 507 return; 508 } 509 if (mCallback != null) { 510 mCallback.onReturnAudioBuffer(sync, audioBuffer.mByteBuffer, 511 audioBuffer.mBufferIndex); 512 } 513 } 514 } 515 }); 516 } 517 } 518 } 519 520 private final void returnAudioBuffers() { 521 synchronized(mAudioLock) { 522 for (AudioBuffer audioBuffer: mAudioBuffers) { 523 postReturnByteBuffer(audioBuffer); 524 } 525 mAudioBuffers.clear(); 526 } 527 } 528 529 private void createAudioThread() { 530 mAudioThread = new Thread() { 531 @Override 532 public void run() { 533 Looper.prepare(); 534 synchronized(mAudioLock) { 535 mAudioLooper = Looper.myLooper(); 536 mAudioHandler = new Handler(); 537 mAudioLock.notify(); 538 } 539 Looper.loop(); 540 } 541 }; 542 mAudioThread.start(); 543 544 synchronized(mAudioLock) { 545 try { 546 mAudioLock.wait(); 547 } catch(InterruptedException e) { 548 } 549 } 550 } 551 552 static { 553 System.loadLibrary("media_jni"); 554 native_init(); 555 } 556 557 private static native final void native_init(); 558} 559