1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "NuPlayerRenderer" 19#include <utils/Log.h> 20 21#include "AWakeLock.h" 22#include "NuPlayerRenderer.h" 23#include <algorithm> 24#include <cutils/properties.h> 25#include <media/stagefright/foundation/ADebug.h> 26#include <media/stagefright/foundation/AMessage.h> 27#include <media/stagefright/foundation/AUtils.h> 28#include <media/stagefright/MediaClock.h> 29#include <media/stagefright/MediaErrors.h> 30#include <media/stagefright/MetaData.h> 31#include <media/stagefright/Utils.h> 32#include <media/stagefright/VideoFrameScheduler.h> 33#include <media/MediaCodecBuffer.h> 34 35#include <inttypes.h> 36 37namespace android { 38 39/* 40 * Example of common configuration settings in shell script form 41 42 #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager 43 adb shell setprop audio.offload.disable 1 44 45 #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager 46 adb shell setprop audio.offload.video 1 47 48 #Use audio callbacks for PCM data 49 adb shell setprop media.stagefright.audio.cbk 1 50 51 #Use deep buffer for PCM data with video (it is generally enabled for audio-only) 52 adb shell setprop media.stagefright.audio.deep 1 53 54 #Set size of buffers for pcm audio sink in msec (example: 1000 msec) 55 adb shell setprop media.stagefright.audio.sink 1000 56 57 * These configurations take effect for the next track played (not the current track). 58 */ 59 60static inline bool getUseAudioCallbackSetting() { 61 return property_get_bool("media.stagefright.audio.cbk", false /* default_value */); 62} 63 64static inline int32_t getAudioSinkPcmMsSetting() { 65 return property_get_int32( 66 "media.stagefright.audio.sink", 500 /* default_value */); 67} 68 69// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink 70// is closed to allow the audio DSP to power down. 71static const int64_t kOffloadPauseMaxUs = 10000000ll; 72 73// Maximum allowed delay from AudioSink, 1.5 seconds. 74static const int64_t kMaxAllowedAudioSinkDelayUs = 1500000ll; 75 76static const int64_t kMinimumAudioClockUpdatePeriodUs = 20 /* msec */ * 1000; 77 78// static 79const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = { 80 AUDIO_CHANNEL_NONE, 81 AUDIO_OUTPUT_FLAG_NONE, 82 AUDIO_FORMAT_INVALID, 83 0, // mNumChannels 84 0 // mSampleRate 85}; 86 87// static 88const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; 89 90NuPlayer::Renderer::Renderer( 91 const sp<MediaPlayerBase::AudioSink> &sink, 92 const sp<MediaClock> &mediaClock, 93 const sp<AMessage> ¬ify, 94 uint32_t flags) 95 : mAudioSink(sink), 96 mUseVirtualAudioSink(false), 97 mNotify(notify), 98 mFlags(flags), 99 mNumFramesWritten(0), 100 mDrainAudioQueuePending(false), 101 mDrainVideoQueuePending(false), 102 mAudioQueueGeneration(0), 103 mVideoQueueGeneration(0), 104 mAudioDrainGeneration(0), 105 mVideoDrainGeneration(0), 106 mAudioEOSGeneration(0), 107 mMediaClock(mediaClock), 108 mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT), 109 mAudioFirstAnchorTimeMediaUs(-1), 110 mAnchorTimeMediaUs(-1), 111 mAnchorNumFramesWritten(-1), 112 mVideoLateByUs(0ll), 113 mNextVideoTimeMediaUs(-1), 114 mHasAudio(false), 115 mHasVideo(false), 116 mNotifyCompleteAudio(false), 117 mNotifyCompleteVideo(false), 118 mSyncQueues(false), 119 mPaused(false), 120 mPauseDrainAudioAllowedUs(0), 121 mVideoSampleReceived(false), 122 mVideoRenderingStarted(false), 123 mVideoRenderingStartGeneration(0), 124 mAudioRenderingStartGeneration(0), 125 mRenderingDataDelivered(false), 126 mNextAudioClockUpdateTimeUs(-1), 127 mLastAudioMediaTimeUs(-1), 128 mAudioOffloadPauseTimeoutGeneration(0), 129 mAudioTornDown(false), 130 mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER), 131 mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER), 132 mTotalBuffersQueued(0), 133 mLastAudioBufferDrained(0), 134 mUseAudioCallback(false), 135 mWakeLock(new AWakeLock()) { 136 CHECK(mediaClock != NULL); 137 mPlaybackRate = mPlaybackSettings.mSpeed; 138 mMediaClock->setPlaybackRate(mPlaybackRate); 139} 140 141NuPlayer::Renderer::~Renderer() { 142 if (offloadingAudio()) { 143 mAudioSink->stop(); 144 mAudioSink->flush(); 145 mAudioSink->close(); 146 } 147 148 // Try to avoid racing condition in case callback is still on. 149 Mutex::Autolock autoLock(mLock); 150 if (mUseAudioCallback) { 151 flushQueue(&mAudioQueue); 152 flushQueue(&mVideoQueue); 153 } 154 mWakeLock.clear(); 155 mVideoScheduler.clear(); 156 mNotify.clear(); 157 mAudioSink.clear(); 158} 159 160void NuPlayer::Renderer::queueBuffer( 161 bool audio, 162 const sp<MediaCodecBuffer> &buffer, 163 const sp<AMessage> ¬ifyConsumed) { 164 sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this); 165 msg->setInt32("queueGeneration", getQueueGeneration(audio)); 166 msg->setInt32("audio", static_cast<int32_t>(audio)); 167 msg->setObject("buffer", buffer); 168 msg->setMessage("notifyConsumed", notifyConsumed); 169 msg->post(); 170} 171 172void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) { 173 CHECK_NE(finalResult, (status_t)OK); 174 175 sp<AMessage> msg = new AMessage(kWhatQueueEOS, this); 176 msg->setInt32("queueGeneration", getQueueGeneration(audio)); 177 msg->setInt32("audio", static_cast<int32_t>(audio)); 178 msg->setInt32("finalResult", finalResult); 179 msg->post(); 180} 181 182status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) { 183 sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this); 184 writeToAMessage(msg, rate); 185 sp<AMessage> response; 186 status_t err = msg->postAndAwaitResponse(&response); 187 if (err == OK && response != NULL) { 188 CHECK(response->findInt32("err", &err)); 189 } 190 return err; 191} 192 193status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) { 194 if (rate.mSpeed == 0.f) { 195 onPause(); 196 // don't call audiosink's setPlaybackRate if pausing, as pitch does not 197 // have to correspond to the any non-0 speed (e.g old speed). Keep 198 // settings nonetheless, using the old speed, in case audiosink changes. 199 AudioPlaybackRate newRate = rate; 200 newRate.mSpeed = mPlaybackSettings.mSpeed; 201 mPlaybackSettings = newRate; 202 return OK; 203 } 204 205 if (mAudioSink != NULL && mAudioSink->ready()) { 206 status_t err = mAudioSink->setPlaybackRate(rate); 207 if (err != OK) { 208 return err; 209 } 210 } 211 mPlaybackSettings = rate; 212 mPlaybackRate = rate.mSpeed; 213 mMediaClock->setPlaybackRate(mPlaybackRate); 214 return OK; 215} 216 217status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) { 218 sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this); 219 sp<AMessage> response; 220 status_t err = msg->postAndAwaitResponse(&response); 221 if (err == OK && response != NULL) { 222 CHECK(response->findInt32("err", &err)); 223 if (err == OK) { 224 readFromAMessage(response, rate); 225 } 226 } 227 return err; 228} 229 230status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) { 231 if (mAudioSink != NULL && mAudioSink->ready()) { 232 status_t err = mAudioSink->getPlaybackRate(rate); 233 if (err == OK) { 234 if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) { 235 ALOGW("correcting mismatch in internal/external playback rate"); 236 } 237 // get playback settings used by audiosink, as it may be 238 // slightly off due to audiosink not taking small changes. 239 mPlaybackSettings = *rate; 240 if (mPaused) { 241 rate->mSpeed = 0.f; 242 } 243 } 244 return err; 245 } 246 *rate = mPlaybackSettings; 247 return OK; 248} 249 250status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) { 251 sp<AMessage> msg = new AMessage(kWhatConfigSync, this); 252 writeToAMessage(msg, sync, videoFpsHint); 253 sp<AMessage> response; 254 status_t err = msg->postAndAwaitResponse(&response); 255 if (err == OK && response != NULL) { 256 CHECK(response->findInt32("err", &err)); 257 } 258 return err; 259} 260 261status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) { 262 if (sync.mSource != AVSYNC_SOURCE_DEFAULT) { 263 return BAD_VALUE; 264 } 265 // TODO: support sync sources 266 return INVALID_OPERATION; 267} 268 269status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) { 270 sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this); 271 sp<AMessage> response; 272 status_t err = msg->postAndAwaitResponse(&response); 273 if (err == OK && response != NULL) { 274 CHECK(response->findInt32("err", &err)); 275 if (err == OK) { 276 readFromAMessage(response, sync, videoFps); 277 } 278 } 279 return err; 280} 281 282status_t NuPlayer::Renderer::onGetSyncSettings( 283 AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) { 284 *sync = mSyncSettings; 285 *videoFps = -1.f; 286 return OK; 287} 288 289void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) { 290 { 291 Mutex::Autolock autoLock(mLock); 292 if (audio) { 293 mNotifyCompleteAudio |= notifyComplete; 294 clearAudioFirstAnchorTime_l(); 295 ++mAudioQueueGeneration; 296 ++mAudioDrainGeneration; 297 } else { 298 mNotifyCompleteVideo |= notifyComplete; 299 ++mVideoQueueGeneration; 300 ++mVideoDrainGeneration; 301 } 302 303 mMediaClock->clearAnchor(); 304 mVideoLateByUs = 0; 305 mNextVideoTimeMediaUs = -1; 306 mSyncQueues = false; 307 } 308 309 sp<AMessage> msg = new AMessage(kWhatFlush, this); 310 msg->setInt32("audio", static_cast<int32_t>(audio)); 311 msg->post(); 312} 313 314void NuPlayer::Renderer::signalTimeDiscontinuity() { 315} 316 317void NuPlayer::Renderer::signalDisableOffloadAudio() { 318 (new AMessage(kWhatDisableOffloadAudio, this))->post(); 319} 320 321void NuPlayer::Renderer::signalEnableOffloadAudio() { 322 (new AMessage(kWhatEnableOffloadAudio, this))->post(); 323} 324 325void NuPlayer::Renderer::pause() { 326 (new AMessage(kWhatPause, this))->post(); 327} 328 329void NuPlayer::Renderer::resume() { 330 (new AMessage(kWhatResume, this))->post(); 331} 332 333void NuPlayer::Renderer::setVideoFrameRate(float fps) { 334 sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this); 335 msg->setFloat("frame-rate", fps); 336 msg->post(); 337} 338 339// Called on any threads without mLock acquired. 340status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) { 341 status_t result = mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs); 342 if (result == OK) { 343 return result; 344 } 345 346 // MediaClock has not started yet. Try to start it if possible. 347 { 348 Mutex::Autolock autoLock(mLock); 349 if (mAudioFirstAnchorTimeMediaUs == -1) { 350 return result; 351 } 352 353 AudioTimestamp ts; 354 status_t res = mAudioSink->getTimestamp(ts); 355 if (res != OK) { 356 return result; 357 } 358 359 // AudioSink has rendered some frames. 360 int64_t nowUs = ALooper::GetNowUs(); 361 int64_t nowMediaUs = mAudioSink->getPlayedOutDurationUs(nowUs) 362 + mAudioFirstAnchorTimeMediaUs; 363 mMediaClock->updateAnchor(nowMediaUs, nowUs, -1); 364 } 365 366 return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs); 367} 368 369void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() { 370 mAudioFirstAnchorTimeMediaUs = -1; 371 mMediaClock->setStartingTimeMedia(-1); 372} 373 374void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) { 375 if (mAudioFirstAnchorTimeMediaUs == -1) { 376 mAudioFirstAnchorTimeMediaUs = mediaUs; 377 mMediaClock->setStartingTimeMedia(mediaUs); 378 } 379} 380 381// Called on renderer looper. 382void NuPlayer::Renderer::clearAnchorTime() { 383 mMediaClock->clearAnchor(); 384 mAnchorTimeMediaUs = -1; 385 mAnchorNumFramesWritten = -1; 386} 387 388void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) { 389 Mutex::Autolock autoLock(mLock); 390 mVideoLateByUs = lateUs; 391} 392 393int64_t NuPlayer::Renderer::getVideoLateByUs() { 394 Mutex::Autolock autoLock(mLock); 395 return mVideoLateByUs; 396} 397 398status_t NuPlayer::Renderer::openAudioSink( 399 const sp<AMessage> &format, 400 bool offloadOnly, 401 bool hasVideo, 402 uint32_t flags, 403 bool *isOffloaded, 404 bool isStreaming) { 405 sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this); 406 msg->setMessage("format", format); 407 msg->setInt32("offload-only", offloadOnly); 408 msg->setInt32("has-video", hasVideo); 409 msg->setInt32("flags", flags); 410 msg->setInt32("isStreaming", isStreaming); 411 412 sp<AMessage> response; 413 status_t postStatus = msg->postAndAwaitResponse(&response); 414 415 int32_t err; 416 if (postStatus != OK || response.get() == nullptr || !response->findInt32("err", &err)) { 417 err = INVALID_OPERATION; 418 } else if (err == OK && isOffloaded != NULL) { 419 int32_t offload; 420 CHECK(response->findInt32("offload", &offload)); 421 *isOffloaded = (offload != 0); 422 } 423 return err; 424} 425 426void NuPlayer::Renderer::closeAudioSink() { 427 sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this); 428 429 sp<AMessage> response; 430 msg->postAndAwaitResponse(&response); 431} 432 433void NuPlayer::Renderer::changeAudioFormat( 434 const sp<AMessage> &format, 435 bool offloadOnly, 436 bool hasVideo, 437 uint32_t flags, 438 bool isStreaming, 439 const sp<AMessage> ¬ify) { 440 sp<AMessage> meta = new AMessage; 441 meta->setMessage("format", format); 442 meta->setInt32("offload-only", offloadOnly); 443 meta->setInt32("has-video", hasVideo); 444 meta->setInt32("flags", flags); 445 meta->setInt32("isStreaming", isStreaming); 446 447 sp<AMessage> msg = new AMessage(kWhatChangeAudioFormat, this); 448 msg->setInt32("queueGeneration", getQueueGeneration(true /* audio */)); 449 msg->setMessage("notify", notify); 450 msg->setMessage("meta", meta); 451 msg->post(); 452} 453 454void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { 455 switch (msg->what()) { 456 case kWhatOpenAudioSink: 457 { 458 sp<AMessage> format; 459 CHECK(msg->findMessage("format", &format)); 460 461 int32_t offloadOnly; 462 CHECK(msg->findInt32("offload-only", &offloadOnly)); 463 464 int32_t hasVideo; 465 CHECK(msg->findInt32("has-video", &hasVideo)); 466 467 uint32_t flags; 468 CHECK(msg->findInt32("flags", (int32_t *)&flags)); 469 470 uint32_t isStreaming; 471 CHECK(msg->findInt32("isStreaming", (int32_t *)&isStreaming)); 472 473 status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags, isStreaming); 474 475 sp<AMessage> response = new AMessage; 476 response->setInt32("err", err); 477 response->setInt32("offload", offloadingAudio()); 478 479 sp<AReplyToken> replyID; 480 CHECK(msg->senderAwaitsResponse(&replyID)); 481 response->postReply(replyID); 482 483 break; 484 } 485 486 case kWhatCloseAudioSink: 487 { 488 sp<AReplyToken> replyID; 489 CHECK(msg->senderAwaitsResponse(&replyID)); 490 491 onCloseAudioSink(); 492 493 sp<AMessage> response = new AMessage; 494 response->postReply(replyID); 495 break; 496 } 497 498 case kWhatStopAudioSink: 499 { 500 mAudioSink->stop(); 501 break; 502 } 503 504 case kWhatChangeAudioFormat: 505 { 506 int32_t queueGeneration; 507 CHECK(msg->findInt32("queueGeneration", &queueGeneration)); 508 509 sp<AMessage> notify; 510 CHECK(msg->findMessage("notify", ¬ify)); 511 512 if (offloadingAudio()) { 513 ALOGW("changeAudioFormat should NOT be called in offload mode"); 514 notify->setInt32("err", INVALID_OPERATION); 515 notify->post(); 516 break; 517 } 518 519 sp<AMessage> meta; 520 CHECK(msg->findMessage("meta", &meta)); 521 522 if (queueGeneration != getQueueGeneration(true /* audio */) 523 || mAudioQueue.empty()) { 524 onChangeAudioFormat(meta, notify); 525 break; 526 } 527 528 QueueEntry entry; 529 entry.mNotifyConsumed = notify; 530 entry.mMeta = meta; 531 532 Mutex::Autolock autoLock(mLock); 533 mAudioQueue.push_back(entry); 534 postDrainAudioQueue_l(); 535 536 break; 537 } 538 539 case kWhatDrainAudioQueue: 540 { 541 mDrainAudioQueuePending = false; 542 543 int32_t generation; 544 CHECK(msg->findInt32("drainGeneration", &generation)); 545 if (generation != getDrainGeneration(true /* audio */)) { 546 break; 547 } 548 549 if (onDrainAudioQueue()) { 550 uint32_t numFramesPlayed; 551 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), 552 (status_t)OK); 553 554 // Handle AudioTrack race when start is immediately called after flush. 555 uint32_t numFramesPendingPlayout = 556 (mNumFramesWritten > numFramesPlayed ? 557 mNumFramesWritten - numFramesPlayed : 0); 558 559 // This is how long the audio sink will have data to 560 // play back. 561 int64_t delayUs = 562 mAudioSink->msecsPerFrame() 563 * numFramesPendingPlayout * 1000ll; 564 if (mPlaybackRate > 1.0f) { 565 delayUs /= mPlaybackRate; 566 } 567 568 // Let's give it more data after about half that time 569 // has elapsed. 570 delayUs /= 2; 571 // check the buffer size to estimate maximum delay permitted. 572 const int64_t maxDrainDelayUs = std::max( 573 mAudioSink->getBufferDurationInUs(), (int64_t)500000 /* half second */); 574 ALOGD_IF(delayUs > maxDrainDelayUs, "postDrainAudioQueue long delay: %lld > %lld", 575 (long long)delayUs, (long long)maxDrainDelayUs); 576 Mutex::Autolock autoLock(mLock); 577 postDrainAudioQueue_l(delayUs); 578 } 579 break; 580 } 581 582 case kWhatDrainVideoQueue: 583 { 584 int32_t generation; 585 CHECK(msg->findInt32("drainGeneration", &generation)); 586 if (generation != getDrainGeneration(false /* audio */)) { 587 break; 588 } 589 590 mDrainVideoQueuePending = false; 591 592 onDrainVideoQueue(); 593 594 postDrainVideoQueue(); 595 break; 596 } 597 598 case kWhatPostDrainVideoQueue: 599 { 600 int32_t generation; 601 CHECK(msg->findInt32("drainGeneration", &generation)); 602 if (generation != getDrainGeneration(false /* audio */)) { 603 break; 604 } 605 606 mDrainVideoQueuePending = false; 607 postDrainVideoQueue(); 608 break; 609 } 610 611 case kWhatQueueBuffer: 612 { 613 onQueueBuffer(msg); 614 break; 615 } 616 617 case kWhatQueueEOS: 618 { 619 onQueueEOS(msg); 620 break; 621 } 622 623 case kWhatEOS: 624 { 625 int32_t generation; 626 CHECK(msg->findInt32("audioEOSGeneration", &generation)); 627 if (generation != mAudioEOSGeneration) { 628 break; 629 } 630 status_t finalResult; 631 CHECK(msg->findInt32("finalResult", &finalResult)); 632 notifyEOS(true /* audio */, finalResult); 633 break; 634 } 635 636 case kWhatConfigPlayback: 637 { 638 sp<AReplyToken> replyID; 639 CHECK(msg->senderAwaitsResponse(&replyID)); 640 AudioPlaybackRate rate; 641 readFromAMessage(msg, &rate); 642 status_t err = onConfigPlayback(rate); 643 sp<AMessage> response = new AMessage; 644 response->setInt32("err", err); 645 response->postReply(replyID); 646 break; 647 } 648 649 case kWhatGetPlaybackSettings: 650 { 651 sp<AReplyToken> replyID; 652 CHECK(msg->senderAwaitsResponse(&replyID)); 653 AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT; 654 status_t err = onGetPlaybackSettings(&rate); 655 sp<AMessage> response = new AMessage; 656 if (err == OK) { 657 writeToAMessage(response, rate); 658 } 659 response->setInt32("err", err); 660 response->postReply(replyID); 661 break; 662 } 663 664 case kWhatConfigSync: 665 { 666 sp<AReplyToken> replyID; 667 CHECK(msg->senderAwaitsResponse(&replyID)); 668 AVSyncSettings sync; 669 float videoFpsHint; 670 readFromAMessage(msg, &sync, &videoFpsHint); 671 status_t err = onConfigSync(sync, videoFpsHint); 672 sp<AMessage> response = new AMessage; 673 response->setInt32("err", err); 674 response->postReply(replyID); 675 break; 676 } 677 678 case kWhatGetSyncSettings: 679 { 680 sp<AReplyToken> replyID; 681 CHECK(msg->senderAwaitsResponse(&replyID)); 682 683 ALOGV("kWhatGetSyncSettings"); 684 AVSyncSettings sync; 685 float videoFps = -1.f; 686 status_t err = onGetSyncSettings(&sync, &videoFps); 687 sp<AMessage> response = new AMessage; 688 if (err == OK) { 689 writeToAMessage(response, sync, videoFps); 690 } 691 response->setInt32("err", err); 692 response->postReply(replyID); 693 break; 694 } 695 696 case kWhatFlush: 697 { 698 onFlush(msg); 699 break; 700 } 701 702 case kWhatDisableOffloadAudio: 703 { 704 onDisableOffloadAudio(); 705 break; 706 } 707 708 case kWhatEnableOffloadAudio: 709 { 710 onEnableOffloadAudio(); 711 break; 712 } 713 714 case kWhatPause: 715 { 716 onPause(); 717 break; 718 } 719 720 case kWhatResume: 721 { 722 onResume(); 723 break; 724 } 725 726 case kWhatSetVideoFrameRate: 727 { 728 float fps; 729 CHECK(msg->findFloat("frame-rate", &fps)); 730 onSetVideoFrameRate(fps); 731 break; 732 } 733 734 case kWhatAudioTearDown: 735 { 736 int32_t reason; 737 CHECK(msg->findInt32("reason", &reason)); 738 739 onAudioTearDown((AudioTearDownReason)reason); 740 break; 741 } 742 743 case kWhatAudioOffloadPauseTimeout: 744 { 745 int32_t generation; 746 CHECK(msg->findInt32("drainGeneration", &generation)); 747 if (generation != mAudioOffloadPauseTimeoutGeneration) { 748 break; 749 } 750 ALOGV("Audio Offload tear down due to pause timeout."); 751 onAudioTearDown(kDueToTimeout); 752 mWakeLock->release(); 753 break; 754 } 755 756 default: 757 TRESPASS(); 758 break; 759 } 760} 761 762void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) { 763 if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) { 764 return; 765 } 766 767 if (mAudioQueue.empty()) { 768 return; 769 } 770 771 // FIXME: if paused, wait until AudioTrack stop() is complete before delivering data. 772 if (mPaused) { 773 const int64_t diffUs = mPauseDrainAudioAllowedUs - ALooper::GetNowUs(); 774 if (diffUs > delayUs) { 775 delayUs = diffUs; 776 } 777 } 778 779 mDrainAudioQueuePending = true; 780 sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this); 781 msg->setInt32("drainGeneration", mAudioDrainGeneration); 782 msg->post(delayUs); 783} 784 785void NuPlayer::Renderer::prepareForMediaRenderingStart_l() { 786 mAudioRenderingStartGeneration = mAudioDrainGeneration; 787 mVideoRenderingStartGeneration = mVideoDrainGeneration; 788 mRenderingDataDelivered = false; 789} 790 791void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() { 792 if (mVideoRenderingStartGeneration == mVideoDrainGeneration && 793 mAudioRenderingStartGeneration == mAudioDrainGeneration) { 794 mRenderingDataDelivered = true; 795 if (mPaused) { 796 return; 797 } 798 mVideoRenderingStartGeneration = -1; 799 mAudioRenderingStartGeneration = -1; 800 801 sp<AMessage> notify = mNotify->dup(); 802 notify->setInt32("what", kWhatMediaRenderingStart); 803 notify->post(); 804 } 805} 806 807// static 808size_t NuPlayer::Renderer::AudioSinkCallback( 809 MediaPlayerBase::AudioSink * /* audioSink */, 810 void *buffer, 811 size_t size, 812 void *cookie, 813 MediaPlayerBase::AudioSink::cb_event_t event) { 814 NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie; 815 816 switch (event) { 817 case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER: 818 { 819 return me->fillAudioBuffer(buffer, size); 820 break; 821 } 822 823 case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END: 824 { 825 ALOGV("AudioSink::CB_EVENT_STREAM_END"); 826 me->notifyEOSCallback(); 827 break; 828 } 829 830 case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN: 831 { 832 ALOGV("AudioSink::CB_EVENT_TEAR_DOWN"); 833 me->notifyAudioTearDown(kDueToError); 834 break; 835 } 836 } 837 838 return 0; 839} 840 841void NuPlayer::Renderer::notifyEOSCallback() { 842 Mutex::Autolock autoLock(mLock); 843 844 if (!mUseAudioCallback) { 845 return; 846 } 847 848 notifyEOS_l(true /* audio */, ERROR_END_OF_STREAM); 849} 850 851size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { 852 Mutex::Autolock autoLock(mLock); 853 854 if (!mUseAudioCallback) { 855 return 0; 856 } 857 858 bool hasEOS = false; 859 860 size_t sizeCopied = 0; 861 bool firstEntry = true; 862 QueueEntry *entry; // will be valid after while loop if hasEOS is set. 863 while (sizeCopied < size && !mAudioQueue.empty()) { 864 entry = &*mAudioQueue.begin(); 865 866 if (entry->mBuffer == NULL) { // EOS 867 hasEOS = true; 868 mAudioQueue.erase(mAudioQueue.begin()); 869 break; 870 } 871 872 if (firstEntry && entry->mOffset == 0) { 873 firstEntry = false; 874 int64_t mediaTimeUs; 875 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 876 ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6); 877 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs); 878 } 879 880 size_t copy = entry->mBuffer->size() - entry->mOffset; 881 size_t sizeRemaining = size - sizeCopied; 882 if (copy > sizeRemaining) { 883 copy = sizeRemaining; 884 } 885 886 memcpy((char *)buffer + sizeCopied, 887 entry->mBuffer->data() + entry->mOffset, 888 copy); 889 890 entry->mOffset += copy; 891 if (entry->mOffset == entry->mBuffer->size()) { 892 entry->mNotifyConsumed->post(); 893 mAudioQueue.erase(mAudioQueue.begin()); 894 entry = NULL; 895 } 896 sizeCopied += copy; 897 898 notifyIfMediaRenderingStarted_l(); 899 } 900 901 if (mAudioFirstAnchorTimeMediaUs >= 0) { 902 int64_t nowUs = ALooper::GetNowUs(); 903 int64_t nowMediaUs = 904 mAudioFirstAnchorTimeMediaUs + mAudioSink->getPlayedOutDurationUs(nowUs); 905 // we don't know how much data we are queueing for offloaded tracks. 906 mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX); 907 } 908 909 // for non-offloaded audio, we need to compute the frames written because 910 // there is no EVENT_STREAM_END notification. The frames written gives 911 // an estimate on the pending played out duration. 912 if (!offloadingAudio()) { 913 mNumFramesWritten += sizeCopied / mAudioSink->frameSize(); 914 } 915 916 if (hasEOS) { 917 (new AMessage(kWhatStopAudioSink, this))->post(); 918 // As there is currently no EVENT_STREAM_END callback notification for 919 // non-offloaded audio tracks, we need to post the EOS ourselves. 920 if (!offloadingAudio()) { 921 int64_t postEOSDelayUs = 0; 922 if (mAudioSink->needsTrailingPadding()) { 923 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs()); 924 } 925 ALOGV("fillAudioBuffer: notifyEOS_l " 926 "mNumFramesWritten:%u finalResult:%d postEOSDelay:%lld", 927 mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs); 928 notifyEOS_l(true /* audio */, entry->mFinalResult, postEOSDelayUs); 929 } 930 } 931 return sizeCopied; 932} 933 934void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() { 935 List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it; 936 bool foundEOS = false; 937 while (it != mAudioQueue.end()) { 938 int32_t eos; 939 QueueEntry *entry = &*it++; 940 if ((entry->mBuffer == nullptr && entry->mNotifyConsumed == nullptr) 941 || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) { 942 itEOS = it; 943 foundEOS = true; 944 } 945 } 946 947 if (foundEOS) { 948 // post all replies before EOS and drop the samples 949 for (it = mAudioQueue.begin(); it != itEOS; it++) { 950 if (it->mBuffer == nullptr) { 951 if (it->mNotifyConsumed == nullptr) { 952 // delay doesn't matter as we don't even have an AudioTrack 953 notifyEOS(true /* audio */, it->mFinalResult); 954 } else { 955 // TAG for re-opening audio sink. 956 onChangeAudioFormat(it->mMeta, it->mNotifyConsumed); 957 } 958 } else { 959 it->mNotifyConsumed->post(); 960 } 961 } 962 mAudioQueue.erase(mAudioQueue.begin(), itEOS); 963 } 964} 965 966bool NuPlayer::Renderer::onDrainAudioQueue() { 967 // do not drain audio during teardown as queued buffers may be invalid. 968 if (mAudioTornDown) { 969 return false; 970 } 971 // TODO: This call to getPosition checks if AudioTrack has been created 972 // in AudioSink before draining audio. If AudioTrack doesn't exist, then 973 // CHECKs on getPosition will fail. 974 // We still need to figure out why AudioTrack is not created when 975 // this function is called. One possible reason could be leftover 976 // audio. Another possible place is to check whether decoder 977 // has received INFO_FORMAT_CHANGED as the first buffer since 978 // AudioSink is opened there, and possible interactions with flush 979 // immediately after start. Investigate error message 980 // "vorbis_dsp_synthesis returned -135", along with RTSP. 981 uint32_t numFramesPlayed; 982 if (mAudioSink->getPosition(&numFramesPlayed) != OK) { 983 // When getPosition fails, renderer will not reschedule the draining 984 // unless new samples are queued. 985 // If we have pending EOS (or "eos" marker for discontinuities), we need 986 // to post these now as NuPlayerDecoder might be waiting for it. 987 drainAudioQueueUntilLastEOS(); 988 989 ALOGW("onDrainAudioQueue(): audio sink is not ready"); 990 return false; 991 } 992 993#if 0 994 ssize_t numFramesAvailableToWrite = 995 mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed); 996 997 if (numFramesAvailableToWrite == mAudioSink->frameCount()) { 998 ALOGI("audio sink underrun"); 999 } else { 1000 ALOGV("audio queue has %d frames left to play", 1001 mAudioSink->frameCount() - numFramesAvailableToWrite); 1002 } 1003#endif 1004 1005 uint32_t prevFramesWritten = mNumFramesWritten; 1006 while (!mAudioQueue.empty()) { 1007 QueueEntry *entry = &*mAudioQueue.begin(); 1008 1009 if (entry->mBuffer == NULL) { 1010 if (entry->mNotifyConsumed != nullptr) { 1011 // TAG for re-open audio sink. 1012 onChangeAudioFormat(entry->mMeta, entry->mNotifyConsumed); 1013 mAudioQueue.erase(mAudioQueue.begin()); 1014 continue; 1015 } 1016 1017 // EOS 1018 if (mPaused) { 1019 // Do not notify EOS when paused. 1020 // This is needed to avoid switch to next clip while in pause. 1021 ALOGV("onDrainAudioQueue(): Do not notify EOS when paused"); 1022 return false; 1023 } 1024 1025 int64_t postEOSDelayUs = 0; 1026 if (mAudioSink->needsTrailingPadding()) { 1027 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs()); 1028 } 1029 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs); 1030 mLastAudioMediaTimeUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten); 1031 1032 mAudioQueue.erase(mAudioQueue.begin()); 1033 entry = NULL; 1034 if (mAudioSink->needsTrailingPadding()) { 1035 // If we're not in gapless playback (i.e. through setNextPlayer), we 1036 // need to stop the track here, because that will play out the last 1037 // little bit at the end of the file. Otherwise short files won't play. 1038 mAudioSink->stop(); 1039 mNumFramesWritten = 0; 1040 } 1041 return false; 1042 } 1043 1044 mLastAudioBufferDrained = entry->mBufferOrdinal; 1045 1046 // ignore 0-sized buffer which could be EOS marker with no data 1047 if (entry->mOffset == 0 && entry->mBuffer->size() > 0) { 1048 int64_t mediaTimeUs; 1049 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 1050 ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs", 1051 mediaTimeUs / 1E6); 1052 onNewAudioMediaTime(mediaTimeUs); 1053 } 1054 1055 size_t copy = entry->mBuffer->size() - entry->mOffset; 1056 1057 ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, 1058 copy, false /* blocking */); 1059 if (written < 0) { 1060 // An error in AudioSink write. Perhaps the AudioSink was not properly opened. 1061 if (written == WOULD_BLOCK) { 1062 ALOGV("AudioSink write would block when writing %zu bytes", copy); 1063 } else { 1064 ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy); 1065 // This can only happen when AudioSink was opened with doNotReconnect flag set to 1066 // true, in which case the NuPlayer will handle the reconnect. 1067 notifyAudioTearDown(kDueToError); 1068 } 1069 break; 1070 } 1071 1072 entry->mOffset += written; 1073 size_t remainder = entry->mBuffer->size() - entry->mOffset; 1074 if ((ssize_t)remainder < mAudioSink->frameSize()) { 1075 if (remainder > 0) { 1076 ALOGW("Corrupted audio buffer has fractional frames, discarding %zu bytes.", 1077 remainder); 1078 entry->mOffset += remainder; 1079 copy -= remainder; 1080 } 1081 1082 entry->mNotifyConsumed->post(); 1083 mAudioQueue.erase(mAudioQueue.begin()); 1084 1085 entry = NULL; 1086 } 1087 1088 size_t copiedFrames = written / mAudioSink->frameSize(); 1089 mNumFramesWritten += copiedFrames; 1090 1091 { 1092 Mutex::Autolock autoLock(mLock); 1093 int64_t maxTimeMedia; 1094 maxTimeMedia = 1095 mAnchorTimeMediaUs + 1096 (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL) 1097 * 1000LL * mAudioSink->msecsPerFrame()); 1098 mMediaClock->updateMaxTimeMedia(maxTimeMedia); 1099 1100 notifyIfMediaRenderingStarted_l(); 1101 } 1102 1103 if (written != (ssize_t)copy) { 1104 // A short count was received from AudioSink::write() 1105 // 1106 // AudioSink write is called in non-blocking mode. 1107 // It may return with a short count when: 1108 // 1109 // 1) Size to be copied is not a multiple of the frame size. Fractional frames are 1110 // discarded. 1111 // 2) The data to be copied exceeds the available buffer in AudioSink. 1112 // 3) An error occurs and data has been partially copied to the buffer in AudioSink. 1113 // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded. 1114 1115 // (Case 1) 1116 // Must be a multiple of the frame size. If it is not a multiple of a frame size, it 1117 // needs to fail, as we should not carry over fractional frames between calls. 1118 CHECK_EQ(copy % mAudioSink->frameSize(), 0u); 1119 1120 // (Case 2, 3, 4) 1121 // Return early to the caller. 1122 // Beware of calling immediately again as this may busy-loop if you are not careful. 1123 ALOGV("AudioSink write short frame count %zd < %zu", written, copy); 1124 break; 1125 } 1126 } 1127 1128 // calculate whether we need to reschedule another write. 1129 bool reschedule = !mAudioQueue.empty() 1130 && (!mPaused 1131 || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers 1132 //ALOGD("reschedule:%d empty:%d mPaused:%d prevFramesWritten:%u mNumFramesWritten:%u", 1133 // reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten); 1134 return reschedule; 1135} 1136 1137int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) { 1138 int32_t sampleRate = offloadingAudio() ? 1139 mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate; 1140 if (sampleRate == 0) { 1141 ALOGE("sampleRate is 0 in %s mode", offloadingAudio() ? "offload" : "non-offload"); 1142 return 0; 1143 } 1144 // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours. 1145 return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate); 1146} 1147 1148// Calculate duration of pending samples if played at normal rate (i.e., 1.0). 1149int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) { 1150 int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten); 1151 if (mUseVirtualAudioSink) { 1152 int64_t nowUs = ALooper::GetNowUs(); 1153 int64_t mediaUs; 1154 if (mMediaClock->getMediaTime(nowUs, &mediaUs) != OK) { 1155 return 0ll; 1156 } else { 1157 return writtenAudioDurationUs - (mediaUs - mAudioFirstAnchorTimeMediaUs); 1158 } 1159 } 1160 1161 const int64_t audioSinkPlayedUs = mAudioSink->getPlayedOutDurationUs(nowUs); 1162 int64_t pendingUs = writtenAudioDurationUs - audioSinkPlayedUs; 1163 if (pendingUs < 0) { 1164 // This shouldn't happen unless the timestamp is stale. 1165 ALOGW("%s: pendingUs %lld < 0, clamping to zero, potential resume after pause " 1166 "writtenAudioDurationUs: %lld, audioSinkPlayedUs: %lld", 1167 __func__, (long long)pendingUs, 1168 (long long)writtenAudioDurationUs, (long long)audioSinkPlayedUs); 1169 pendingUs = 0; 1170 } 1171 return pendingUs; 1172} 1173 1174int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) { 1175 int64_t realUs; 1176 if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) { 1177 // If failed to get current position, e.g. due to audio clock is 1178 // not ready, then just play out video immediately without delay. 1179 return nowUs; 1180 } 1181 return realUs; 1182} 1183 1184void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) { 1185 Mutex::Autolock autoLock(mLock); 1186 // TRICKY: vorbis decoder generates multiple frames with the same 1187 // timestamp, so only update on the first frame with a given timestamp 1188 if (mediaTimeUs == mAnchorTimeMediaUs) { 1189 return; 1190 } 1191 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs); 1192 1193 // mNextAudioClockUpdateTimeUs is -1 if we're waiting for audio sink to start 1194 if (mNextAudioClockUpdateTimeUs == -1) { 1195 AudioTimestamp ts; 1196 if (mAudioSink->getTimestamp(ts) == OK && ts.mPosition > 0) { 1197 mNextAudioClockUpdateTimeUs = 0; // start our clock updates 1198 } 1199 } 1200 int64_t nowUs = ALooper::GetNowUs(); 1201 if (mNextAudioClockUpdateTimeUs >= 0) { 1202 if (nowUs >= mNextAudioClockUpdateTimeUs) { 1203 int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs); 1204 mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs); 1205 mUseVirtualAudioSink = false; 1206 mNextAudioClockUpdateTimeUs = nowUs + kMinimumAudioClockUpdatePeriodUs; 1207 } 1208 } else { 1209 int64_t unused; 1210 if ((mMediaClock->getMediaTime(nowUs, &unused) != OK) 1211 && (getDurationUsIfPlayedAtSampleRate(mNumFramesWritten) 1212 > kMaxAllowedAudioSinkDelayUs)) { 1213 // Enough data has been sent to AudioSink, but AudioSink has not rendered 1214 // any data yet. Something is wrong with AudioSink, e.g., the device is not 1215 // connected to audio out. 1216 // Switch to system clock. This essentially creates a virtual AudioSink with 1217 // initial latenty of getDurationUsIfPlayedAtSampleRate(mNumFramesWritten). 1218 // This virtual AudioSink renders audio data starting from the very first sample 1219 // and it's paced by system clock. 1220 ALOGW("AudioSink stuck. ARE YOU CONNECTED TO AUDIO OUT? Switching to system clock."); 1221 mMediaClock->updateAnchor(mAudioFirstAnchorTimeMediaUs, nowUs, mediaTimeUs); 1222 mUseVirtualAudioSink = true; 1223 } 1224 } 1225 mAnchorNumFramesWritten = mNumFramesWritten; 1226 mAnchorTimeMediaUs = mediaTimeUs; 1227} 1228 1229// Called without mLock acquired. 1230void NuPlayer::Renderer::postDrainVideoQueue() { 1231 if (mDrainVideoQueuePending 1232 || getSyncQueues() 1233 || (mPaused && mVideoSampleReceived)) { 1234 return; 1235 } 1236 1237 if (mVideoQueue.empty()) { 1238 return; 1239 } 1240 1241 QueueEntry &entry = *mVideoQueue.begin(); 1242 1243 sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this); 1244 msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */)); 1245 1246 if (entry.mBuffer == NULL) { 1247 // EOS doesn't carry a timestamp. 1248 msg->post(); 1249 mDrainVideoQueuePending = true; 1250 return; 1251 } 1252 1253 int64_t nowUs = ALooper::GetNowUs(); 1254 if (mFlags & FLAG_REAL_TIME) { 1255 int64_t realTimeUs; 1256 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &realTimeUs)); 1257 1258 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000; 1259 1260 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000); 1261 1262 int64_t delayUs = realTimeUs - nowUs; 1263 1264 ALOGW_IF(delayUs > 500000, "unusually high delayUs: %lld", (long long)delayUs); 1265 // post 2 display refreshes before rendering is due 1266 msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0); 1267 1268 mDrainVideoQueuePending = true; 1269 return; 1270 } 1271 1272 int64_t mediaTimeUs; 1273 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 1274 1275 { 1276 Mutex::Autolock autoLock(mLock); 1277 if (mAnchorTimeMediaUs < 0) { 1278 mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs); 1279 mAnchorTimeMediaUs = mediaTimeUs; 1280 } 1281 } 1282 mNextVideoTimeMediaUs = mediaTimeUs + 100000; 1283 if (!mHasAudio) { 1284 // smooth out videos >= 10fps 1285 mMediaClock->updateMaxTimeMedia(mNextVideoTimeMediaUs); 1286 } 1287 1288 if (!mVideoSampleReceived || mediaTimeUs < mAudioFirstAnchorTimeMediaUs) { 1289 msg->post(); 1290 } else { 1291 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000); 1292 1293 // post 2 display refreshes before rendering is due 1294 mMediaClock->addTimer(msg, mediaTimeUs, -twoVsyncsUs); 1295 } 1296 1297 mDrainVideoQueuePending = true; 1298} 1299 1300void NuPlayer::Renderer::onDrainVideoQueue() { 1301 if (mVideoQueue.empty()) { 1302 return; 1303 } 1304 1305 QueueEntry *entry = &*mVideoQueue.begin(); 1306 1307 if (entry->mBuffer == NULL) { 1308 // EOS 1309 1310 notifyEOS(false /* audio */, entry->mFinalResult); 1311 1312 mVideoQueue.erase(mVideoQueue.begin()); 1313 entry = NULL; 1314 1315 setVideoLateByUs(0); 1316 return; 1317 } 1318 1319 int64_t nowUs = ALooper::GetNowUs(); 1320 int64_t realTimeUs; 1321 int64_t mediaTimeUs = -1; 1322 if (mFlags & FLAG_REAL_TIME) { 1323 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs)); 1324 } else { 1325 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 1326 1327 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); 1328 } 1329 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000; 1330 1331 bool tooLate = false; 1332 1333 if (!mPaused) { 1334 setVideoLateByUs(nowUs - realTimeUs); 1335 tooLate = (mVideoLateByUs > 40000); 1336 1337 if (tooLate) { 1338 ALOGV("video late by %lld us (%.2f secs)", 1339 (long long)mVideoLateByUs, mVideoLateByUs / 1E6); 1340 } else { 1341 int64_t mediaUs = 0; 1342 mMediaClock->getMediaTime(realTimeUs, &mediaUs); 1343 ALOGV("rendering video at media time %.2f secs", 1344 (mFlags & FLAG_REAL_TIME ? realTimeUs : 1345 mediaUs) / 1E6); 1346 1347 if (!(mFlags & FLAG_REAL_TIME) 1348 && mLastAudioMediaTimeUs != -1 1349 && mediaTimeUs > mLastAudioMediaTimeUs) { 1350 // If audio ends before video, video continues to drive media clock. 1351 // Also smooth out videos >= 10fps. 1352 mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000); 1353 } 1354 } 1355 } else { 1356 setVideoLateByUs(0); 1357 if (!mVideoSampleReceived && !mHasAudio) { 1358 // This will ensure that the first frame after a flush won't be used as anchor 1359 // when renderer is in paused state, because resume can happen any time after seek. 1360 clearAnchorTime(); 1361 } 1362 } 1363 1364 // Always render the first video frame while keeping stats on A/V sync. 1365 if (!mVideoSampleReceived) { 1366 realTimeUs = nowUs; 1367 tooLate = false; 1368 } 1369 1370 entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll); 1371 entry->mNotifyConsumed->setInt32("render", !tooLate); 1372 entry->mNotifyConsumed->post(); 1373 mVideoQueue.erase(mVideoQueue.begin()); 1374 entry = NULL; 1375 1376 mVideoSampleReceived = true; 1377 1378 if (!mPaused) { 1379 if (!mVideoRenderingStarted) { 1380 mVideoRenderingStarted = true; 1381 notifyVideoRenderingStart(); 1382 } 1383 Mutex::Autolock autoLock(mLock); 1384 notifyIfMediaRenderingStarted_l(); 1385 } 1386} 1387 1388void NuPlayer::Renderer::notifyVideoRenderingStart() { 1389 sp<AMessage> notify = mNotify->dup(); 1390 notify->setInt32("what", kWhatVideoRenderingStart); 1391 notify->post(); 1392} 1393 1394void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) { 1395 Mutex::Autolock autoLock(mLock); 1396 notifyEOS_l(audio, finalResult, delayUs); 1397} 1398 1399void NuPlayer::Renderer::notifyEOS_l(bool audio, status_t finalResult, int64_t delayUs) { 1400 if (audio && delayUs > 0) { 1401 sp<AMessage> msg = new AMessage(kWhatEOS, this); 1402 msg->setInt32("audioEOSGeneration", mAudioEOSGeneration); 1403 msg->setInt32("finalResult", finalResult); 1404 msg->post(delayUs); 1405 return; 1406 } 1407 sp<AMessage> notify = mNotify->dup(); 1408 notify->setInt32("what", kWhatEOS); 1409 notify->setInt32("audio", static_cast<int32_t>(audio)); 1410 notify->setInt32("finalResult", finalResult); 1411 notify->post(delayUs); 1412 1413 if (audio) { 1414 // Video might outlive audio. Clear anchor to enable video only case. 1415 mAnchorTimeMediaUs = -1; 1416 mHasAudio = false; 1417 if (mNextVideoTimeMediaUs >= 0) { 1418 int64_t mediaUs = 0; 1419 mMediaClock->getMediaTime(ALooper::GetNowUs(), &mediaUs); 1420 if (mNextVideoTimeMediaUs > mediaUs) { 1421 mMediaClock->updateMaxTimeMedia(mNextVideoTimeMediaUs); 1422 } 1423 } 1424 } 1425} 1426 1427void NuPlayer::Renderer::notifyAudioTearDown(AudioTearDownReason reason) { 1428 sp<AMessage> msg = new AMessage(kWhatAudioTearDown, this); 1429 msg->setInt32("reason", reason); 1430 msg->post(); 1431} 1432 1433void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { 1434 int32_t audio; 1435 CHECK(msg->findInt32("audio", &audio)); 1436 1437 if (dropBufferIfStale(audio, msg)) { 1438 return; 1439 } 1440 1441 if (audio) { 1442 mHasAudio = true; 1443 } else { 1444 mHasVideo = true; 1445 } 1446 1447 if (mHasVideo) { 1448 if (mVideoScheduler == NULL) { 1449 mVideoScheduler = new VideoFrameScheduler(); 1450 mVideoScheduler->init(); 1451 } 1452 } 1453 1454 sp<RefBase> obj; 1455 CHECK(msg->findObject("buffer", &obj)); 1456 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get()); 1457 1458 sp<AMessage> notifyConsumed; 1459 CHECK(msg->findMessage("notifyConsumed", ¬ifyConsumed)); 1460 1461 QueueEntry entry; 1462 entry.mBuffer = buffer; 1463 entry.mNotifyConsumed = notifyConsumed; 1464 entry.mOffset = 0; 1465 entry.mFinalResult = OK; 1466 entry.mBufferOrdinal = ++mTotalBuffersQueued; 1467 1468 if (audio) { 1469 Mutex::Autolock autoLock(mLock); 1470 mAudioQueue.push_back(entry); 1471 postDrainAudioQueue_l(); 1472 } else { 1473 mVideoQueue.push_back(entry); 1474 postDrainVideoQueue(); 1475 } 1476 1477 Mutex::Autolock autoLock(mLock); 1478 if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) { 1479 return; 1480 } 1481 1482 sp<MediaCodecBuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer; 1483 sp<MediaCodecBuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer; 1484 1485 if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) { 1486 // EOS signalled on either queue. 1487 syncQueuesDone_l(); 1488 return; 1489 } 1490 1491 int64_t firstAudioTimeUs; 1492 int64_t firstVideoTimeUs; 1493 CHECK(firstAudioBuffer->meta() 1494 ->findInt64("timeUs", &firstAudioTimeUs)); 1495 CHECK(firstVideoBuffer->meta() 1496 ->findInt64("timeUs", &firstVideoTimeUs)); 1497 1498 int64_t diff = firstVideoTimeUs - firstAudioTimeUs; 1499 1500 ALOGV("queueDiff = %.2f secs", diff / 1E6); 1501 1502 if (diff > 100000ll) { 1503 // Audio data starts More than 0.1 secs before video. 1504 // Drop some audio. 1505 1506 (*mAudioQueue.begin()).mNotifyConsumed->post(); 1507 mAudioQueue.erase(mAudioQueue.begin()); 1508 return; 1509 } 1510 1511 syncQueuesDone_l(); 1512} 1513 1514void NuPlayer::Renderer::syncQueuesDone_l() { 1515 if (!mSyncQueues) { 1516 return; 1517 } 1518 1519 mSyncQueues = false; 1520 1521 if (!mAudioQueue.empty()) { 1522 postDrainAudioQueue_l(); 1523 } 1524 1525 if (!mVideoQueue.empty()) { 1526 mLock.unlock(); 1527 postDrainVideoQueue(); 1528 mLock.lock(); 1529 } 1530} 1531 1532void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) { 1533 int32_t audio; 1534 CHECK(msg->findInt32("audio", &audio)); 1535 1536 if (dropBufferIfStale(audio, msg)) { 1537 return; 1538 } 1539 1540 int32_t finalResult; 1541 CHECK(msg->findInt32("finalResult", &finalResult)); 1542 1543 QueueEntry entry; 1544 entry.mOffset = 0; 1545 entry.mFinalResult = finalResult; 1546 1547 if (audio) { 1548 Mutex::Autolock autoLock(mLock); 1549 if (mAudioQueue.empty() && mSyncQueues) { 1550 syncQueuesDone_l(); 1551 } 1552 mAudioQueue.push_back(entry); 1553 postDrainAudioQueue_l(); 1554 } else { 1555 if (mVideoQueue.empty() && getSyncQueues()) { 1556 Mutex::Autolock autoLock(mLock); 1557 syncQueuesDone_l(); 1558 } 1559 mVideoQueue.push_back(entry); 1560 postDrainVideoQueue(); 1561 } 1562} 1563 1564void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { 1565 int32_t audio, notifyComplete; 1566 CHECK(msg->findInt32("audio", &audio)); 1567 1568 { 1569 Mutex::Autolock autoLock(mLock); 1570 if (audio) { 1571 notifyComplete = mNotifyCompleteAudio; 1572 mNotifyCompleteAudio = false; 1573 mLastAudioMediaTimeUs = -1; 1574 } else { 1575 notifyComplete = mNotifyCompleteVideo; 1576 mNotifyCompleteVideo = false; 1577 } 1578 1579 // If we're currently syncing the queues, i.e. dropping audio while 1580 // aligning the first audio/video buffer times and only one of the 1581 // two queues has data, we may starve that queue by not requesting 1582 // more buffers from the decoder. If the other source then encounters 1583 // a discontinuity that leads to flushing, we'll never find the 1584 // corresponding discontinuity on the other queue. 1585 // Therefore we'll stop syncing the queues if at least one of them 1586 // is flushed. 1587 syncQueuesDone_l(); 1588 } 1589 clearAnchorTime(); 1590 1591 ALOGV("flushing %s", audio ? "audio" : "video"); 1592 if (audio) { 1593 { 1594 Mutex::Autolock autoLock(mLock); 1595 flushQueue(&mAudioQueue); 1596 1597 ++mAudioDrainGeneration; 1598 ++mAudioEOSGeneration; 1599 prepareForMediaRenderingStart_l(); 1600 1601 // the frame count will be reset after flush. 1602 clearAudioFirstAnchorTime_l(); 1603 } 1604 1605 mDrainAudioQueuePending = false; 1606 1607 if (offloadingAudio()) { 1608 mAudioSink->pause(); 1609 mAudioSink->flush(); 1610 if (!mPaused) { 1611 mAudioSink->start(); 1612 } 1613 } else { 1614 mAudioSink->pause(); 1615 mAudioSink->flush(); 1616 // Call stop() to signal to the AudioSink to completely fill the 1617 // internal buffer before resuming playback. 1618 // FIXME: this is ignored after flush(). 1619 mAudioSink->stop(); 1620 if (!mPaused) { 1621 mAudioSink->start(); 1622 } 1623 mNumFramesWritten = 0; 1624 } 1625 mNextAudioClockUpdateTimeUs = -1; 1626 } else { 1627 flushQueue(&mVideoQueue); 1628 1629 mDrainVideoQueuePending = false; 1630 1631 if (mVideoScheduler != NULL) { 1632 mVideoScheduler->restart(); 1633 } 1634 1635 Mutex::Autolock autoLock(mLock); 1636 ++mVideoDrainGeneration; 1637 prepareForMediaRenderingStart_l(); 1638 } 1639 1640 mVideoSampleReceived = false; 1641 1642 if (notifyComplete) { 1643 notifyFlushComplete(audio); 1644 } 1645} 1646 1647void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) { 1648 while (!queue->empty()) { 1649 QueueEntry *entry = &*queue->begin(); 1650 1651 if (entry->mBuffer != NULL) { 1652 entry->mNotifyConsumed->post(); 1653 } else if (entry->mNotifyConsumed != nullptr) { 1654 // Is it needed to open audio sink now? 1655 onChangeAudioFormat(entry->mMeta, entry->mNotifyConsumed); 1656 } 1657 1658 queue->erase(queue->begin()); 1659 entry = NULL; 1660 } 1661} 1662 1663void NuPlayer::Renderer::notifyFlushComplete(bool audio) { 1664 sp<AMessage> notify = mNotify->dup(); 1665 notify->setInt32("what", kWhatFlushComplete); 1666 notify->setInt32("audio", static_cast<int32_t>(audio)); 1667 notify->post(); 1668} 1669 1670bool NuPlayer::Renderer::dropBufferIfStale( 1671 bool audio, const sp<AMessage> &msg) { 1672 int32_t queueGeneration; 1673 CHECK(msg->findInt32("queueGeneration", &queueGeneration)); 1674 1675 if (queueGeneration == getQueueGeneration(audio)) { 1676 return false; 1677 } 1678 1679 sp<AMessage> notifyConsumed; 1680 if (msg->findMessage("notifyConsumed", ¬ifyConsumed)) { 1681 notifyConsumed->post(); 1682 } 1683 1684 return true; 1685} 1686 1687void NuPlayer::Renderer::onAudioSinkChanged() { 1688 if (offloadingAudio()) { 1689 return; 1690 } 1691 CHECK(!mDrainAudioQueuePending); 1692 mNumFramesWritten = 0; 1693 mAnchorNumFramesWritten = -1; 1694 uint32_t written; 1695 if (mAudioSink->getFramesWritten(&written) == OK) { 1696 mNumFramesWritten = written; 1697 } 1698} 1699 1700void NuPlayer::Renderer::onDisableOffloadAudio() { 1701 Mutex::Autolock autoLock(mLock); 1702 mFlags &= ~FLAG_OFFLOAD_AUDIO; 1703 ++mAudioDrainGeneration; 1704 if (mAudioRenderingStartGeneration != -1) { 1705 prepareForMediaRenderingStart_l(); 1706 // PauseTimeout is applied to offload mode only. Cancel pending timer. 1707 cancelAudioOffloadPauseTimeout(); 1708 } 1709} 1710 1711void NuPlayer::Renderer::onEnableOffloadAudio() { 1712 Mutex::Autolock autoLock(mLock); 1713 mFlags |= FLAG_OFFLOAD_AUDIO; 1714 ++mAudioDrainGeneration; 1715 if (mAudioRenderingStartGeneration != -1) { 1716 prepareForMediaRenderingStart_l(); 1717 } 1718} 1719 1720void NuPlayer::Renderer::onPause() { 1721 if (mPaused) { 1722 return; 1723 } 1724 1725 { 1726 Mutex::Autolock autoLock(mLock); 1727 // we do not increment audio drain generation so that we fill audio buffer during pause. 1728 ++mVideoDrainGeneration; 1729 prepareForMediaRenderingStart_l(); 1730 mPaused = true; 1731 mMediaClock->setPlaybackRate(0.0); 1732 } 1733 1734 mDrainAudioQueuePending = false; 1735 mDrainVideoQueuePending = false; 1736 1737 // Note: audio data may not have been decoded, and the AudioSink may not be opened. 1738 mAudioSink->pause(); 1739 startAudioOffloadPauseTimeout(); 1740 1741 ALOGV("now paused audio queue has %zu entries, video has %zu entries", 1742 mAudioQueue.size(), mVideoQueue.size()); 1743} 1744 1745void NuPlayer::Renderer::onResume() { 1746 if (!mPaused) { 1747 return; 1748 } 1749 1750 // Note: audio data may not have been decoded, and the AudioSink may not be opened. 1751 cancelAudioOffloadPauseTimeout(); 1752 if (mAudioSink->ready()) { 1753 status_t err = mAudioSink->start(); 1754 if (err != OK) { 1755 ALOGE("cannot start AudioSink err %d", err); 1756 notifyAudioTearDown(kDueToError); 1757 } 1758 } 1759 1760 { 1761 Mutex::Autolock autoLock(mLock); 1762 mPaused = false; 1763 // rendering started message may have been delayed if we were paused. 1764 if (mRenderingDataDelivered) { 1765 notifyIfMediaRenderingStarted_l(); 1766 } 1767 // configure audiosink as we did not do it when pausing 1768 if (mAudioSink != NULL && mAudioSink->ready()) { 1769 mAudioSink->setPlaybackRate(mPlaybackSettings); 1770 } 1771 1772 mMediaClock->setPlaybackRate(mPlaybackRate); 1773 1774 if (!mAudioQueue.empty()) { 1775 postDrainAudioQueue_l(); 1776 } 1777 } 1778 1779 if (!mVideoQueue.empty()) { 1780 postDrainVideoQueue(); 1781 } 1782} 1783 1784void NuPlayer::Renderer::onSetVideoFrameRate(float fps) { 1785 if (mVideoScheduler == NULL) { 1786 mVideoScheduler = new VideoFrameScheduler(); 1787 } 1788 mVideoScheduler->init(fps); 1789} 1790 1791int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) { 1792 Mutex::Autolock autoLock(mLock); 1793 return (audio ? mAudioQueueGeneration : mVideoQueueGeneration); 1794} 1795 1796int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) { 1797 Mutex::Autolock autoLock(mLock); 1798 return (audio ? mAudioDrainGeneration : mVideoDrainGeneration); 1799} 1800 1801bool NuPlayer::Renderer::getSyncQueues() { 1802 Mutex::Autolock autoLock(mLock); 1803 return mSyncQueues; 1804} 1805 1806void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) { 1807 if (mAudioTornDown) { 1808 return; 1809 } 1810 1811 // TimeoutWhenPaused is only for offload mode. 1812 if (reason == kDueToTimeout && !offloadingAudio()) { 1813 return; 1814 } 1815 1816 mAudioTornDown = true; 1817 1818 int64_t currentPositionUs; 1819 sp<AMessage> notify = mNotify->dup(); 1820 if (getCurrentPosition(¤tPositionUs) == OK) { 1821 notify->setInt64("positionUs", currentPositionUs); 1822 } 1823 1824 mAudioSink->stop(); 1825 mAudioSink->flush(); 1826 1827 notify->setInt32("what", kWhatAudioTearDown); 1828 notify->setInt32("reason", reason); 1829 notify->post(); 1830} 1831 1832void NuPlayer::Renderer::startAudioOffloadPauseTimeout() { 1833 if (offloadingAudio()) { 1834 mWakeLock->acquire(); 1835 sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this); 1836 msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration); 1837 msg->post(kOffloadPauseMaxUs); 1838 } 1839} 1840 1841void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() { 1842 // We may have called startAudioOffloadPauseTimeout() without 1843 // the AudioSink open and with offloadingAudio enabled. 1844 // 1845 // When we cancel, it may be that offloadingAudio is subsequently disabled, so regardless 1846 // we always release the wakelock and increment the pause timeout generation. 1847 // 1848 // Note: The acquired wakelock prevents the device from suspending 1849 // immediately after offload pause (in case a resume happens shortly thereafter). 1850 mWakeLock->release(true); 1851 ++mAudioOffloadPauseTimeoutGeneration; 1852} 1853 1854status_t NuPlayer::Renderer::onOpenAudioSink( 1855 const sp<AMessage> &format, 1856 bool offloadOnly, 1857 bool hasVideo, 1858 uint32_t flags, 1859 bool isStreaming) { 1860 ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)", 1861 offloadOnly, offloadingAudio()); 1862 bool audioSinkChanged = false; 1863 1864 int32_t numChannels; 1865 CHECK(format->findInt32("channel-count", &numChannels)); 1866 1867 int32_t channelMask; 1868 if (!format->findInt32("channel-mask", &channelMask)) { 1869 // signal to the AudioSink to derive the mask from count. 1870 channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER; 1871 } 1872 1873 int32_t sampleRate; 1874 CHECK(format->findInt32("sample-rate", &sampleRate)); 1875 1876 if (offloadingAudio()) { 1877 audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT; 1878 AString mime; 1879 CHECK(format->findString("mime", &mime)); 1880 status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str()); 1881 1882 if (err != OK) { 1883 ALOGE("Couldn't map mime \"%s\" to a valid " 1884 "audio_format", mime.c_str()); 1885 onDisableOffloadAudio(); 1886 } else { 1887 ALOGV("Mime \"%s\" mapped to audio_format 0x%x", 1888 mime.c_str(), audioFormat); 1889 1890 int avgBitRate = -1; 1891 format->findInt32("bitrate", &avgBitRate); 1892 1893 int32_t aacProfile = -1; 1894 if (audioFormat == AUDIO_FORMAT_AAC 1895 && format->findInt32("aac-profile", &aacProfile)) { 1896 // Redefine AAC format as per aac profile 1897 mapAACProfileToAudioFormat( 1898 audioFormat, 1899 aacProfile); 1900 } 1901 1902 audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER; 1903 offloadInfo.duration_us = -1; 1904 format->findInt64( 1905 "durationUs", &offloadInfo.duration_us); 1906 offloadInfo.sample_rate = sampleRate; 1907 offloadInfo.channel_mask = channelMask; 1908 offloadInfo.format = audioFormat; 1909 offloadInfo.stream_type = AUDIO_STREAM_MUSIC; 1910 offloadInfo.bit_rate = avgBitRate; 1911 offloadInfo.has_video = hasVideo; 1912 offloadInfo.is_streaming = isStreaming; 1913 1914 if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) { 1915 ALOGV("openAudioSink: no change in offload mode"); 1916 // no change from previous configuration, everything ok. 1917 return OK; 1918 } 1919 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER; 1920 1921 ALOGV("openAudioSink: try to open AudioSink in offload mode"); 1922 uint32_t offloadFlags = flags; 1923 offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; 1924 offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER; 1925 audioSinkChanged = true; 1926 mAudioSink->close(); 1927 1928 err = mAudioSink->open( 1929 sampleRate, 1930 numChannels, 1931 (audio_channel_mask_t)channelMask, 1932 audioFormat, 1933 0 /* bufferCount - unused */, 1934 &NuPlayer::Renderer::AudioSinkCallback, 1935 this, 1936 (audio_output_flags_t)offloadFlags, 1937 &offloadInfo); 1938 1939 if (err == OK) { 1940 err = mAudioSink->setPlaybackRate(mPlaybackSettings); 1941 } 1942 1943 if (err == OK) { 1944 // If the playback is offloaded to h/w, we pass 1945 // the HAL some metadata information. 1946 // We don't want to do this for PCM because it 1947 // will be going through the AudioFlinger mixer 1948 // before reaching the hardware. 1949 // TODO 1950 mCurrentOffloadInfo = offloadInfo; 1951 if (!mPaused) { // for preview mode, don't start if paused 1952 err = mAudioSink->start(); 1953 } 1954 ALOGV_IF(err == OK, "openAudioSink: offload succeeded"); 1955 } 1956 if (err != OK) { 1957 // Clean up, fall back to non offload mode. 1958 mAudioSink->close(); 1959 onDisableOffloadAudio(); 1960 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER; 1961 ALOGV("openAudioSink: offload failed"); 1962 if (offloadOnly) { 1963 notifyAudioTearDown(kForceNonOffload); 1964 } 1965 } else { 1966 mUseAudioCallback = true; // offload mode transfers data through callback 1967 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message. 1968 } 1969 } 1970 } 1971 if (!offloadOnly && !offloadingAudio()) { 1972 ALOGV("openAudioSink: open AudioSink in NON-offload mode"); 1973 uint32_t pcmFlags = flags; 1974 pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; 1975 1976 const PcmInfo info = { 1977 (audio_channel_mask_t)channelMask, 1978 (audio_output_flags_t)pcmFlags, 1979 AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat 1980 numChannels, 1981 sampleRate 1982 }; 1983 if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) { 1984 ALOGV("openAudioSink: no change in pcm mode"); 1985 // no change from previous configuration, everything ok. 1986 return OK; 1987 } 1988 1989 audioSinkChanged = true; 1990 mAudioSink->close(); 1991 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER; 1992 // Note: It is possible to set up the callback, but not use it to send audio data. 1993 // This requires a fix in AudioSink to explicitly specify the transfer mode. 1994 mUseAudioCallback = getUseAudioCallbackSetting(); 1995 if (mUseAudioCallback) { 1996 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message. 1997 } 1998 1999 // Compute the desired buffer size. 2000 // For callback mode, the amount of time before wakeup is about half the buffer size. 2001 const uint32_t frameCount = 2002 (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000; 2003 2004 // The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct 2005 // AudioSink. We don't want this when there's video because it will cause a video seek to 2006 // the previous I frame. But we do want this when there's only audio because it will give 2007 // NuPlayer a chance to switch from non-offload mode to offload mode. 2008 // So we only set doNotReconnect when there's no video. 2009 const bool doNotReconnect = !hasVideo; 2010 2011 // We should always be able to set our playback settings if the sink is closed. 2012 LOG_ALWAYS_FATAL_IF(mAudioSink->setPlaybackRate(mPlaybackSettings) != OK, 2013 "onOpenAudioSink: can't set playback rate on closed sink"); 2014 status_t err = mAudioSink->open( 2015 sampleRate, 2016 numChannels, 2017 (audio_channel_mask_t)channelMask, 2018 AUDIO_FORMAT_PCM_16_BIT, 2019 0 /* bufferCount - unused */, 2020 mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL, 2021 mUseAudioCallback ? this : NULL, 2022 (audio_output_flags_t)pcmFlags, 2023 NULL, 2024 doNotReconnect, 2025 frameCount); 2026 if (err != OK) { 2027 ALOGW("openAudioSink: non offloaded open failed status: %d", err); 2028 mAudioSink->close(); 2029 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER; 2030 return err; 2031 } 2032 mCurrentPcmInfo = info; 2033 if (!mPaused) { // for preview mode, don't start if paused 2034 mAudioSink->start(); 2035 } 2036 } 2037 if (audioSinkChanged) { 2038 onAudioSinkChanged(); 2039 } 2040 mAudioTornDown = false; 2041 return OK; 2042} 2043 2044void NuPlayer::Renderer::onCloseAudioSink() { 2045 mAudioSink->close(); 2046 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER; 2047 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER; 2048} 2049 2050void NuPlayer::Renderer::onChangeAudioFormat( 2051 const sp<AMessage> &meta, const sp<AMessage> ¬ify) { 2052 sp<AMessage> format; 2053 CHECK(meta->findMessage("format", &format)); 2054 2055 int32_t offloadOnly; 2056 CHECK(meta->findInt32("offload-only", &offloadOnly)); 2057 2058 int32_t hasVideo; 2059 CHECK(meta->findInt32("has-video", &hasVideo)); 2060 2061 uint32_t flags; 2062 CHECK(meta->findInt32("flags", (int32_t *)&flags)); 2063 2064 uint32_t isStreaming; 2065 CHECK(meta->findInt32("isStreaming", (int32_t *)&isStreaming)); 2066 2067 status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags, isStreaming); 2068 2069 if (err != OK) { 2070 notify->setInt32("err", err); 2071 } 2072 notify->post(); 2073} 2074 2075} // namespace android 2076 2077