NuPlayerRenderer.cpp revision 4d7ac854c5a45d0e3af3d0af78b5a8c9807cbec6
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "NuPlayerRenderer" 19#include <utils/Log.h> 20 21#include "NuPlayerRenderer.h" 22#include <cutils/properties.h> 23#include <media/stagefright/foundation/ABuffer.h> 24#include <media/stagefright/foundation/ADebug.h> 25#include <media/stagefright/foundation/AMessage.h> 26#include <media/stagefright/foundation/AUtils.h> 27#include <media/stagefright/foundation/AWakeLock.h> 28#include <media/stagefright/MediaClock.h> 29#include <media/stagefright/MediaErrors.h> 30#include <media/stagefright/MetaData.h> 31#include <media/stagefright/Utils.h> 32#include <media/stagefright/VideoFrameScheduler.h> 33 34#include <inttypes.h> 35 36namespace android { 37 38/* 39 * Example of common configuration settings in shell script form 40 41 #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager 42 adb shell setprop audio.offload.disable 1 43 44 #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager 45 adb shell setprop audio.offload.video 1 46 47 #Use audio callbacks for PCM data 48 adb shell setprop media.stagefright.audio.cbk 1 49 50 #Use deep buffer for PCM data with video (it is generally enabled for audio-only) 51 adb shell setprop media.stagefright.audio.deep 1 52 53 #Set size of buffers for pcm audio sink in msec (example: 1000 msec) 54 adb shell setprop media.stagefright.audio.sink 1000 55 56 * These configurations take effect for the next track played (not the current track). 57 */ 58 59static inline bool getUseAudioCallbackSetting() { 60 return property_get_bool("media.stagefright.audio.cbk", false /* default_value */); 61} 62 63static inline int32_t getAudioSinkPcmMsSetting() { 64 return property_get_int32( 65 "media.stagefright.audio.sink", 500 /* default_value */); 66} 67 68// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink 69// is closed to allow the audio DSP to power down. 70static const int64_t kOffloadPauseMaxUs = 10000000ll; 71 72// static 73const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = { 74 AUDIO_CHANNEL_NONE, 75 AUDIO_OUTPUT_FLAG_NONE, 76 AUDIO_FORMAT_INVALID, 77 0, // mNumChannels 78 0 // mSampleRate 79}; 80 81// static 82const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; 83 84NuPlayer::Renderer::Renderer( 85 const sp<MediaPlayerBase::AudioSink> &sink, 86 const sp<AMessage> ¬ify, 87 uint32_t flags) 88 : mAudioSink(sink), 89 mNotify(notify), 90 mFlags(flags), 91 mNumFramesWritten(0), 92 mDrainAudioQueuePending(false), 93 mDrainVideoQueuePending(false), 94 mAudioQueueGeneration(0), 95 mVideoQueueGeneration(0), 96 mAudioDrainGeneration(0), 97 mVideoDrainGeneration(0), 98 mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT), 99 mAudioFirstAnchorTimeMediaUs(-1), 100 mAnchorTimeMediaUs(-1), 101 mAnchorNumFramesWritten(-1), 102 mVideoLateByUs(0ll), 103 mHasAudio(false), 104 mHasVideo(false), 105 mNotifyCompleteAudio(false), 106 mNotifyCompleteVideo(false), 107 mSyncQueues(false), 108 mPaused(false), 109 mVideoSampleReceived(false), 110 mVideoRenderingStarted(false), 111 mVideoRenderingStartGeneration(0), 112 mAudioRenderingStartGeneration(0), 113 mAudioOffloadPauseTimeoutGeneration(0), 114 mAudioTornDown(false), 115 mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER), 116 mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER), 117 mTotalBuffersQueued(0), 118 mLastAudioBufferDrained(0), 119 mUseAudioCallback(false), 120 mWakeLock(new AWakeLock()) { 121 mMediaClock = new MediaClock; 122 mPlaybackRate = mPlaybackSettings.mSpeed; 123 mMediaClock->setPlaybackRate(mPlaybackRate); 124} 125 126NuPlayer::Renderer::~Renderer() { 127 if (offloadingAudio()) { 128 mAudioSink->stop(); 129 mAudioSink->flush(); 130 mAudioSink->close(); 131 } 132} 133 134void NuPlayer::Renderer::queueBuffer( 135 bool audio, 136 const sp<ABuffer> &buffer, 137 const sp<AMessage> ¬ifyConsumed) { 138 sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this); 139 msg->setInt32("queueGeneration", getQueueGeneration(audio)); 140 msg->setInt32("audio", static_cast<int32_t>(audio)); 141 msg->setBuffer("buffer", buffer); 142 msg->setMessage("notifyConsumed", notifyConsumed); 143 msg->post(); 144} 145 146void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) { 147 CHECK_NE(finalResult, (status_t)OK); 148 149 sp<AMessage> msg = new AMessage(kWhatQueueEOS, this); 150 msg->setInt32("queueGeneration", getQueueGeneration(audio)); 151 msg->setInt32("audio", static_cast<int32_t>(audio)); 152 msg->setInt32("finalResult", finalResult); 153 msg->post(); 154} 155 156status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) { 157 sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this); 158 writeToAMessage(msg, rate); 159 sp<AMessage> response; 160 status_t err = msg->postAndAwaitResponse(&response); 161 if (err == OK && response != NULL) { 162 CHECK(response->findInt32("err", &err)); 163 } 164 return err; 165} 166 167status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) { 168 if (rate.mSpeed == 0.f) { 169 onPause(); 170 // don't call audiosink's setPlaybackRate if pausing, as pitch does not 171 // have to correspond to the any non-0 speed (e.g old speed). Keep 172 // settings nonetheless, using the old speed, in case audiosink changes. 173 AudioPlaybackRate newRate = rate; 174 newRate.mSpeed = mPlaybackSettings.mSpeed; 175 mPlaybackSettings = newRate; 176 return OK; 177 } 178 179 if (mAudioSink != NULL && mAudioSink->ready()) { 180 status_t err = mAudioSink->setPlaybackRate(rate); 181 if (err != OK) { 182 return err; 183 } 184 } 185 mPlaybackSettings = rate; 186 mPlaybackRate = rate.mSpeed; 187 mMediaClock->setPlaybackRate(mPlaybackRate); 188 return OK; 189} 190 191status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) { 192 sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this); 193 sp<AMessage> response; 194 status_t err = msg->postAndAwaitResponse(&response); 195 if (err == OK && response != NULL) { 196 CHECK(response->findInt32("err", &err)); 197 if (err == OK) { 198 readFromAMessage(response, rate); 199 } 200 } 201 return err; 202} 203 204status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) { 205 if (mAudioSink != NULL && mAudioSink->ready()) { 206 status_t err = mAudioSink->getPlaybackRate(rate); 207 if (err == OK) { 208 if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) { 209 ALOGW("correcting mismatch in internal/external playback rate"); 210 } 211 // get playback settings used by audiosink, as it may be 212 // slightly off due to audiosink not taking small changes. 213 mPlaybackSettings = *rate; 214 if (mPaused) { 215 rate->mSpeed = 0.f; 216 } 217 } 218 return err; 219 } 220 *rate = mPlaybackSettings; 221 return OK; 222} 223 224status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) { 225 sp<AMessage> msg = new AMessage(kWhatConfigSync, this); 226 writeToAMessage(msg, sync, videoFpsHint); 227 sp<AMessage> response; 228 status_t err = msg->postAndAwaitResponse(&response); 229 if (err == OK && response != NULL) { 230 CHECK(response->findInt32("err", &err)); 231 } 232 return err; 233} 234 235status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) { 236 if (sync.mSource != AVSYNC_SOURCE_DEFAULT) { 237 return BAD_VALUE; 238 } 239 // TODO: support sync sources 240 return INVALID_OPERATION; 241} 242 243status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) { 244 sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this); 245 sp<AMessage> response; 246 status_t err = msg->postAndAwaitResponse(&response); 247 if (err == OK && response != NULL) { 248 CHECK(response->findInt32("err", &err)); 249 if (err == OK) { 250 readFromAMessage(response, sync, videoFps); 251 } 252 } 253 return err; 254} 255 256status_t NuPlayer::Renderer::onGetSyncSettings( 257 AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) { 258 *sync = mSyncSettings; 259 *videoFps = -1.f; 260 return OK; 261} 262 263void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) { 264 { 265 Mutex::Autolock autoLock(mLock); 266 if (audio) { 267 mNotifyCompleteAudio |= notifyComplete; 268 clearAudioFirstAnchorTime_l(); 269 ++mAudioQueueGeneration; 270 ++mAudioDrainGeneration; 271 } else { 272 mNotifyCompleteVideo |= notifyComplete; 273 ++mVideoQueueGeneration; 274 ++mVideoDrainGeneration; 275 } 276 277 clearAnchorTime_l(); 278 mVideoLateByUs = 0; 279 mSyncQueues = false; 280 } 281 282 sp<AMessage> msg = new AMessage(kWhatFlush, this); 283 msg->setInt32("audio", static_cast<int32_t>(audio)); 284 msg->post(); 285} 286 287void NuPlayer::Renderer::signalTimeDiscontinuity() { 288} 289 290void NuPlayer::Renderer::signalDisableOffloadAudio() { 291 (new AMessage(kWhatDisableOffloadAudio, this))->post(); 292} 293 294void NuPlayer::Renderer::signalEnableOffloadAudio() { 295 (new AMessage(kWhatEnableOffloadAudio, this))->post(); 296} 297 298void NuPlayer::Renderer::pause() { 299 (new AMessage(kWhatPause, this))->post(); 300} 301 302void NuPlayer::Renderer::resume() { 303 (new AMessage(kWhatResume, this))->post(); 304} 305 306void NuPlayer::Renderer::setVideoFrameRate(float fps) { 307 sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this); 308 msg->setFloat("frame-rate", fps); 309 msg->post(); 310} 311 312// Called on any threads. 313status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) { 314 return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs); 315} 316 317void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() { 318 mAudioFirstAnchorTimeMediaUs = -1; 319 mMediaClock->setStartingTimeMedia(-1); 320} 321 322void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) { 323 if (mAudioFirstAnchorTimeMediaUs == -1) { 324 mAudioFirstAnchorTimeMediaUs = mediaUs; 325 mMediaClock->setStartingTimeMedia(mediaUs); 326 } 327} 328 329void NuPlayer::Renderer::clearAnchorTime_l() { 330 mMediaClock->clearAnchor(); 331 mAnchorTimeMediaUs = -1; 332 mAnchorNumFramesWritten = -1; 333} 334 335void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) { 336 Mutex::Autolock autoLock(mLock); 337 mVideoLateByUs = lateUs; 338} 339 340int64_t NuPlayer::Renderer::getVideoLateByUs() { 341 Mutex::Autolock autoLock(mLock); 342 return mVideoLateByUs; 343} 344 345status_t NuPlayer::Renderer::openAudioSink( 346 const sp<AMessage> &format, 347 bool offloadOnly, 348 bool hasVideo, 349 uint32_t flags, 350 bool *isOffloaded) { 351 sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this); 352 msg->setMessage("format", format); 353 msg->setInt32("offload-only", offloadOnly); 354 msg->setInt32("has-video", hasVideo); 355 msg->setInt32("flags", flags); 356 357 sp<AMessage> response; 358 msg->postAndAwaitResponse(&response); 359 360 int32_t err; 361 if (!response->findInt32("err", &err)) { 362 err = INVALID_OPERATION; 363 } else if (err == OK && isOffloaded != NULL) { 364 int32_t offload; 365 CHECK(response->findInt32("offload", &offload)); 366 *isOffloaded = (offload != 0); 367 } 368 return err; 369} 370 371void NuPlayer::Renderer::closeAudioSink() { 372 sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this); 373 374 sp<AMessage> response; 375 msg->postAndAwaitResponse(&response); 376} 377 378void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { 379 switch (msg->what()) { 380 case kWhatOpenAudioSink: 381 { 382 sp<AMessage> format; 383 CHECK(msg->findMessage("format", &format)); 384 385 int32_t offloadOnly; 386 CHECK(msg->findInt32("offload-only", &offloadOnly)); 387 388 int32_t hasVideo; 389 CHECK(msg->findInt32("has-video", &hasVideo)); 390 391 uint32_t flags; 392 CHECK(msg->findInt32("flags", (int32_t *)&flags)); 393 394 status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags); 395 396 sp<AMessage> response = new AMessage; 397 response->setInt32("err", err); 398 response->setInt32("offload", offloadingAudio()); 399 400 sp<AReplyToken> replyID; 401 CHECK(msg->senderAwaitsResponse(&replyID)); 402 response->postReply(replyID); 403 404 break; 405 } 406 407 case kWhatCloseAudioSink: 408 { 409 sp<AReplyToken> replyID; 410 CHECK(msg->senderAwaitsResponse(&replyID)); 411 412 onCloseAudioSink(); 413 414 sp<AMessage> response = new AMessage; 415 response->postReply(replyID); 416 break; 417 } 418 419 case kWhatStopAudioSink: 420 { 421 mAudioSink->stop(); 422 break; 423 } 424 425 case kWhatDrainAudioQueue: 426 { 427 mDrainAudioQueuePending = false; 428 429 int32_t generation; 430 CHECK(msg->findInt32("drainGeneration", &generation)); 431 if (generation != getDrainGeneration(true /* audio */)) { 432 break; 433 } 434 435 if (onDrainAudioQueue()) { 436 uint32_t numFramesPlayed; 437 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), 438 (status_t)OK); 439 440 uint32_t numFramesPendingPlayout = 441 mNumFramesWritten - numFramesPlayed; 442 443 // This is how long the audio sink will have data to 444 // play back. 445 int64_t delayUs = 446 mAudioSink->msecsPerFrame() 447 * numFramesPendingPlayout * 1000ll; 448 if (mPlaybackRate > 1.0f) { 449 delayUs /= mPlaybackRate; 450 } 451 452 // Let's give it more data after about half that time 453 // has elapsed. 454 Mutex::Autolock autoLock(mLock); 455 postDrainAudioQueue_l(delayUs / 2); 456 } 457 break; 458 } 459 460 case kWhatDrainVideoQueue: 461 { 462 int32_t generation; 463 CHECK(msg->findInt32("drainGeneration", &generation)); 464 if (generation != getDrainGeneration(false /* audio */)) { 465 break; 466 } 467 468 mDrainVideoQueuePending = false; 469 470 onDrainVideoQueue(); 471 472 postDrainVideoQueue(); 473 break; 474 } 475 476 case kWhatPostDrainVideoQueue: 477 { 478 int32_t generation; 479 CHECK(msg->findInt32("drainGeneration", &generation)); 480 if (generation != getDrainGeneration(false /* audio */)) { 481 break; 482 } 483 484 mDrainVideoQueuePending = false; 485 postDrainVideoQueue(); 486 break; 487 } 488 489 case kWhatQueueBuffer: 490 { 491 onQueueBuffer(msg); 492 break; 493 } 494 495 case kWhatQueueEOS: 496 { 497 onQueueEOS(msg); 498 break; 499 } 500 501 case kWhatConfigPlayback: 502 { 503 sp<AReplyToken> replyID; 504 CHECK(msg->senderAwaitsResponse(&replyID)); 505 AudioPlaybackRate rate; 506 readFromAMessage(msg, &rate); 507 status_t err = onConfigPlayback(rate); 508 sp<AMessage> response = new AMessage; 509 response->setInt32("err", err); 510 response->postReply(replyID); 511 break; 512 } 513 514 case kWhatGetPlaybackSettings: 515 { 516 sp<AReplyToken> replyID; 517 CHECK(msg->senderAwaitsResponse(&replyID)); 518 AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT; 519 status_t err = onGetPlaybackSettings(&rate); 520 sp<AMessage> response = new AMessage; 521 if (err == OK) { 522 writeToAMessage(response, rate); 523 } 524 response->setInt32("err", err); 525 response->postReply(replyID); 526 break; 527 } 528 529 case kWhatConfigSync: 530 { 531 sp<AReplyToken> replyID; 532 CHECK(msg->senderAwaitsResponse(&replyID)); 533 AVSyncSettings sync; 534 float videoFpsHint; 535 readFromAMessage(msg, &sync, &videoFpsHint); 536 status_t err = onConfigSync(sync, videoFpsHint); 537 sp<AMessage> response = new AMessage; 538 response->setInt32("err", err); 539 response->postReply(replyID); 540 break; 541 } 542 543 case kWhatGetSyncSettings: 544 { 545 sp<AReplyToken> replyID; 546 CHECK(msg->senderAwaitsResponse(&replyID)); 547 548 ALOGV("kWhatGetSyncSettings"); 549 AVSyncSettings sync; 550 float videoFps = -1.f; 551 status_t err = onGetSyncSettings(&sync, &videoFps); 552 sp<AMessage> response = new AMessage; 553 if (err == OK) { 554 writeToAMessage(response, sync, videoFps); 555 } 556 response->setInt32("err", err); 557 response->postReply(replyID); 558 break; 559 } 560 561 case kWhatFlush: 562 { 563 onFlush(msg); 564 break; 565 } 566 567 case kWhatDisableOffloadAudio: 568 { 569 onDisableOffloadAudio(); 570 break; 571 } 572 573 case kWhatEnableOffloadAudio: 574 { 575 onEnableOffloadAudio(); 576 break; 577 } 578 579 case kWhatPause: 580 { 581 onPause(); 582 break; 583 } 584 585 case kWhatResume: 586 { 587 onResume(); 588 break; 589 } 590 591 case kWhatSetVideoFrameRate: 592 { 593 float fps; 594 CHECK(msg->findFloat("frame-rate", &fps)); 595 onSetVideoFrameRate(fps); 596 break; 597 } 598 599 case kWhatAudioTearDown: 600 { 601 onAudioTearDown(kDueToError); 602 break; 603 } 604 605 case kWhatAudioOffloadPauseTimeout: 606 { 607 int32_t generation; 608 CHECK(msg->findInt32("drainGeneration", &generation)); 609 if (generation != mAudioOffloadPauseTimeoutGeneration) { 610 break; 611 } 612 ALOGV("Audio Offload tear down due to pause timeout."); 613 onAudioTearDown(kDueToTimeout); 614 mWakeLock->release(); 615 break; 616 } 617 618 default: 619 TRESPASS(); 620 break; 621 } 622} 623 624void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) { 625 if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) { 626 return; 627 } 628 629 if (mAudioQueue.empty()) { 630 return; 631 } 632 633 mDrainAudioQueuePending = true; 634 sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this); 635 msg->setInt32("drainGeneration", mAudioDrainGeneration); 636 msg->post(delayUs); 637} 638 639void NuPlayer::Renderer::prepareForMediaRenderingStart_l() { 640 mAudioRenderingStartGeneration = mAudioDrainGeneration; 641 mVideoRenderingStartGeneration = mVideoDrainGeneration; 642} 643 644void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() { 645 if (mVideoRenderingStartGeneration == mVideoDrainGeneration && 646 mAudioRenderingStartGeneration == mAudioDrainGeneration) { 647 mVideoRenderingStartGeneration = -1; 648 mAudioRenderingStartGeneration = -1; 649 650 sp<AMessage> notify = mNotify->dup(); 651 notify->setInt32("what", kWhatMediaRenderingStart); 652 notify->post(); 653 } 654} 655 656// static 657size_t NuPlayer::Renderer::AudioSinkCallback( 658 MediaPlayerBase::AudioSink * /* audioSink */, 659 void *buffer, 660 size_t size, 661 void *cookie, 662 MediaPlayerBase::AudioSink::cb_event_t event) { 663 NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie; 664 665 switch (event) { 666 case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER: 667 { 668 return me->fillAudioBuffer(buffer, size); 669 break; 670 } 671 672 case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END: 673 { 674 ALOGV("AudioSink::CB_EVENT_STREAM_END"); 675 me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM); 676 break; 677 } 678 679 case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN: 680 { 681 ALOGV("AudioSink::CB_EVENT_TEAR_DOWN"); 682 me->notifyAudioTearDown(); 683 break; 684 } 685 } 686 687 return 0; 688} 689 690size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { 691 Mutex::Autolock autoLock(mLock); 692 693 if (!mUseAudioCallback) { 694 return 0; 695 } 696 697 bool hasEOS = false; 698 699 size_t sizeCopied = 0; 700 bool firstEntry = true; 701 QueueEntry *entry; // will be valid after while loop if hasEOS is set. 702 while (sizeCopied < size && !mAudioQueue.empty()) { 703 entry = &*mAudioQueue.begin(); 704 705 if (entry->mBuffer == NULL) { // EOS 706 hasEOS = true; 707 mAudioQueue.erase(mAudioQueue.begin()); 708 break; 709 } 710 711 if (firstEntry && entry->mOffset == 0) { 712 firstEntry = false; 713 int64_t mediaTimeUs; 714 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 715 ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6); 716 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs); 717 } 718 719 size_t copy = entry->mBuffer->size() - entry->mOffset; 720 size_t sizeRemaining = size - sizeCopied; 721 if (copy > sizeRemaining) { 722 copy = sizeRemaining; 723 } 724 725 memcpy((char *)buffer + sizeCopied, 726 entry->mBuffer->data() + entry->mOffset, 727 copy); 728 729 entry->mOffset += copy; 730 if (entry->mOffset == entry->mBuffer->size()) { 731 entry->mNotifyConsumed->post(); 732 mAudioQueue.erase(mAudioQueue.begin()); 733 entry = NULL; 734 } 735 sizeCopied += copy; 736 737 notifyIfMediaRenderingStarted_l(); 738 } 739 740 if (mAudioFirstAnchorTimeMediaUs >= 0) { 741 int64_t nowUs = ALooper::GetNowUs(); 742 int64_t nowMediaUs = 743 mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs); 744 // we don't know how much data we are queueing for offloaded tracks. 745 mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX); 746 } 747 748 // for non-offloaded audio, we need to compute the frames written because 749 // there is no EVENT_STREAM_END notification. The frames written gives 750 // an estimate on the pending played out duration. 751 if (!offloadingAudio()) { 752 mNumFramesWritten += sizeCopied / mAudioSink->frameSize(); 753 } 754 755 if (hasEOS) { 756 (new AMessage(kWhatStopAudioSink, this))->post(); 757 // As there is currently no EVENT_STREAM_END callback notification for 758 // non-offloaded audio tracks, we need to post the EOS ourselves. 759 if (!offloadingAudio()) { 760 int64_t postEOSDelayUs = 0; 761 if (mAudioSink->needsTrailingPadding()) { 762 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs()); 763 } 764 ALOGV("fillAudioBuffer: notifyEOS " 765 "mNumFramesWritten:%u finalResult:%d postEOSDelay:%lld", 766 mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs); 767 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs); 768 } 769 } 770 return sizeCopied; 771} 772 773void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() { 774 List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it; 775 bool foundEOS = false; 776 while (it != mAudioQueue.end()) { 777 int32_t eos; 778 QueueEntry *entry = &*it++; 779 if (entry->mBuffer == NULL 780 || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) { 781 itEOS = it; 782 foundEOS = true; 783 } 784 } 785 786 if (foundEOS) { 787 // post all replies before EOS and drop the samples 788 for (it = mAudioQueue.begin(); it != itEOS; it++) { 789 if (it->mBuffer == NULL) { 790 // delay doesn't matter as we don't even have an AudioTrack 791 notifyEOS(true /* audio */, it->mFinalResult); 792 } else { 793 it->mNotifyConsumed->post(); 794 } 795 } 796 mAudioQueue.erase(mAudioQueue.begin(), itEOS); 797 } 798} 799 800bool NuPlayer::Renderer::onDrainAudioQueue() { 801 // TODO: This call to getPosition checks if AudioTrack has been created 802 // in AudioSink before draining audio. If AudioTrack doesn't exist, then 803 // CHECKs on getPosition will fail. 804 // We still need to figure out why AudioTrack is not created when 805 // this function is called. One possible reason could be leftover 806 // audio. Another possible place is to check whether decoder 807 // has received INFO_FORMAT_CHANGED as the first buffer since 808 // AudioSink is opened there, and possible interactions with flush 809 // immediately after start. Investigate error message 810 // "vorbis_dsp_synthesis returned -135", along with RTSP. 811 uint32_t numFramesPlayed; 812 if (mAudioSink->getPosition(&numFramesPlayed) != OK) { 813 // When getPosition fails, renderer will not reschedule the draining 814 // unless new samples are queued. 815 // If we have pending EOS (or "eos" marker for discontinuities), we need 816 // to post these now as NuPlayerDecoder might be waiting for it. 817 drainAudioQueueUntilLastEOS(); 818 819 ALOGW("onDrainAudioQueue(): audio sink is not ready"); 820 return false; 821 } 822 823#if 0 824 ssize_t numFramesAvailableToWrite = 825 mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed); 826 827 if (numFramesAvailableToWrite == mAudioSink->frameCount()) { 828 ALOGI("audio sink underrun"); 829 } else { 830 ALOGV("audio queue has %d frames left to play", 831 mAudioSink->frameCount() - numFramesAvailableToWrite); 832 } 833#endif 834 835 uint32_t prevFramesWritten = mNumFramesWritten; 836 while (!mAudioQueue.empty()) { 837 QueueEntry *entry = &*mAudioQueue.begin(); 838 839 mLastAudioBufferDrained = entry->mBufferOrdinal; 840 841 if (entry->mBuffer == NULL) { 842 // EOS 843 int64_t postEOSDelayUs = 0; 844 if (mAudioSink->needsTrailingPadding()) { 845 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs()); 846 } 847 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs); 848 849 mAudioQueue.erase(mAudioQueue.begin()); 850 entry = NULL; 851 if (mAudioSink->needsTrailingPadding()) { 852 // If we're not in gapless playback (i.e. through setNextPlayer), we 853 // need to stop the track here, because that will play out the last 854 // little bit at the end of the file. Otherwise short files won't play. 855 mAudioSink->stop(); 856 mNumFramesWritten = 0; 857 } 858 return false; 859 } 860 861 // ignore 0-sized buffer which could be EOS marker with no data 862 if (entry->mOffset == 0 && entry->mBuffer->size() > 0) { 863 int64_t mediaTimeUs; 864 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 865 ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs", 866 mediaTimeUs / 1E6); 867 onNewAudioMediaTime(mediaTimeUs); 868 } 869 870 size_t copy = entry->mBuffer->size() - entry->mOffset; 871 872 ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, 873 copy, false /* blocking */); 874 if (written < 0) { 875 // An error in AudioSink write. Perhaps the AudioSink was not properly opened. 876 if (written == WOULD_BLOCK) { 877 ALOGV("AudioSink write would block when writing %zu bytes", copy); 878 } else { 879 ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy); 880 // This can only happen when AudioSink was opened with doNotReconnect flag set to 881 // true, in which case the NuPlayer will handle the reconnect. 882 notifyAudioTearDown(); 883 } 884 break; 885 } 886 887 entry->mOffset += written; 888 if (entry->mOffset == entry->mBuffer->size()) { 889 entry->mNotifyConsumed->post(); 890 mAudioQueue.erase(mAudioQueue.begin()); 891 892 entry = NULL; 893 } 894 895 size_t copiedFrames = written / mAudioSink->frameSize(); 896 mNumFramesWritten += copiedFrames; 897 898 { 899 Mutex::Autolock autoLock(mLock); 900 notifyIfMediaRenderingStarted_l(); 901 } 902 903 if (written != (ssize_t)copy) { 904 // A short count was received from AudioSink::write() 905 // 906 // AudioSink write is called in non-blocking mode. 907 // It may return with a short count when: 908 // 909 // 1) Size to be copied is not a multiple of the frame size. We consider this fatal. 910 // 2) The data to be copied exceeds the available buffer in AudioSink. 911 // 3) An error occurs and data has been partially copied to the buffer in AudioSink. 912 // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded. 913 914 // (Case 1) 915 // Must be a multiple of the frame size. If it is not a multiple of a frame size, it 916 // needs to fail, as we should not carry over fractional frames between calls. 917 CHECK_EQ(copy % mAudioSink->frameSize(), 0); 918 919 // (Case 2, 3, 4) 920 // Return early to the caller. 921 // Beware of calling immediately again as this may busy-loop if you are not careful. 922 ALOGV("AudioSink write short frame count %zd < %zu", written, copy); 923 break; 924 } 925 } 926 int64_t maxTimeMedia; 927 { 928 Mutex::Autolock autoLock(mLock); 929 maxTimeMedia = 930 mAnchorTimeMediaUs + 931 (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL) 932 * 1000LL * mAudioSink->msecsPerFrame()); 933 } 934 mMediaClock->updateMaxTimeMedia(maxTimeMedia); 935 936 // calculate whether we need to reschedule another write. 937 bool reschedule = !mAudioQueue.empty() 938 && (!mPaused 939 || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers 940 //ALOGD("reschedule:%d empty:%d mPaused:%d prevFramesWritten:%u mNumFramesWritten:%u", 941 // reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten); 942 return reschedule; 943} 944 945int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) { 946 int32_t sampleRate = offloadingAudio() ? 947 mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate; 948 if (sampleRate == 0) { 949 ALOGE("sampleRate is 0 in %s mode", offloadingAudio() ? "offload" : "non-offload"); 950 return 0; 951 } 952 // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours. 953 return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate); 954} 955 956// Calculate duration of pending samples if played at normal rate (i.e., 1.0). 957int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) { 958 int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten); 959 return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs); 960} 961 962int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) { 963 int64_t realUs; 964 if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) { 965 // If failed to get current position, e.g. due to audio clock is 966 // not ready, then just play out video immediately without delay. 967 return nowUs; 968 } 969 return realUs; 970} 971 972void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) { 973 Mutex::Autolock autoLock(mLock); 974 // TRICKY: vorbis decoder generates multiple frames with the same 975 // timestamp, so only update on the first frame with a given timestamp 976 if (mediaTimeUs == mAnchorTimeMediaUs) { 977 return; 978 } 979 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs); 980 int64_t nowUs = ALooper::GetNowUs(); 981 int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs); 982 mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs); 983 mAnchorNumFramesWritten = mNumFramesWritten; 984 mAnchorTimeMediaUs = mediaTimeUs; 985} 986 987// Called without mLock acquired. 988void NuPlayer::Renderer::postDrainVideoQueue() { 989 if (mDrainVideoQueuePending 990 || getSyncQueues() 991 || (mPaused && mVideoSampleReceived)) { 992 return; 993 } 994 995 if (mVideoQueue.empty()) { 996 return; 997 } 998 999 QueueEntry &entry = *mVideoQueue.begin(); 1000 1001 sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this); 1002 msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */)); 1003 1004 if (entry.mBuffer == NULL) { 1005 // EOS doesn't carry a timestamp. 1006 msg->post(); 1007 mDrainVideoQueuePending = true; 1008 return; 1009 } 1010 1011 int64_t delayUs; 1012 int64_t nowUs = ALooper::GetNowUs(); 1013 int64_t realTimeUs; 1014 if (mFlags & FLAG_REAL_TIME) { 1015 int64_t mediaTimeUs; 1016 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 1017 realTimeUs = mediaTimeUs; 1018 } else { 1019 int64_t mediaTimeUs; 1020 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 1021 1022 { 1023 Mutex::Autolock autoLock(mLock); 1024 if (mAnchorTimeMediaUs < 0) { 1025 mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs); 1026 mAnchorTimeMediaUs = mediaTimeUs; 1027 realTimeUs = nowUs; 1028 } else { 1029 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); 1030 } 1031 } 1032 if (!mHasAudio) { 1033 // smooth out videos >= 10fps 1034 mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000); 1035 } 1036 1037 // Heuristics to handle situation when media time changed without a 1038 // discontinuity. If we have not drained an audio buffer that was 1039 // received after this buffer, repost in 10 msec. Otherwise repost 1040 // in 500 msec. 1041 delayUs = realTimeUs - nowUs; 1042 if (delayUs > 500000) { 1043 int64_t postDelayUs = 500000; 1044 if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) { 1045 postDelayUs = 10000; 1046 } 1047 msg->setWhat(kWhatPostDrainVideoQueue); 1048 msg->post(postDelayUs); 1049 mVideoScheduler->restart(); 1050 ALOGI("possible video time jump of %dms, retrying in %dms", 1051 (int)(delayUs / 1000), (int)(postDelayUs / 1000)); 1052 mDrainVideoQueuePending = true; 1053 return; 1054 } 1055 } 1056 1057 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000; 1058 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000); 1059 1060 delayUs = realTimeUs - nowUs; 1061 1062 ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs); 1063 // post 2 display refreshes before rendering is due 1064 msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0); 1065 1066 mDrainVideoQueuePending = true; 1067} 1068 1069void NuPlayer::Renderer::onDrainVideoQueue() { 1070 if (mVideoQueue.empty()) { 1071 return; 1072 } 1073 1074 QueueEntry *entry = &*mVideoQueue.begin(); 1075 1076 if (entry->mBuffer == NULL) { 1077 // EOS 1078 1079 notifyEOS(false /* audio */, entry->mFinalResult); 1080 1081 mVideoQueue.erase(mVideoQueue.begin()); 1082 entry = NULL; 1083 1084 setVideoLateByUs(0); 1085 return; 1086 } 1087 1088 int64_t nowUs = -1; 1089 int64_t realTimeUs; 1090 if (mFlags & FLAG_REAL_TIME) { 1091 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs)); 1092 } else { 1093 int64_t mediaTimeUs; 1094 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 1095 1096 nowUs = ALooper::GetNowUs(); 1097 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); 1098 } 1099 1100 bool tooLate = false; 1101 1102 if (!mPaused) { 1103 if (nowUs == -1) { 1104 nowUs = ALooper::GetNowUs(); 1105 } 1106 setVideoLateByUs(nowUs - realTimeUs); 1107 tooLate = (mVideoLateByUs > 40000); 1108 1109 if (tooLate) { 1110 ALOGV("video late by %lld us (%.2f secs)", 1111 (long long)mVideoLateByUs, mVideoLateByUs / 1E6); 1112 } else { 1113 int64_t mediaUs = 0; 1114 mMediaClock->getMediaTime(realTimeUs, &mediaUs); 1115 ALOGV("rendering video at media time %.2f secs", 1116 (mFlags & FLAG_REAL_TIME ? realTimeUs : 1117 mediaUs) / 1E6); 1118 } 1119 } else { 1120 setVideoLateByUs(0); 1121 if (!mVideoSampleReceived && !mHasAudio) { 1122 // This will ensure that the first frame after a flush won't be used as anchor 1123 // when renderer is in paused state, because resume can happen any time after seek. 1124 Mutex::Autolock autoLock(mLock); 1125 clearAnchorTime_l(); 1126 } 1127 } 1128 1129 entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll); 1130 entry->mNotifyConsumed->setInt32("render", !tooLate); 1131 entry->mNotifyConsumed->post(); 1132 mVideoQueue.erase(mVideoQueue.begin()); 1133 entry = NULL; 1134 1135 mVideoSampleReceived = true; 1136 1137 if (!mPaused) { 1138 if (!mVideoRenderingStarted) { 1139 mVideoRenderingStarted = true; 1140 notifyVideoRenderingStart(); 1141 } 1142 Mutex::Autolock autoLock(mLock); 1143 notifyIfMediaRenderingStarted_l(); 1144 } 1145} 1146 1147void NuPlayer::Renderer::notifyVideoRenderingStart() { 1148 sp<AMessage> notify = mNotify->dup(); 1149 notify->setInt32("what", kWhatVideoRenderingStart); 1150 notify->post(); 1151} 1152 1153void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) { 1154 sp<AMessage> notify = mNotify->dup(); 1155 notify->setInt32("what", kWhatEOS); 1156 notify->setInt32("audio", static_cast<int32_t>(audio)); 1157 notify->setInt32("finalResult", finalResult); 1158 notify->post(delayUs); 1159} 1160 1161void NuPlayer::Renderer::notifyAudioTearDown() { 1162 (new AMessage(kWhatAudioTearDown, this))->post(); 1163} 1164 1165void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { 1166 int32_t audio; 1167 CHECK(msg->findInt32("audio", &audio)); 1168 1169 if (dropBufferIfStale(audio, msg)) { 1170 return; 1171 } 1172 1173 if (audio) { 1174 mHasAudio = true; 1175 } else { 1176 mHasVideo = true; 1177 } 1178 1179 if (mHasVideo) { 1180 if (mVideoScheduler == NULL) { 1181 mVideoScheduler = new VideoFrameScheduler(); 1182 mVideoScheduler->init(); 1183 } 1184 } 1185 1186 sp<ABuffer> buffer; 1187 CHECK(msg->findBuffer("buffer", &buffer)); 1188 1189 sp<AMessage> notifyConsumed; 1190 CHECK(msg->findMessage("notifyConsumed", ¬ifyConsumed)); 1191 1192 QueueEntry entry; 1193 entry.mBuffer = buffer; 1194 entry.mNotifyConsumed = notifyConsumed; 1195 entry.mOffset = 0; 1196 entry.mFinalResult = OK; 1197 entry.mBufferOrdinal = ++mTotalBuffersQueued; 1198 1199 if (audio) { 1200 Mutex::Autolock autoLock(mLock); 1201 mAudioQueue.push_back(entry); 1202 postDrainAudioQueue_l(); 1203 } else { 1204 mVideoQueue.push_back(entry); 1205 postDrainVideoQueue(); 1206 } 1207 1208 Mutex::Autolock autoLock(mLock); 1209 if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) { 1210 return; 1211 } 1212 1213 sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer; 1214 sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer; 1215 1216 if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) { 1217 // EOS signalled on either queue. 1218 syncQueuesDone_l(); 1219 return; 1220 } 1221 1222 int64_t firstAudioTimeUs; 1223 int64_t firstVideoTimeUs; 1224 CHECK(firstAudioBuffer->meta() 1225 ->findInt64("timeUs", &firstAudioTimeUs)); 1226 CHECK(firstVideoBuffer->meta() 1227 ->findInt64("timeUs", &firstVideoTimeUs)); 1228 1229 int64_t diff = firstVideoTimeUs - firstAudioTimeUs; 1230 1231 ALOGV("queueDiff = %.2f secs", diff / 1E6); 1232 1233 if (diff > 100000ll) { 1234 // Audio data starts More than 0.1 secs before video. 1235 // Drop some audio. 1236 1237 (*mAudioQueue.begin()).mNotifyConsumed->post(); 1238 mAudioQueue.erase(mAudioQueue.begin()); 1239 return; 1240 } 1241 1242 syncQueuesDone_l(); 1243} 1244 1245void NuPlayer::Renderer::syncQueuesDone_l() { 1246 if (!mSyncQueues) { 1247 return; 1248 } 1249 1250 mSyncQueues = false; 1251 1252 if (!mAudioQueue.empty()) { 1253 postDrainAudioQueue_l(); 1254 } 1255 1256 if (!mVideoQueue.empty()) { 1257 mLock.unlock(); 1258 postDrainVideoQueue(); 1259 mLock.lock(); 1260 } 1261} 1262 1263void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) { 1264 int32_t audio; 1265 CHECK(msg->findInt32("audio", &audio)); 1266 1267 if (dropBufferIfStale(audio, msg)) { 1268 return; 1269 } 1270 1271 int32_t finalResult; 1272 CHECK(msg->findInt32("finalResult", &finalResult)); 1273 1274 QueueEntry entry; 1275 entry.mOffset = 0; 1276 entry.mFinalResult = finalResult; 1277 1278 if (audio) { 1279 Mutex::Autolock autoLock(mLock); 1280 if (mAudioQueue.empty() && mSyncQueues) { 1281 syncQueuesDone_l(); 1282 } 1283 mAudioQueue.push_back(entry); 1284 postDrainAudioQueue_l(); 1285 } else { 1286 if (mVideoQueue.empty() && getSyncQueues()) { 1287 Mutex::Autolock autoLock(mLock); 1288 syncQueuesDone_l(); 1289 } 1290 mVideoQueue.push_back(entry); 1291 postDrainVideoQueue(); 1292 } 1293} 1294 1295void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { 1296 int32_t audio, notifyComplete; 1297 CHECK(msg->findInt32("audio", &audio)); 1298 1299 { 1300 Mutex::Autolock autoLock(mLock); 1301 if (audio) { 1302 notifyComplete = mNotifyCompleteAudio; 1303 mNotifyCompleteAudio = false; 1304 } else { 1305 notifyComplete = mNotifyCompleteVideo; 1306 mNotifyCompleteVideo = false; 1307 } 1308 1309 // If we're currently syncing the queues, i.e. dropping audio while 1310 // aligning the first audio/video buffer times and only one of the 1311 // two queues has data, we may starve that queue by not requesting 1312 // more buffers from the decoder. If the other source then encounters 1313 // a discontinuity that leads to flushing, we'll never find the 1314 // corresponding discontinuity on the other queue. 1315 // Therefore we'll stop syncing the queues if at least one of them 1316 // is flushed. 1317 syncQueuesDone_l(); 1318 clearAnchorTime_l(); 1319 } 1320 1321 ALOGV("flushing %s", audio ? "audio" : "video"); 1322 if (audio) { 1323 { 1324 Mutex::Autolock autoLock(mLock); 1325 flushQueue(&mAudioQueue); 1326 1327 ++mAudioDrainGeneration; 1328 prepareForMediaRenderingStart_l(); 1329 1330 // the frame count will be reset after flush. 1331 clearAudioFirstAnchorTime_l(); 1332 } 1333 1334 mDrainAudioQueuePending = false; 1335 1336 if (offloadingAudio()) { 1337 mAudioSink->pause(); 1338 mAudioSink->flush(); 1339 if (!mPaused) { 1340 mAudioSink->start(); 1341 } 1342 } else { 1343 mAudioSink->pause(); 1344 mAudioSink->flush(); 1345 // Call stop() to signal to the AudioSink to completely fill the 1346 // internal buffer before resuming playback. 1347 mAudioSink->stop(); 1348 if (!mPaused) { 1349 mAudioSink->start(); 1350 } 1351 mNumFramesWritten = 0; 1352 } 1353 } else { 1354 flushQueue(&mVideoQueue); 1355 1356 mDrainVideoQueuePending = false; 1357 1358 if (mVideoScheduler != NULL) { 1359 mVideoScheduler->restart(); 1360 } 1361 1362 Mutex::Autolock autoLock(mLock); 1363 ++mVideoDrainGeneration; 1364 prepareForMediaRenderingStart_l(); 1365 } 1366 1367 mVideoSampleReceived = false; 1368 1369 if (notifyComplete) { 1370 notifyFlushComplete(audio); 1371 } 1372} 1373 1374void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) { 1375 while (!queue->empty()) { 1376 QueueEntry *entry = &*queue->begin(); 1377 1378 if (entry->mBuffer != NULL) { 1379 entry->mNotifyConsumed->post(); 1380 } 1381 1382 queue->erase(queue->begin()); 1383 entry = NULL; 1384 } 1385} 1386 1387void NuPlayer::Renderer::notifyFlushComplete(bool audio) { 1388 sp<AMessage> notify = mNotify->dup(); 1389 notify->setInt32("what", kWhatFlushComplete); 1390 notify->setInt32("audio", static_cast<int32_t>(audio)); 1391 notify->post(); 1392} 1393 1394bool NuPlayer::Renderer::dropBufferIfStale( 1395 bool audio, const sp<AMessage> &msg) { 1396 int32_t queueGeneration; 1397 CHECK(msg->findInt32("queueGeneration", &queueGeneration)); 1398 1399 if (queueGeneration == getQueueGeneration(audio)) { 1400 return false; 1401 } 1402 1403 sp<AMessage> notifyConsumed; 1404 if (msg->findMessage("notifyConsumed", ¬ifyConsumed)) { 1405 notifyConsumed->post(); 1406 } 1407 1408 return true; 1409} 1410 1411void NuPlayer::Renderer::onAudioSinkChanged() { 1412 if (offloadingAudio()) { 1413 return; 1414 } 1415 CHECK(!mDrainAudioQueuePending); 1416 mNumFramesWritten = 0; 1417 { 1418 Mutex::Autolock autoLock(mLock); 1419 mAnchorNumFramesWritten = -1; 1420 } 1421 uint32_t written; 1422 if (mAudioSink->getFramesWritten(&written) == OK) { 1423 mNumFramesWritten = written; 1424 } 1425} 1426 1427void NuPlayer::Renderer::onDisableOffloadAudio() { 1428 Mutex::Autolock autoLock(mLock); 1429 mFlags &= ~FLAG_OFFLOAD_AUDIO; 1430 ++mAudioDrainGeneration; 1431 if (mAudioRenderingStartGeneration != -1) { 1432 prepareForMediaRenderingStart_l(); 1433 } 1434} 1435 1436void NuPlayer::Renderer::onEnableOffloadAudio() { 1437 Mutex::Autolock autoLock(mLock); 1438 mFlags |= FLAG_OFFLOAD_AUDIO; 1439 ++mAudioDrainGeneration; 1440 if (mAudioRenderingStartGeneration != -1) { 1441 prepareForMediaRenderingStart_l(); 1442 } 1443} 1444 1445void NuPlayer::Renderer::onPause() { 1446 if (mPaused) { 1447 return; 1448 } 1449 1450 { 1451 Mutex::Autolock autoLock(mLock); 1452 // we do not increment audio drain generation so that we fill audio buffer during pause. 1453 ++mVideoDrainGeneration; 1454 prepareForMediaRenderingStart_l(); 1455 mPaused = true; 1456 mMediaClock->setPlaybackRate(0.0); 1457 } 1458 1459 mDrainAudioQueuePending = false; 1460 mDrainVideoQueuePending = false; 1461 1462 if (mHasAudio) { 1463 mAudioSink->pause(); 1464 startAudioOffloadPauseTimeout(); 1465 } 1466 1467 ALOGV("now paused audio queue has %zu entries, video has %zu entries", 1468 mAudioQueue.size(), mVideoQueue.size()); 1469} 1470 1471void NuPlayer::Renderer::onResume() { 1472 if (!mPaused) { 1473 return; 1474 } 1475 1476 if (mHasAudio) { 1477 cancelAudioOffloadPauseTimeout(); 1478 status_t err = mAudioSink->start(); 1479 if (err != OK) { 1480 notifyAudioTearDown(); 1481 } 1482 } 1483 1484 { 1485 Mutex::Autolock autoLock(mLock); 1486 mPaused = false; 1487 1488 // configure audiosink as we did not do it when pausing 1489 if (mAudioSink != NULL && mAudioSink->ready()) { 1490 mAudioSink->setPlaybackRate(mPlaybackSettings); 1491 } 1492 1493 mMediaClock->setPlaybackRate(mPlaybackRate); 1494 1495 if (!mAudioQueue.empty()) { 1496 postDrainAudioQueue_l(); 1497 } 1498 } 1499 1500 if (!mVideoQueue.empty()) { 1501 postDrainVideoQueue(); 1502 } 1503} 1504 1505void NuPlayer::Renderer::onSetVideoFrameRate(float fps) { 1506 if (mVideoScheduler == NULL) { 1507 mVideoScheduler = new VideoFrameScheduler(); 1508 } 1509 mVideoScheduler->init(fps); 1510} 1511 1512int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) { 1513 Mutex::Autolock autoLock(mLock); 1514 return (audio ? mAudioQueueGeneration : mVideoQueueGeneration); 1515} 1516 1517int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) { 1518 Mutex::Autolock autoLock(mLock); 1519 return (audio ? mAudioDrainGeneration : mVideoDrainGeneration); 1520} 1521 1522bool NuPlayer::Renderer::getSyncQueues() { 1523 Mutex::Autolock autoLock(mLock); 1524 return mSyncQueues; 1525} 1526 1527// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs() 1528// as it acquires locks and may query the audio driver. 1529// 1530// Some calls could conceivably retrieve extrapolated data instead of 1531// accessing getTimestamp() or getPosition() every time a data buffer with 1532// a media time is received. 1533// 1534// Calculate duration of played samples if played at normal rate (i.e., 1.0). 1535int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) { 1536 uint32_t numFramesPlayed; 1537 int64_t numFramesPlayedAt; 1538 AudioTimestamp ts; 1539 static const int64_t kStaleTimestamp100ms = 100000; 1540 1541 status_t res = mAudioSink->getTimestamp(ts); 1542 if (res == OK) { // case 1: mixing audio tracks and offloaded tracks. 1543 numFramesPlayed = ts.mPosition; 1544 numFramesPlayedAt = 1545 ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000; 1546 const int64_t timestampAge = nowUs - numFramesPlayedAt; 1547 if (timestampAge > kStaleTimestamp100ms) { 1548 // This is an audio FIXME. 1549 // getTimestamp returns a timestamp which may come from audio mixing threads. 1550 // After pausing, the MixerThread may go idle, thus the mTime estimate may 1551 // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms, 1552 // the max latency should be about 25ms with an average around 12ms (to be verified). 1553 // For safety we use 100ms. 1554 ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)", 1555 (long long)nowUs, (long long)numFramesPlayedAt); 1556 numFramesPlayedAt = nowUs - kStaleTimestamp100ms; 1557 } 1558 //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt); 1559 } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track 1560 numFramesPlayed = 0; 1561 numFramesPlayedAt = nowUs; 1562 //ALOGD("getTimestamp: WOULD_BLOCK %d %lld", 1563 // numFramesPlayed, (long long)numFramesPlayedAt); 1564 } else { // case 3: transitory at new track or audio fast tracks. 1565 res = mAudioSink->getPosition(&numFramesPlayed); 1566 CHECK_EQ(res, (status_t)OK); 1567 numFramesPlayedAt = nowUs; 1568 numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */ 1569 //ALOGD("getPosition: %u %lld", numFramesPlayed, (long long)numFramesPlayedAt); 1570 } 1571 1572 //CHECK_EQ(numFramesPlayed & (1 << 31), 0); // can't be negative until 12.4 hrs, test 1573 int64_t durationUs = getDurationUsIfPlayedAtSampleRate(numFramesPlayed) 1574 + nowUs - numFramesPlayedAt; 1575 if (durationUs < 0) { 1576 // Occurs when numFramesPlayed position is very small and the following: 1577 // (1) In case 1, the time nowUs is computed before getTimestamp() is called and 1578 // numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed. 1579 // (2) In case 3, using getPosition and adding mAudioSink->latency() to 1580 // numFramesPlayedAt, by a time amount greater than numFramesPlayed. 1581 // 1582 // Both of these are transitory conditions. 1583 ALOGV("getPlayedOutAudioDurationUs: negative duration %lld set to zero", (long long)durationUs); 1584 durationUs = 0; 1585 } 1586 ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)", 1587 (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt); 1588 return durationUs; 1589} 1590 1591void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) { 1592 if (mAudioTornDown) { 1593 return; 1594 } 1595 mAudioTornDown = true; 1596 1597 int64_t currentPositionUs; 1598 sp<AMessage> notify = mNotify->dup(); 1599 if (getCurrentPosition(¤tPositionUs) == OK) { 1600 notify->setInt64("positionUs", currentPositionUs); 1601 } 1602 1603 mAudioSink->stop(); 1604 mAudioSink->flush(); 1605 1606 notify->setInt32("what", kWhatAudioTearDown); 1607 notify->setInt32("reason", reason); 1608 notify->post(); 1609} 1610 1611void NuPlayer::Renderer::startAudioOffloadPauseTimeout() { 1612 if (offloadingAudio()) { 1613 mWakeLock->acquire(); 1614 sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this); 1615 msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration); 1616 msg->post(kOffloadPauseMaxUs); 1617 } 1618} 1619 1620void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() { 1621 if (offloadingAudio()) { 1622 mWakeLock->release(true); 1623 ++mAudioOffloadPauseTimeoutGeneration; 1624 } 1625} 1626 1627status_t NuPlayer::Renderer::onOpenAudioSink( 1628 const sp<AMessage> &format, 1629 bool offloadOnly, 1630 bool hasVideo, 1631 uint32_t flags) { 1632 ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)", 1633 offloadOnly, offloadingAudio()); 1634 bool audioSinkChanged = false; 1635 1636 int32_t numChannels; 1637 CHECK(format->findInt32("channel-count", &numChannels)); 1638 1639 int32_t channelMask; 1640 if (!format->findInt32("channel-mask", &channelMask)) { 1641 // signal to the AudioSink to derive the mask from count. 1642 channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER; 1643 } 1644 1645 int32_t sampleRate; 1646 CHECK(format->findInt32("sample-rate", &sampleRate)); 1647 1648 if (offloadingAudio()) { 1649 audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT; 1650 AString mime; 1651 CHECK(format->findString("mime", &mime)); 1652 status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str()); 1653 1654 if (err != OK) { 1655 ALOGE("Couldn't map mime \"%s\" to a valid " 1656 "audio_format", mime.c_str()); 1657 onDisableOffloadAudio(); 1658 } else { 1659 ALOGV("Mime \"%s\" mapped to audio_format 0x%x", 1660 mime.c_str(), audioFormat); 1661 1662 int avgBitRate = -1; 1663 format->findInt32("bit-rate", &avgBitRate); 1664 1665 int32_t aacProfile = -1; 1666 if (audioFormat == AUDIO_FORMAT_AAC 1667 && format->findInt32("aac-profile", &aacProfile)) { 1668 // Redefine AAC format as per aac profile 1669 mapAACProfileToAudioFormat( 1670 audioFormat, 1671 aacProfile); 1672 } 1673 1674 audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER; 1675 offloadInfo.duration_us = -1; 1676 format->findInt64( 1677 "durationUs", &offloadInfo.duration_us); 1678 offloadInfo.sample_rate = sampleRate; 1679 offloadInfo.channel_mask = channelMask; 1680 offloadInfo.format = audioFormat; 1681 offloadInfo.stream_type = AUDIO_STREAM_MUSIC; 1682 offloadInfo.bit_rate = avgBitRate; 1683 offloadInfo.has_video = hasVideo; 1684 offloadInfo.is_streaming = true; 1685 1686 if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) { 1687 ALOGV("openAudioSink: no change in offload mode"); 1688 // no change from previous configuration, everything ok. 1689 return OK; 1690 } 1691 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER; 1692 1693 ALOGV("openAudioSink: try to open AudioSink in offload mode"); 1694 uint32_t offloadFlags = flags; 1695 offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; 1696 offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER; 1697 audioSinkChanged = true; 1698 mAudioSink->close(); 1699 1700 err = mAudioSink->open( 1701 sampleRate, 1702 numChannels, 1703 (audio_channel_mask_t)channelMask, 1704 audioFormat, 1705 0 /* bufferCount - unused */, 1706 &NuPlayer::Renderer::AudioSinkCallback, 1707 this, 1708 (audio_output_flags_t)offloadFlags, 1709 &offloadInfo); 1710 1711 if (err == OK) { 1712 err = mAudioSink->setPlaybackRate(mPlaybackSettings); 1713 } 1714 1715 if (err == OK) { 1716 // If the playback is offloaded to h/w, we pass 1717 // the HAL some metadata information. 1718 // We don't want to do this for PCM because it 1719 // will be going through the AudioFlinger mixer 1720 // before reaching the hardware. 1721 // TODO 1722 mCurrentOffloadInfo = offloadInfo; 1723 if (!mPaused) { // for preview mode, don't start if paused 1724 err = mAudioSink->start(); 1725 } 1726 ALOGV_IF(err == OK, "openAudioSink: offload succeeded"); 1727 } 1728 if (err != OK) { 1729 // Clean up, fall back to non offload mode. 1730 mAudioSink->close(); 1731 onDisableOffloadAudio(); 1732 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER; 1733 ALOGV("openAudioSink: offload failed"); 1734 } else { 1735 mUseAudioCallback = true; // offload mode transfers data through callback 1736 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message. 1737 } 1738 } 1739 } 1740 if (!offloadOnly && !offloadingAudio()) { 1741 ALOGV("openAudioSink: open AudioSink in NON-offload mode"); 1742 uint32_t pcmFlags = flags; 1743 pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; 1744 1745 const PcmInfo info = { 1746 (audio_channel_mask_t)channelMask, 1747 (audio_output_flags_t)pcmFlags, 1748 AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat 1749 numChannels, 1750 sampleRate 1751 }; 1752 if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) { 1753 ALOGV("openAudioSink: no change in pcm mode"); 1754 // no change from previous configuration, everything ok. 1755 return OK; 1756 } 1757 1758 audioSinkChanged = true; 1759 mAudioSink->close(); 1760 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER; 1761 // Note: It is possible to set up the callback, but not use it to send audio data. 1762 // This requires a fix in AudioSink to explicitly specify the transfer mode. 1763 mUseAudioCallback = getUseAudioCallbackSetting(); 1764 if (mUseAudioCallback) { 1765 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message. 1766 } 1767 1768 // Compute the desired buffer size. 1769 // For callback mode, the amount of time before wakeup is about half the buffer size. 1770 const uint32_t frameCount = 1771 (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000; 1772 1773 // The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct 1774 // AudioSink. We don't want this when there's video because it will cause a video seek to 1775 // the previous I frame. But we do want this when there's only audio because it will give 1776 // NuPlayer a chance to switch from non-offload mode to offload mode. 1777 // So we only set doNotReconnect when there's no video. 1778 const bool doNotReconnect = !hasVideo; 1779 status_t err = mAudioSink->open( 1780 sampleRate, 1781 numChannels, 1782 (audio_channel_mask_t)channelMask, 1783 AUDIO_FORMAT_PCM_16_BIT, 1784 0 /* bufferCount - unused */, 1785 mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL, 1786 mUseAudioCallback ? this : NULL, 1787 (audio_output_flags_t)pcmFlags, 1788 NULL, 1789 doNotReconnect, 1790 frameCount); 1791 if (err == OK) { 1792 err = mAudioSink->setPlaybackRate(mPlaybackSettings); 1793 } 1794 if (err != OK) { 1795 ALOGW("openAudioSink: non offloaded open failed status: %d", err); 1796 mAudioSink->close(); 1797 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER; 1798 return err; 1799 } 1800 mCurrentPcmInfo = info; 1801 if (!mPaused) { // for preview mode, don't start if paused 1802 mAudioSink->start(); 1803 } 1804 } 1805 if (audioSinkChanged) { 1806 onAudioSinkChanged(); 1807 } 1808 mAudioTornDown = false; 1809 return OK; 1810} 1811 1812void NuPlayer::Renderer::onCloseAudioSink() { 1813 mAudioSink->close(); 1814 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER; 1815 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER; 1816} 1817 1818} // namespace android 1819 1820