NuPlayerRenderer.cpp revision 2abde2c118a94f843a7450818c925d3f0b673cd3
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "NuPlayerRenderer" 19#include <utils/Log.h> 20 21#include "NuPlayerRenderer.h" 22 23#include <media/stagefright/foundation/ABuffer.h> 24#include <media/stagefright/foundation/ADebug.h> 25#include <media/stagefright/foundation/AMessage.h> 26#include <media/stagefright/MediaErrors.h> 27#include <media/stagefright/MetaData.h> 28 29#include <VideoFrameScheduler.h> 30 31#include <inttypes.h> 32 33namespace android { 34 35// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink 36// is closed to allow the audio DSP to power down. 37static const int64_t kOffloadPauseMaxUs = 60000000ll; 38 39// static 40const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; 41 42NuPlayer::Renderer::Renderer( 43 const sp<MediaPlayerBase::AudioSink> &sink, 44 const sp<AMessage> ¬ify, 45 uint32_t flags) 46 : mAudioSink(sink), 47 mNotify(notify), 48 mFlags(flags), 49 mNumFramesWritten(0), 50 mDrainAudioQueuePending(false), 51 mDrainVideoQueuePending(false), 52 mAudioQueueGeneration(0), 53 mVideoQueueGeneration(0), 54 mFirstAnchorTimeMediaUs(-1), 55 mAnchorTimeMediaUs(-1), 56 mAnchorTimeRealUs(-1), 57 mFlushingAudio(false), 58 mFlushingVideo(false), 59 mHasAudio(false), 60 mHasVideo(false), 61 mSyncQueues(false), 62 mPaused(false), 63 mVideoSampleReceived(false), 64 mVideoRenderingStarted(false), 65 mVideoRenderingStartGeneration(0), 66 mAudioRenderingStartGeneration(0), 67 mLastPositionUpdateUs(-1ll), 68 mVideoLateByUs(0ll), 69 mAudioOffloadPauseTimeoutGeneration(0), 70 mAudioOffloadTornDown(false) { 71} 72 73NuPlayer::Renderer::~Renderer() { 74 if (offloadingAudio()) { 75 mAudioSink->stop(); 76 mAudioSink->flush(); 77 mAudioSink->close(); 78 } 79} 80 81void NuPlayer::Renderer::queueBuffer( 82 bool audio, 83 const sp<ABuffer> &buffer, 84 const sp<AMessage> ¬ifyConsumed) { 85 sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id()); 86 msg->setInt32("audio", static_cast<int32_t>(audio)); 87 msg->setBuffer("buffer", buffer); 88 msg->setMessage("notifyConsumed", notifyConsumed); 89 msg->post(); 90} 91 92void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) { 93 CHECK_NE(finalResult, (status_t)OK); 94 95 sp<AMessage> msg = new AMessage(kWhatQueueEOS, id()); 96 msg->setInt32("audio", static_cast<int32_t>(audio)); 97 msg->setInt32("finalResult", finalResult); 98 msg->post(); 99} 100 101void NuPlayer::Renderer::flush(bool audio) { 102 { 103 Mutex::Autolock autoLock(mFlushLock); 104 if (audio) { 105 if (mFlushingAudio) { 106 return; 107 } 108 mFlushingAudio = true; 109 } else { 110 if (mFlushingVideo) { 111 return; 112 } 113 mFlushingVideo = true; 114 } 115 } 116 117 sp<AMessage> msg = new AMessage(kWhatFlush, id()); 118 msg->setInt32("audio", static_cast<int32_t>(audio)); 119 msg->post(); 120} 121 122void NuPlayer::Renderer::signalTimeDiscontinuity() { 123 Mutex::Autolock autoLock(mLock); 124 // CHECK(mAudioQueue.empty()); 125 // CHECK(mVideoQueue.empty()); 126 mFirstAnchorTimeMediaUs = -1; 127 mAnchorTimeMediaUs = -1; 128 mAnchorTimeRealUs = -1; 129 mSyncQueues = false; 130} 131 132void NuPlayer::Renderer::signalAudioSinkChanged() { 133 (new AMessage(kWhatAudioSinkChanged, id()))->post(); 134} 135 136void NuPlayer::Renderer::signalDisableOffloadAudio() { 137 (new AMessage(kWhatDisableOffloadAudio, id()))->post(); 138} 139 140void NuPlayer::Renderer::pause() { 141 (new AMessage(kWhatPause, id()))->post(); 142} 143 144void NuPlayer::Renderer::resume() { 145 (new AMessage(kWhatResume, id()))->post(); 146} 147 148void NuPlayer::Renderer::setVideoFrameRate(float fps) { 149 sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, id()); 150 msg->setFloat("frame-rate", fps); 151 msg->post(); 152} 153 154void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { 155 switch (msg->what()) { 156 case kWhatStopAudioSink: 157 { 158 mAudioSink->stop(); 159 break; 160 } 161 162 case kWhatDrainAudioQueue: 163 { 164 int32_t generation; 165 CHECK(msg->findInt32("generation", &generation)); 166 if (generation != mAudioQueueGeneration) { 167 break; 168 } 169 170 mDrainAudioQueuePending = false; 171 172 if (onDrainAudioQueue()) { 173 uint32_t numFramesPlayed; 174 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), 175 (status_t)OK); 176 177 uint32_t numFramesPendingPlayout = 178 mNumFramesWritten - numFramesPlayed; 179 180 // This is how long the audio sink will have data to 181 // play back. 182 int64_t delayUs = 183 mAudioSink->msecsPerFrame() 184 * numFramesPendingPlayout * 1000ll; 185 186 // Let's give it more data after about half that time 187 // has elapsed. 188 // kWhatDrainAudioQueue is used for non-offloading mode, 189 // and mLock is used only for offloading mode. Therefore, 190 // no need to acquire mLock here. 191 postDrainAudioQueue_l(delayUs / 2); 192 } 193 break; 194 } 195 196 case kWhatDrainVideoQueue: 197 { 198 int32_t generation; 199 CHECK(msg->findInt32("generation", &generation)); 200 if (generation != mVideoQueueGeneration) { 201 break; 202 } 203 204 mDrainVideoQueuePending = false; 205 206 onDrainVideoQueue(); 207 208 postDrainVideoQueue(); 209 break; 210 } 211 212 case kWhatQueueBuffer: 213 { 214 onQueueBuffer(msg); 215 break; 216 } 217 218 case kWhatQueueEOS: 219 { 220 onQueueEOS(msg); 221 break; 222 } 223 224 case kWhatFlush: 225 { 226 onFlush(msg); 227 break; 228 } 229 230 case kWhatAudioSinkChanged: 231 { 232 onAudioSinkChanged(); 233 break; 234 } 235 236 case kWhatDisableOffloadAudio: 237 { 238 onDisableOffloadAudio(); 239 break; 240 } 241 242 case kWhatPause: 243 { 244 onPause(); 245 break; 246 } 247 248 case kWhatResume: 249 { 250 onResume(); 251 break; 252 } 253 254 case kWhatSetVideoFrameRate: 255 { 256 float fps; 257 CHECK(msg->findFloat("frame-rate", &fps)); 258 onSetVideoFrameRate(fps); 259 break; 260 } 261 262 case kWhatAudioOffloadTearDown: 263 { 264 onAudioOffloadTearDown(); 265 break; 266 } 267 268 case kWhatAudioOffloadPauseTimeout: 269 { 270 int32_t generation; 271 CHECK(msg->findInt32("generation", &generation)); 272 if (generation != mAudioOffloadPauseTimeoutGeneration) { 273 break; 274 } 275 onAudioOffloadTearDown(); 276 break; 277 } 278 279 default: 280 TRESPASS(); 281 break; 282 } 283} 284 285void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) { 286 if (mDrainAudioQueuePending || mSyncQueues || mPaused 287 || offloadingAudio()) { 288 return; 289 } 290 291 if (mAudioQueue.empty()) { 292 return; 293 } 294 295 mDrainAudioQueuePending = true; 296 sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id()); 297 msg->setInt32("generation", mAudioQueueGeneration); 298 msg->post(delayUs); 299} 300 301void NuPlayer::Renderer::prepareForMediaRenderingStart() { 302 mAudioRenderingStartGeneration = mAudioQueueGeneration; 303 mVideoRenderingStartGeneration = mVideoQueueGeneration; 304} 305 306void NuPlayer::Renderer::notifyIfMediaRenderingStarted() { 307 if (mVideoRenderingStartGeneration == mVideoQueueGeneration && 308 mAudioRenderingStartGeneration == mAudioQueueGeneration) { 309 mVideoRenderingStartGeneration = -1; 310 mAudioRenderingStartGeneration = -1; 311 312 sp<AMessage> notify = mNotify->dup(); 313 notify->setInt32("what", kWhatMediaRenderingStart); 314 notify->post(); 315 } 316} 317 318// static 319size_t NuPlayer::Renderer::AudioSinkCallback( 320 MediaPlayerBase::AudioSink * /* audioSink */, 321 void *buffer, 322 size_t size, 323 void *cookie, 324 MediaPlayerBase::AudioSink::cb_event_t event) { 325 NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie; 326 327 switch (event) { 328 case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER: 329 { 330 return me->fillAudioBuffer(buffer, size); 331 break; 332 } 333 334 case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END: 335 { 336 me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM); 337 break; 338 } 339 340 case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN: 341 { 342 me->notifyAudioOffloadTearDown(); 343 break; 344 } 345 } 346 347 return 0; 348} 349 350size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { 351 Mutex::Autolock autoLock(mLock); 352 353 if (!offloadingAudio() || mPaused) { 354 return 0; 355 } 356 357 bool hasEOS = false; 358 359 size_t sizeCopied = 0; 360 bool firstEntry = true; 361 while (sizeCopied < size && !mAudioQueue.empty()) { 362 QueueEntry *entry = &*mAudioQueue.begin(); 363 364 if (entry->mBuffer == NULL) { // EOS 365 hasEOS = true; 366 mAudioQueue.erase(mAudioQueue.begin()); 367 entry = NULL; 368 break; 369 } 370 371 if (firstEntry && entry->mOffset == 0) { 372 firstEntry = false; 373 int64_t mediaTimeUs; 374 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 375 ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); 376 if (mFirstAnchorTimeMediaUs == -1) { 377 mFirstAnchorTimeMediaUs = mediaTimeUs; 378 } 379 380 int64_t nowUs = ALooper::GetNowUs(); 381 mAnchorTimeMediaUs = 382 mFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs); 383 mAnchorTimeRealUs = nowUs; 384 385 notifyPosition(); 386 } 387 388 size_t copy = entry->mBuffer->size() - entry->mOffset; 389 size_t sizeRemaining = size - sizeCopied; 390 if (copy > sizeRemaining) { 391 copy = sizeRemaining; 392 } 393 394 memcpy((char *)buffer + sizeCopied, 395 entry->mBuffer->data() + entry->mOffset, 396 copy); 397 398 entry->mOffset += copy; 399 if (entry->mOffset == entry->mBuffer->size()) { 400 entry->mNotifyConsumed->post(); 401 mAudioQueue.erase(mAudioQueue.begin()); 402 entry = NULL; 403 } 404 sizeCopied += copy; 405 notifyIfMediaRenderingStarted(); 406 } 407 408 if (hasEOS) { 409 (new AMessage(kWhatStopAudioSink, id()))->post(); 410 } 411 412 return sizeCopied; 413} 414 415bool NuPlayer::Renderer::onDrainAudioQueue() { 416 uint32_t numFramesPlayed; 417 if (mAudioSink->getPosition(&numFramesPlayed) != OK) { 418 return false; 419 } 420 421 ssize_t numFramesAvailableToWrite = 422 mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed); 423 424#if 0 425 if (numFramesAvailableToWrite == mAudioSink->frameCount()) { 426 ALOGI("audio sink underrun"); 427 } else { 428 ALOGV("audio queue has %d frames left to play", 429 mAudioSink->frameCount() - numFramesAvailableToWrite); 430 } 431#endif 432 433 size_t numBytesAvailableToWrite = 434 numFramesAvailableToWrite * mAudioSink->frameSize(); 435 436 while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) { 437 QueueEntry *entry = &*mAudioQueue.begin(); 438 439 if (entry->mBuffer == NULL) { 440 // EOS 441 int64_t postEOSDelayUs = 0; 442 if (mAudioSink->needsTrailingPadding()) { 443 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs()); 444 } 445 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs); 446 447 mAudioQueue.erase(mAudioQueue.begin()); 448 entry = NULL; 449 return false; 450 } 451 452 if (entry->mOffset == 0) { 453 int64_t mediaTimeUs; 454 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 455 ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); 456 if (mFirstAnchorTimeMediaUs == -1) { 457 mFirstAnchorTimeMediaUs = mediaTimeUs; 458 } 459 mAnchorTimeMediaUs = mediaTimeUs; 460 461 int64_t nowUs = ALooper::GetNowUs(); 462 mAnchorTimeRealUs = nowUs + getPendingAudioPlayoutDurationUs(nowUs); 463 464 notifyPosition(); 465 } 466 467 size_t copy = entry->mBuffer->size() - entry->mOffset; 468 if (copy > numBytesAvailableToWrite) { 469 copy = numBytesAvailableToWrite; 470 } 471 472 ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, copy); 473 if (written < 0) { 474 // An error in AudioSink write is fatal here. 475 LOG_ALWAYS_FATAL("AudioSink write error(%zd) when writing %zu bytes", written, copy); 476 } 477 478 entry->mOffset += written; 479 if (entry->mOffset == entry->mBuffer->size()) { 480 entry->mNotifyConsumed->post(); 481 mAudioQueue.erase(mAudioQueue.begin()); 482 483 entry = NULL; 484 } 485 486 numBytesAvailableToWrite -= written; 487 size_t copiedFrames = written / mAudioSink->frameSize(); 488 mNumFramesWritten += copiedFrames; 489 490 notifyIfMediaRenderingStarted(); 491 492 if (written != (ssize_t)copy) { 493 // A short count was received from AudioSink::write() 494 // 495 // AudioSink write should block until exactly the number of bytes are delivered. 496 // But it may return with a short count (without an error) when: 497 // 498 // 1) Size to be copied is not a multiple of the frame size. We consider this fatal. 499 // 2) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded. 500 501 // (Case 1) 502 // Must be a multiple of the frame size. If it is not a multiple of a frame size, it 503 // needs to fail, as we should not carry over fractional frames between calls. 504 CHECK_EQ(copy % mAudioSink->frameSize(), 0); 505 506 // (Case 2) 507 // Return early to the caller. 508 // Beware of calling immediately again as this may busy-loop if you are not careful. 509 ALOGW("AudioSink write short frame count %zd < %zu", written, copy); 510 break; 511 } 512 } 513 return !mAudioQueue.empty(); 514} 515 516int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) { 517 int64_t writtenAudioDurationUs = 518 mNumFramesWritten * 1000LL * mAudioSink->msecsPerFrame(); 519 return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs); 520} 521 522void NuPlayer::Renderer::postDrainVideoQueue() { 523 if (mDrainVideoQueuePending 524 || mSyncQueues 525 || (mPaused && mVideoSampleReceived)) { 526 return; 527 } 528 529 if (mVideoQueue.empty()) { 530 return; 531 } 532 533 QueueEntry &entry = *mVideoQueue.begin(); 534 535 sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id()); 536 msg->setInt32("generation", mVideoQueueGeneration); 537 538 if (entry.mBuffer == NULL) { 539 // EOS doesn't carry a timestamp. 540 msg->post(); 541 mDrainVideoQueuePending = true; 542 return; 543 } 544 545 int64_t delayUs; 546 int64_t nowUs = ALooper::GetNowUs(); 547 int64_t realTimeUs; 548 if (mFlags & FLAG_REAL_TIME) { 549 int64_t mediaTimeUs; 550 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 551 realTimeUs = mediaTimeUs; 552 } else { 553 int64_t mediaTimeUs; 554 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 555 556 if (mFirstAnchorTimeMediaUs == -1 && !mHasAudio) { 557 mFirstAnchorTimeMediaUs = mediaTimeUs; 558 } 559 if (mAnchorTimeMediaUs < 0) { 560 if (!mHasAudio) { 561 mAnchorTimeMediaUs = mediaTimeUs; 562 mAnchorTimeRealUs = nowUs; 563 notifyPosition(); 564 } 565 realTimeUs = nowUs; 566 } else { 567 realTimeUs = 568 (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs; 569 } 570 } 571 572 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000; 573 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000); 574 575 delayUs = realTimeUs - nowUs; 576 577 ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs); 578 // post 2 display refreshes before rendering is due 579 msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0); 580 581 mDrainVideoQueuePending = true; 582} 583 584void NuPlayer::Renderer::onDrainVideoQueue() { 585 if (mVideoQueue.empty()) { 586 return; 587 } 588 589 QueueEntry *entry = &*mVideoQueue.begin(); 590 591 if (entry->mBuffer == NULL) { 592 // EOS 593 594 notifyEOS(false /* audio */, entry->mFinalResult); 595 596 mVideoQueue.erase(mVideoQueue.begin()); 597 entry = NULL; 598 599 mVideoLateByUs = 0ll; 600 return; 601 } 602 603 int64_t realTimeUs; 604 if (mFlags & FLAG_REAL_TIME) { 605 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs)); 606 } else { 607 int64_t mediaTimeUs; 608 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 609 610 realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs; 611 } 612 613 bool tooLate = false; 614 615 if (!mPaused) { 616 mVideoLateByUs = ALooper::GetNowUs() - realTimeUs; 617 tooLate = (mVideoLateByUs > 40000); 618 619 if (tooLate) { 620 ALOGV("video late by %lld us (%.2f secs)", 621 mVideoLateByUs, mVideoLateByUs / 1E6); 622 } else { 623 ALOGV("rendering video at media time %.2f secs", 624 (mFlags & FLAG_REAL_TIME ? realTimeUs : 625 (realTimeUs + mAnchorTimeMediaUs - mAnchorTimeRealUs)) / 1E6); 626 } 627 } else { 628 mVideoLateByUs = 0ll; 629 } 630 631 entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll); 632 entry->mNotifyConsumed->setInt32("render", !tooLate); 633 entry->mNotifyConsumed->post(); 634 mVideoQueue.erase(mVideoQueue.begin()); 635 entry = NULL; 636 637 mVideoSampleReceived = true; 638 639 if (!mPaused) { 640 if (!mVideoRenderingStarted) { 641 mVideoRenderingStarted = true; 642 notifyVideoRenderingStart(); 643 } 644 notifyIfMediaRenderingStarted(); 645 } 646} 647 648void NuPlayer::Renderer::notifyVideoRenderingStart() { 649 sp<AMessage> notify = mNotify->dup(); 650 notify->setInt32("what", kWhatVideoRenderingStart); 651 notify->post(); 652} 653 654void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) { 655 sp<AMessage> notify = mNotify->dup(); 656 notify->setInt32("what", kWhatEOS); 657 notify->setInt32("audio", static_cast<int32_t>(audio)); 658 notify->setInt32("finalResult", finalResult); 659 notify->post(delayUs); 660} 661 662void NuPlayer::Renderer::notifyAudioOffloadTearDown() { 663 (new AMessage(kWhatAudioOffloadTearDown, id()))->post(); 664} 665 666void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { 667 int32_t audio; 668 CHECK(msg->findInt32("audio", &audio)); 669 670 if (audio) { 671 mHasAudio = true; 672 } else { 673 mHasVideo = true; 674 if (mVideoScheduler == NULL) { 675 mVideoScheduler = new VideoFrameScheduler(); 676 mVideoScheduler->init(); 677 } 678 } 679 680 if (dropBufferWhileFlushing(audio, msg)) { 681 return; 682 } 683 684 sp<ABuffer> buffer; 685 CHECK(msg->findBuffer("buffer", &buffer)); 686 687 sp<AMessage> notifyConsumed; 688 CHECK(msg->findMessage("notifyConsumed", ¬ifyConsumed)); 689 690 QueueEntry entry; 691 entry.mBuffer = buffer; 692 entry.mNotifyConsumed = notifyConsumed; 693 entry.mOffset = 0; 694 entry.mFinalResult = OK; 695 696 if (audio) { 697 Mutex::Autolock autoLock(mLock); 698 mAudioQueue.push_back(entry); 699 postDrainAudioQueue_l(); 700 } else { 701 mVideoQueue.push_back(entry); 702 postDrainVideoQueue(); 703 } 704 705 Mutex::Autolock autoLock(mLock); 706 if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) { 707 return; 708 } 709 710 sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer; 711 sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer; 712 713 if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) { 714 // EOS signalled on either queue. 715 syncQueuesDone_l(); 716 return; 717 } 718 719 int64_t firstAudioTimeUs; 720 int64_t firstVideoTimeUs; 721 CHECK(firstAudioBuffer->meta() 722 ->findInt64("timeUs", &firstAudioTimeUs)); 723 CHECK(firstVideoBuffer->meta() 724 ->findInt64("timeUs", &firstVideoTimeUs)); 725 726 int64_t diff = firstVideoTimeUs - firstAudioTimeUs; 727 728 ALOGV("queueDiff = %.2f secs", diff / 1E6); 729 730 if (diff > 100000ll) { 731 // Audio data starts More than 0.1 secs before video. 732 // Drop some audio. 733 734 (*mAudioQueue.begin()).mNotifyConsumed->post(); 735 mAudioQueue.erase(mAudioQueue.begin()); 736 return; 737 } 738 739 syncQueuesDone_l(); 740} 741 742void NuPlayer::Renderer::syncQueuesDone_l() { 743 if (!mSyncQueues) { 744 return; 745 } 746 747 mSyncQueues = false; 748 749 if (!mAudioQueue.empty()) { 750 postDrainAudioQueue_l(); 751 } 752 753 if (!mVideoQueue.empty()) { 754 postDrainVideoQueue(); 755 } 756} 757 758void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) { 759 int32_t audio; 760 CHECK(msg->findInt32("audio", &audio)); 761 762 if (dropBufferWhileFlushing(audio, msg)) { 763 return; 764 } 765 766 int32_t finalResult; 767 CHECK(msg->findInt32("finalResult", &finalResult)); 768 769 QueueEntry entry; 770 entry.mOffset = 0; 771 entry.mFinalResult = finalResult; 772 773 if (audio) { 774 Mutex::Autolock autoLock(mLock); 775 if (mAudioQueue.empty() && mSyncQueues) { 776 syncQueuesDone_l(); 777 } 778 mAudioQueue.push_back(entry); 779 postDrainAudioQueue_l(); 780 } else { 781 if (mVideoQueue.empty() && mSyncQueues) { 782 Mutex::Autolock autoLock(mLock); 783 syncQueuesDone_l(); 784 } 785 mVideoQueue.push_back(entry); 786 postDrainVideoQueue(); 787 } 788} 789 790void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { 791 int32_t audio; 792 CHECK(msg->findInt32("audio", &audio)); 793 794 { 795 Mutex::Autolock autoLock(mFlushLock); 796 if (audio) { 797 mFlushingAudio = false; 798 } else { 799 mFlushingVideo = false; 800 } 801 } 802 803 // If we're currently syncing the queues, i.e. dropping audio while 804 // aligning the first audio/video buffer times and only one of the 805 // two queues has data, we may starve that queue by not requesting 806 // more buffers from the decoder. If the other source then encounters 807 // a discontinuity that leads to flushing, we'll never find the 808 // corresponding discontinuity on the other queue. 809 // Therefore we'll stop syncing the queues if at least one of them 810 // is flushed. 811 { 812 Mutex::Autolock autoLock(mLock); 813 syncQueuesDone_l(); 814 } 815 816 ALOGV("flushing %s", audio ? "audio" : "video"); 817 if (audio) { 818 { 819 Mutex::Autolock autoLock(mLock); 820 flushQueue(&mAudioQueue); 821 822 ++mAudioQueueGeneration; 823 prepareForMediaRenderingStart(); 824 825 if (offloadingAudio()) { 826 mFirstAnchorTimeMediaUs = -1; 827 } 828 } 829 830 mDrainAudioQueuePending = false; 831 832 if (offloadingAudio()) { 833 mAudioSink->pause(); 834 mAudioSink->flush(); 835 mAudioSink->start(); 836 } 837 } else { 838 flushQueue(&mVideoQueue); 839 840 mDrainVideoQueuePending = false; 841 ++mVideoQueueGeneration; 842 843 if (mVideoScheduler != NULL) { 844 mVideoScheduler->restart(); 845 } 846 847 prepareForMediaRenderingStart(); 848 } 849 850 mVideoSampleReceived = false; 851 notifyFlushComplete(audio); 852} 853 854void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) { 855 while (!queue->empty()) { 856 QueueEntry *entry = &*queue->begin(); 857 858 if (entry->mBuffer != NULL) { 859 entry->mNotifyConsumed->post(); 860 } 861 862 queue->erase(queue->begin()); 863 entry = NULL; 864 } 865} 866 867void NuPlayer::Renderer::notifyFlushComplete(bool audio) { 868 sp<AMessage> notify = mNotify->dup(); 869 notify->setInt32("what", kWhatFlushComplete); 870 notify->setInt32("audio", static_cast<int32_t>(audio)); 871 notify->post(); 872} 873 874bool NuPlayer::Renderer::dropBufferWhileFlushing( 875 bool audio, const sp<AMessage> &msg) { 876 bool flushing = false; 877 878 { 879 Mutex::Autolock autoLock(mFlushLock); 880 if (audio) { 881 flushing = mFlushingAudio; 882 } else { 883 flushing = mFlushingVideo; 884 } 885 } 886 887 if (!flushing) { 888 return false; 889 } 890 891 sp<AMessage> notifyConsumed; 892 if (msg->findMessage("notifyConsumed", ¬ifyConsumed)) { 893 notifyConsumed->post(); 894 } 895 896 return true; 897} 898 899void NuPlayer::Renderer::onAudioSinkChanged() { 900 if (offloadingAudio()) { 901 return; 902 } 903 CHECK(!mDrainAudioQueuePending); 904 mNumFramesWritten = 0; 905 uint32_t written; 906 if (mAudioSink->getFramesWritten(&written) == OK) { 907 mNumFramesWritten = written; 908 } 909} 910 911void NuPlayer::Renderer::onDisableOffloadAudio() { 912 Mutex::Autolock autoLock(mLock); 913 mFlags &= ~FLAG_OFFLOAD_AUDIO; 914 ++mAudioQueueGeneration; 915} 916 917void NuPlayer::Renderer::notifyPosition() { 918 // notifyPosition() must be called only after setting mAnchorTimeRealUs 919 // and mAnchorTimeMediaUs, and must not be paused as it extrapolates position. 920 //CHECK_GE(mAnchorTimeRealUs, 0); 921 //CHECK_GE(mAnchorTimeMediaUs, 0); 922 //CHECK(!mPaused || !mHasAudio); // video-only does display in paused mode. 923 924 int64_t nowUs = ALooper::GetNowUs(); 925 926 if (mLastPositionUpdateUs >= 0 927 && nowUs < mLastPositionUpdateUs + kMinPositionUpdateDelayUs) { 928 return; 929 } 930 mLastPositionUpdateUs = nowUs; 931 932 int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs; 933 934 //ALOGD("notifyPosition: positionUs(%lld) nowUs(%lld) mAnchorTimeRealUs(%lld)" 935 // " mAnchorTimeMediaUs(%lld) mFirstAnchorTimeMediaUs(%lld)", 936 // (long long)positionUs, (long long)nowUs, (long long)mAnchorTimeRealUs, 937 // (long long)mAnchorTimeMediaUs, (long long)mFirstAnchorTimeMediaUs); 938 939 // Due to adding the latency to mAnchorTimeRealUs in onDrainAudioQueue(), 940 // positionUs may be less than the first media time. This is avoided 941 // here to prevent potential retrograde motion of the position bar 942 // when starting up after a seek. 943 if (positionUs < mFirstAnchorTimeMediaUs) { 944 positionUs = mFirstAnchorTimeMediaUs; 945 } 946 sp<AMessage> notify = mNotify->dup(); 947 notify->setInt32("what", kWhatPosition); 948 notify->setInt64("positionUs", positionUs); 949 notify->setInt64("videoLateByUs", mVideoLateByUs); 950 notify->post(); 951} 952 953void NuPlayer::Renderer::onPause() { 954 if (mPaused) { 955 ALOGW("Renderer::onPause() called while already paused!"); 956 return; 957 } 958 { 959 Mutex::Autolock autoLock(mLock); 960 ++mAudioQueueGeneration; 961 ++mVideoQueueGeneration; 962 prepareForMediaRenderingStart(); 963 mPaused = true; 964 } 965 966 mDrainAudioQueuePending = false; 967 mDrainVideoQueuePending = false; 968 969 if (mHasAudio) { 970 mAudioSink->pause(); 971 startAudioOffloadPauseTimeout(); 972 } 973 974 ALOGV("now paused audio queue has %d entries, video has %d entries", 975 mAudioQueue.size(), mVideoQueue.size()); 976} 977 978void NuPlayer::Renderer::onResume() { 979 if (!mPaused) { 980 return; 981 } 982 983 if (mHasAudio) { 984 cancelAudioOffloadPauseTimeout(); 985 mAudioSink->start(); 986 } 987 988 Mutex::Autolock autoLock(mLock); 989 mPaused = false; 990 991 if (!mAudioQueue.empty()) { 992 postDrainAudioQueue_l(); 993 } 994 995 if (!mVideoQueue.empty()) { 996 postDrainVideoQueue(); 997 } 998} 999 1000void NuPlayer::Renderer::onSetVideoFrameRate(float fps) { 1001 if (mVideoScheduler == NULL) { 1002 mVideoScheduler = new VideoFrameScheduler(); 1003 } 1004 mVideoScheduler->init(fps); 1005} 1006 1007// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs() 1008// as it acquires locks and may query the audio driver. 1009// 1010// Some calls are not needed since notifyPosition() doesn't always deliver a message. 1011// Some calls could conceivably retrieve extrapolated data instead of 1012// accessing getTimestamp() or getPosition() every time a data buffer with 1013// a media time is received. 1014// 1015int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) { 1016 uint32_t numFramesPlayed; 1017 int64_t numFramesPlayedAt; 1018 AudioTimestamp ts; 1019 static const int64_t kStaleTimestamp100ms = 100000; 1020 1021 status_t res = mAudioSink->getTimestamp(ts); 1022 if (res == OK) { // case 1: mixing audio tracks and offloaded tracks. 1023 numFramesPlayed = ts.mPosition; 1024 numFramesPlayedAt = 1025 ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000; 1026 const int64_t timestampAge = nowUs - numFramesPlayedAt; 1027 if (timestampAge > kStaleTimestamp100ms) { 1028 // This is an audio FIXME. 1029 // getTimestamp returns a timestamp which may come from audio mixing threads. 1030 // After pausing, the MixerThread may go idle, thus the mTime estimate may 1031 // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms, 1032 // the max latency should be about 25ms with an average around 12ms (to be verified). 1033 // For safety we use 100ms. 1034 ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)", 1035 (long long)nowUs, (long long)numFramesPlayedAt); 1036 numFramesPlayedAt = nowUs - kStaleTimestamp100ms; 1037 } 1038 //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt); 1039 } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track 1040 numFramesPlayed = 0; 1041 numFramesPlayedAt = nowUs; 1042 //ALOGD("getTimestamp: WOULD_BLOCK %d %lld", 1043 // numFramesPlayed, (long long)numFramesPlayedAt); 1044 } else { // case 3: transitory at new track or audio fast tracks. 1045 res = mAudioSink->getPosition(&numFramesPlayed); 1046 CHECK_EQ(res, (status_t)OK); 1047 numFramesPlayedAt = nowUs; 1048 numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */ 1049 //ALOGD("getPosition: %d %lld", numFramesPlayed, numFramesPlayedAt); 1050 } 1051 1052 // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours. 1053 //CHECK_EQ(numFramesPlayed & (1 << 31), 0); // can't be negative until 12.4 hrs, test 1054 int64_t durationUs = (int32_t)numFramesPlayed * 1000LL * mAudioSink->msecsPerFrame() 1055 + nowUs - numFramesPlayedAt; 1056 if (durationUs < 0) { 1057 // Occurs when numFramesPlayed position is very small and the following: 1058 // (1) In case 1, the time nowUs is computed before getTimestamp() is called and 1059 // numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed. 1060 // (2) In case 3, using getPosition and adding mAudioSink->latency() to 1061 // numFramesPlayedAt, by a time amount greater than numFramesPlayed. 1062 // 1063 // Both of these are transitory conditions. 1064 ALOGV("getPlayedOutAudioDurationUs: negative duration %lld set to zero", (long long)durationUs); 1065 durationUs = 0; 1066 } 1067 ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)", 1068 (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt); 1069 return durationUs; 1070} 1071 1072void NuPlayer::Renderer::onAudioOffloadTearDown() { 1073 if (mAudioOffloadTornDown) { 1074 return; 1075 } 1076 mAudioOffloadTornDown = true; 1077 1078 int64_t firstAudioTimeUs; 1079 { 1080 Mutex::Autolock autoLock(mLock); 1081 firstAudioTimeUs = mFirstAnchorTimeMediaUs; 1082 } 1083 1084 int64_t currentPositionUs = 1085 firstAudioTimeUs + getPlayedOutAudioDurationUs(ALooper::GetNowUs()); 1086 1087 mAudioSink->stop(); 1088 mAudioSink->flush(); 1089 1090 sp<AMessage> notify = mNotify->dup(); 1091 notify->setInt32("what", kWhatAudioOffloadTearDown); 1092 notify->setInt64("positionUs", currentPositionUs); 1093 notify->post(); 1094} 1095 1096void NuPlayer::Renderer::startAudioOffloadPauseTimeout() { 1097 if (offloadingAudio()) { 1098 sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, id()); 1099 msg->setInt32("generation", mAudioOffloadPauseTimeoutGeneration); 1100 msg->post(kOffloadPauseMaxUs); 1101 } 1102} 1103 1104void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() { 1105 if (offloadingAudio()) { 1106 ++mAudioOffloadPauseTimeoutGeneration; 1107 } 1108} 1109 1110} // namespace android 1111 1112