NuPlayerRenderer.cpp revision c851b5de495169d7e9528644c2592746021bd968
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "NuPlayerRenderer" 19#include <utils/Log.h> 20 21#include "NuPlayerRenderer.h" 22 23#include <media/stagefright/foundation/ABuffer.h> 24#include <media/stagefright/foundation/ADebug.h> 25#include <media/stagefright/foundation/AMessage.h> 26#include <media/stagefright/MediaErrors.h> 27#include <media/stagefright/MetaData.h> 28 29#include <VideoFrameScheduler.h> 30 31#include <inttypes.h> 32 33namespace android { 34 35// static 36const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; 37 38NuPlayer::Renderer::Renderer( 39 const sp<MediaPlayerBase::AudioSink> &sink, 40 const sp<AMessage> ¬ify, 41 uint32_t flags) 42 : mAudioSink(sink), 43 mNotify(notify), 44 mFlags(flags), 45 mNumFramesWritten(0), 46 mDrainAudioQueuePending(false), 47 mDrainVideoQueuePending(false), 48 mAudioQueueGeneration(0), 49 mVideoQueueGeneration(0), 50 mFirstAnchorTimeMediaUs(-1), 51 mAnchorTimeMediaUs(-1), 52 mAnchorTimeRealUs(-1), 53 mFlushingAudio(false), 54 mFlushingVideo(false), 55 mHasAudio(false), 56 mHasVideo(false), 57 mSyncQueues(false), 58 mPaused(false), 59 mVideoSampleReceived(false), 60 mVideoRenderingStarted(false), 61 mVideoRenderingStartGeneration(0), 62 mAudioRenderingStartGeneration(0), 63 mLastPositionUpdateUs(-1ll), 64 mVideoLateByUs(0ll) { 65} 66 67NuPlayer::Renderer::~Renderer() { 68 if (offloadingAudio()) { 69 mAudioSink->stop(); 70 mAudioSink->flush(); 71 mAudioSink->close(); 72 } 73} 74 75void NuPlayer::Renderer::queueBuffer( 76 bool audio, 77 const sp<ABuffer> &buffer, 78 const sp<AMessage> ¬ifyConsumed) { 79 sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id()); 80 msg->setInt32("audio", static_cast<int32_t>(audio)); 81 msg->setBuffer("buffer", buffer); 82 msg->setMessage("notifyConsumed", notifyConsumed); 83 msg->post(); 84} 85 86void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) { 87 CHECK_NE(finalResult, (status_t)OK); 88 89 sp<AMessage> msg = new AMessage(kWhatQueueEOS, id()); 90 msg->setInt32("audio", static_cast<int32_t>(audio)); 91 msg->setInt32("finalResult", finalResult); 92 msg->post(); 93} 94 95void NuPlayer::Renderer::flush(bool audio) { 96 { 97 Mutex::Autolock autoLock(mFlushLock); 98 if (audio) { 99 if (mFlushingAudio) { 100 return; 101 } 102 mFlushingAudio = true; 103 } else { 104 if (mFlushingVideo) { 105 return; 106 } 107 mFlushingVideo = true; 108 } 109 } 110 111 sp<AMessage> msg = new AMessage(kWhatFlush, id()); 112 msg->setInt32("audio", static_cast<int32_t>(audio)); 113 msg->post(); 114} 115 116void NuPlayer::Renderer::signalTimeDiscontinuity() { 117 Mutex::Autolock autoLock(mLock); 118 // CHECK(mAudioQueue.empty()); 119 // CHECK(mVideoQueue.empty()); 120 mFirstAnchorTimeMediaUs = -1; 121 mAnchorTimeMediaUs = -1; 122 mAnchorTimeRealUs = -1; 123 mSyncQueues = false; 124} 125 126void NuPlayer::Renderer::signalAudioSinkChanged() { 127 (new AMessage(kWhatAudioSinkChanged, id()))->post(); 128} 129 130void NuPlayer::Renderer::signalDisableOffloadAudio() { 131 (new AMessage(kWhatDisableOffloadAudio, id()))->post(); 132} 133 134void NuPlayer::Renderer::pause() { 135 (new AMessage(kWhatPause, id()))->post(); 136} 137 138void NuPlayer::Renderer::resume() { 139 (new AMessage(kWhatResume, id()))->post(); 140} 141 142void NuPlayer::Renderer::setVideoFrameRate(float fps) { 143 sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, id()); 144 msg->setFloat("frame-rate", fps); 145 msg->post(); 146} 147 148void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { 149 switch (msg->what()) { 150 case kWhatStopAudioSink: 151 { 152 mAudioSink->stop(); 153 break; 154 } 155 156 case kWhatDrainAudioQueue: 157 { 158 int32_t generation; 159 CHECK(msg->findInt32("generation", &generation)); 160 if (generation != mAudioQueueGeneration) { 161 break; 162 } 163 164 mDrainAudioQueuePending = false; 165 166 if (onDrainAudioQueue()) { 167 uint32_t numFramesPlayed; 168 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), 169 (status_t)OK); 170 171 uint32_t numFramesPendingPlayout = 172 mNumFramesWritten - numFramesPlayed; 173 174 // This is how long the audio sink will have data to 175 // play back. 176 int64_t delayUs = 177 mAudioSink->msecsPerFrame() 178 * numFramesPendingPlayout * 1000ll; 179 180 // Let's give it more data after about half that time 181 // has elapsed. 182 // kWhatDrainAudioQueue is used for non-offloading mode, 183 // and mLock is used only for offloading mode. Therefore, 184 // no need to acquire mLock here. 185 postDrainAudioQueue_l(delayUs / 2); 186 } 187 break; 188 } 189 190 case kWhatDrainVideoQueue: 191 { 192 int32_t generation; 193 CHECK(msg->findInt32("generation", &generation)); 194 if (generation != mVideoQueueGeneration) { 195 break; 196 } 197 198 mDrainVideoQueuePending = false; 199 200 onDrainVideoQueue(); 201 202 postDrainVideoQueue(); 203 break; 204 } 205 206 case kWhatQueueBuffer: 207 { 208 onQueueBuffer(msg); 209 break; 210 } 211 212 case kWhatQueueEOS: 213 { 214 onQueueEOS(msg); 215 break; 216 } 217 218 case kWhatFlush: 219 { 220 onFlush(msg); 221 break; 222 } 223 224 case kWhatAudioSinkChanged: 225 { 226 onAudioSinkChanged(); 227 break; 228 } 229 230 case kWhatDisableOffloadAudio: 231 { 232 onDisableOffloadAudio(); 233 break; 234 } 235 236 case kWhatPause: 237 { 238 onPause(); 239 break; 240 } 241 242 case kWhatResume: 243 { 244 onResume(); 245 break; 246 } 247 248 case kWhatSetVideoFrameRate: 249 { 250 float fps; 251 CHECK(msg->findFloat("frame-rate", &fps)); 252 onSetVideoFrameRate(fps); 253 break; 254 } 255 256 case kWhatAudioOffloadTearDown: 257 { 258 onAudioOffloadTearDown(); 259 break; 260 } 261 262 default: 263 TRESPASS(); 264 break; 265 } 266} 267 268void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) { 269 if (mDrainAudioQueuePending || mSyncQueues || mPaused 270 || offloadingAudio()) { 271 return; 272 } 273 274 if (mAudioQueue.empty()) { 275 return; 276 } 277 278 mDrainAudioQueuePending = true; 279 sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id()); 280 msg->setInt32("generation", mAudioQueueGeneration); 281 msg->post(delayUs); 282} 283 284void NuPlayer::Renderer::prepareForMediaRenderingStart() { 285 mAudioRenderingStartGeneration = mAudioQueueGeneration; 286 mVideoRenderingStartGeneration = mVideoQueueGeneration; 287} 288 289void NuPlayer::Renderer::notifyIfMediaRenderingStarted() { 290 if (mVideoRenderingStartGeneration == mVideoQueueGeneration && 291 mAudioRenderingStartGeneration == mAudioQueueGeneration) { 292 mVideoRenderingStartGeneration = -1; 293 mAudioRenderingStartGeneration = -1; 294 295 sp<AMessage> notify = mNotify->dup(); 296 notify->setInt32("what", kWhatMediaRenderingStart); 297 notify->post(); 298 } 299} 300 301// static 302size_t NuPlayer::Renderer::AudioSinkCallback( 303 MediaPlayerBase::AudioSink * /* audioSink */, 304 void *buffer, 305 size_t size, 306 void *cookie, 307 MediaPlayerBase::AudioSink::cb_event_t event) { 308 NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie; 309 310 switch (event) { 311 case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER: 312 { 313 return me->fillAudioBuffer(buffer, size); 314 break; 315 } 316 317 case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END: 318 { 319 me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM); 320 break; 321 } 322 323 case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN: 324 { 325 me->notifyAudioOffloadTearDown(); 326 break; 327 } 328 } 329 330 return 0; 331} 332 333size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { 334 Mutex::Autolock autoLock(mLock); 335 336 if (!offloadingAudio() || mPaused) { 337 return 0; 338 } 339 340 bool hasEOS = false; 341 342 size_t sizeCopied = 0; 343 bool firstEntry = true; 344 while (sizeCopied < size && !mAudioQueue.empty()) { 345 QueueEntry *entry = &*mAudioQueue.begin(); 346 347 if (entry->mBuffer == NULL) { // EOS 348 hasEOS = true; 349 mAudioQueue.erase(mAudioQueue.begin()); 350 entry = NULL; 351 break; 352 } 353 354 if (firstEntry && entry->mOffset == 0) { 355 firstEntry = false; 356 int64_t mediaTimeUs; 357 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 358 ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); 359 if (mFirstAnchorTimeMediaUs == -1) { 360 mFirstAnchorTimeMediaUs = mediaTimeUs; 361 } 362 363 int64_t nowUs = ALooper::GetNowUs(); 364 mAnchorTimeMediaUs = 365 mFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs); 366 mAnchorTimeRealUs = nowUs; 367 368 notifyPosition(); 369 } 370 371 size_t copy = entry->mBuffer->size() - entry->mOffset; 372 size_t sizeRemaining = size - sizeCopied; 373 if (copy > sizeRemaining) { 374 copy = sizeRemaining; 375 } 376 377 memcpy((char *)buffer + sizeCopied, 378 entry->mBuffer->data() + entry->mOffset, 379 copy); 380 381 entry->mOffset += copy; 382 if (entry->mOffset == entry->mBuffer->size()) { 383 entry->mNotifyConsumed->post(); 384 mAudioQueue.erase(mAudioQueue.begin()); 385 entry = NULL; 386 } 387 sizeCopied += copy; 388 notifyIfMediaRenderingStarted(); 389 } 390 391 if (hasEOS) { 392 (new AMessage(kWhatStopAudioSink, id()))->post(); 393 } 394 395 return sizeCopied; 396} 397 398bool NuPlayer::Renderer::onDrainAudioQueue() { 399 uint32_t numFramesPlayed; 400 if (mAudioSink->getPosition(&numFramesPlayed) != OK) { 401 return false; 402 } 403 404 ssize_t numFramesAvailableToWrite = 405 mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed); 406 407#if 0 408 if (numFramesAvailableToWrite == mAudioSink->frameCount()) { 409 ALOGI("audio sink underrun"); 410 } else { 411 ALOGV("audio queue has %d frames left to play", 412 mAudioSink->frameCount() - numFramesAvailableToWrite); 413 } 414#endif 415 416 size_t numBytesAvailableToWrite = 417 numFramesAvailableToWrite * mAudioSink->frameSize(); 418 419 while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) { 420 QueueEntry *entry = &*mAudioQueue.begin(); 421 422 if (entry->mBuffer == NULL) { 423 // EOS 424 int64_t postEOSDelayUs = 0; 425 if (mAudioSink->needsTrailingPadding()) { 426 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs()); 427 } 428 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs); 429 430 mAudioQueue.erase(mAudioQueue.begin()); 431 entry = NULL; 432 return false; 433 } 434 435 if (entry->mOffset == 0) { 436 int64_t mediaTimeUs; 437 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 438 ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); 439 if (mFirstAnchorTimeMediaUs == -1) { 440 mFirstAnchorTimeMediaUs = mediaTimeUs; 441 } 442 mAnchorTimeMediaUs = mediaTimeUs; 443 444 int64_t nowUs = ALooper::GetNowUs(); 445 mAnchorTimeRealUs = nowUs + getPendingAudioPlayoutDurationUs(nowUs); 446 447 notifyPosition(); 448 } 449 450 size_t copy = entry->mBuffer->size() - entry->mOffset; 451 if (copy > numBytesAvailableToWrite) { 452 copy = numBytesAvailableToWrite; 453 } 454 455 ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, copy); 456 if (written < 0) { 457 // An error in AudioSink write is fatal here. 458 LOG_ALWAYS_FATAL("AudioSink write error(%zd) when writing %zu bytes", written, copy); 459 } 460 461 entry->mOffset += written; 462 if (entry->mOffset == entry->mBuffer->size()) { 463 entry->mNotifyConsumed->post(); 464 mAudioQueue.erase(mAudioQueue.begin()); 465 466 entry = NULL; 467 } 468 469 numBytesAvailableToWrite -= written; 470 size_t copiedFrames = written / mAudioSink->frameSize(); 471 mNumFramesWritten += copiedFrames; 472 473 notifyIfMediaRenderingStarted(); 474 475 if (written != (ssize_t)copy) { 476 // A short count was received from AudioSink::write() 477 // 478 // AudioSink write should block until exactly the number of bytes are delivered. 479 // But it may return with a short count (without an error) when: 480 // 481 // 1) Size to be copied is not a multiple of the frame size. We consider this fatal. 482 // 2) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded. 483 484 // (Case 1) 485 // Must be a multiple of the frame size. If it is not a multiple of a frame size, it 486 // needs to fail, as we should not carry over fractional frames between calls. 487 CHECK_EQ(copy % mAudioSink->frameSize(), 0); 488 489 // (Case 2) 490 // Return early to the caller. 491 // Beware of calling immediately again as this may busy-loop if you are not careful. 492 ALOGW("AudioSink write short frame count %zd < %zu", written, copy); 493 break; 494 } 495 } 496 return !mAudioQueue.empty(); 497} 498 499int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) { 500 int64_t writtenAudioDurationUs = 501 mNumFramesWritten * 1000LL * mAudioSink->msecsPerFrame(); 502 return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs); 503} 504 505void NuPlayer::Renderer::postDrainVideoQueue() { 506 if (mDrainVideoQueuePending 507 || mSyncQueues 508 || (mPaused && mVideoSampleReceived)) { 509 return; 510 } 511 512 if (mVideoQueue.empty()) { 513 return; 514 } 515 516 QueueEntry &entry = *mVideoQueue.begin(); 517 518 sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id()); 519 msg->setInt32("generation", mVideoQueueGeneration); 520 521 if (entry.mBuffer == NULL) { 522 // EOS doesn't carry a timestamp. 523 msg->post(); 524 mDrainVideoQueuePending = true; 525 return; 526 } 527 528 int64_t delayUs; 529 int64_t nowUs = ALooper::GetNowUs(); 530 int64_t realTimeUs; 531 if (mFlags & FLAG_REAL_TIME) { 532 int64_t mediaTimeUs; 533 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 534 realTimeUs = mediaTimeUs; 535 } else { 536 int64_t mediaTimeUs; 537 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 538 539 if (mFirstAnchorTimeMediaUs == -1 && !mHasAudio) { 540 mFirstAnchorTimeMediaUs = mediaTimeUs; 541 } 542 if (mAnchorTimeMediaUs < 0) { 543 if (!mHasAudio) { 544 mAnchorTimeMediaUs = mediaTimeUs; 545 mAnchorTimeRealUs = nowUs; 546 notifyPosition(); 547 } 548 realTimeUs = nowUs; 549 } else { 550 realTimeUs = 551 (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs; 552 } 553 } 554 555 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000; 556 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000); 557 558 delayUs = realTimeUs - nowUs; 559 560 ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs); 561 // post 2 display refreshes before rendering is due 562 msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0); 563 564 mDrainVideoQueuePending = true; 565} 566 567void NuPlayer::Renderer::onDrainVideoQueue() { 568 if (mVideoQueue.empty()) { 569 return; 570 } 571 572 QueueEntry *entry = &*mVideoQueue.begin(); 573 574 if (entry->mBuffer == NULL) { 575 // EOS 576 577 notifyEOS(false /* audio */, entry->mFinalResult); 578 579 mVideoQueue.erase(mVideoQueue.begin()); 580 entry = NULL; 581 582 mVideoLateByUs = 0ll; 583 return; 584 } 585 586 int64_t realTimeUs; 587 if (mFlags & FLAG_REAL_TIME) { 588 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs)); 589 } else { 590 int64_t mediaTimeUs; 591 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 592 593 realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs; 594 } 595 596 bool tooLate = false; 597 598 if (!mPaused) { 599 mVideoLateByUs = ALooper::GetNowUs() - realTimeUs; 600 tooLate = (mVideoLateByUs > 40000); 601 602 if (tooLate) { 603 ALOGV("video late by %lld us (%.2f secs)", 604 mVideoLateByUs, mVideoLateByUs / 1E6); 605 } else { 606 ALOGV("rendering video at media time %.2f secs", 607 (mFlags & FLAG_REAL_TIME ? realTimeUs : 608 (realTimeUs + mAnchorTimeMediaUs - mAnchorTimeRealUs)) / 1E6); 609 } 610 } else { 611 mVideoLateByUs = 0ll; 612 } 613 614 entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll); 615 entry->mNotifyConsumed->setInt32("render", !tooLate); 616 entry->mNotifyConsumed->post(); 617 mVideoQueue.erase(mVideoQueue.begin()); 618 entry = NULL; 619 620 mVideoSampleReceived = true; 621 622 if (!mPaused) { 623 if (!mVideoRenderingStarted) { 624 mVideoRenderingStarted = true; 625 notifyVideoRenderingStart(); 626 } 627 notifyIfMediaRenderingStarted(); 628 } 629} 630 631void NuPlayer::Renderer::notifyVideoRenderingStart() { 632 sp<AMessage> notify = mNotify->dup(); 633 notify->setInt32("what", kWhatVideoRenderingStart); 634 notify->post(); 635} 636 637void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) { 638 sp<AMessage> notify = mNotify->dup(); 639 notify->setInt32("what", kWhatEOS); 640 notify->setInt32("audio", static_cast<int32_t>(audio)); 641 notify->setInt32("finalResult", finalResult); 642 notify->post(delayUs); 643} 644 645void NuPlayer::Renderer::notifyAudioOffloadTearDown() { 646 (new AMessage(kWhatAudioOffloadTearDown, id()))->post(); 647} 648 649void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { 650 int32_t audio; 651 CHECK(msg->findInt32("audio", &audio)); 652 653 if (audio) { 654 mHasAudio = true; 655 } else { 656 mHasVideo = true; 657 if (mVideoScheduler == NULL) { 658 mVideoScheduler = new VideoFrameScheduler(); 659 mVideoScheduler->init(); 660 } 661 } 662 663 if (dropBufferWhileFlushing(audio, msg)) { 664 return; 665 } 666 667 sp<ABuffer> buffer; 668 CHECK(msg->findBuffer("buffer", &buffer)); 669 670 sp<AMessage> notifyConsumed; 671 CHECK(msg->findMessage("notifyConsumed", ¬ifyConsumed)); 672 673 QueueEntry entry; 674 entry.mBuffer = buffer; 675 entry.mNotifyConsumed = notifyConsumed; 676 entry.mOffset = 0; 677 entry.mFinalResult = OK; 678 679 if (audio) { 680 Mutex::Autolock autoLock(mLock); 681 mAudioQueue.push_back(entry); 682 postDrainAudioQueue_l(); 683 } else { 684 mVideoQueue.push_back(entry); 685 postDrainVideoQueue(); 686 } 687 688 Mutex::Autolock autoLock(mLock); 689 if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) { 690 return; 691 } 692 693 sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer; 694 sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer; 695 696 if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) { 697 // EOS signalled on either queue. 698 syncQueuesDone_l(); 699 return; 700 } 701 702 int64_t firstAudioTimeUs; 703 int64_t firstVideoTimeUs; 704 CHECK(firstAudioBuffer->meta() 705 ->findInt64("timeUs", &firstAudioTimeUs)); 706 CHECK(firstVideoBuffer->meta() 707 ->findInt64("timeUs", &firstVideoTimeUs)); 708 709 int64_t diff = firstVideoTimeUs - firstAudioTimeUs; 710 711 ALOGV("queueDiff = %.2f secs", diff / 1E6); 712 713 if (diff > 100000ll) { 714 // Audio data starts More than 0.1 secs before video. 715 // Drop some audio. 716 717 (*mAudioQueue.begin()).mNotifyConsumed->post(); 718 mAudioQueue.erase(mAudioQueue.begin()); 719 return; 720 } 721 722 syncQueuesDone_l(); 723} 724 725void NuPlayer::Renderer::syncQueuesDone_l() { 726 if (!mSyncQueues) { 727 return; 728 } 729 730 mSyncQueues = false; 731 732 if (!mAudioQueue.empty()) { 733 postDrainAudioQueue_l(); 734 } 735 736 if (!mVideoQueue.empty()) { 737 postDrainVideoQueue(); 738 } 739} 740 741void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) { 742 int32_t audio; 743 CHECK(msg->findInt32("audio", &audio)); 744 745 if (dropBufferWhileFlushing(audio, msg)) { 746 return; 747 } 748 749 int32_t finalResult; 750 CHECK(msg->findInt32("finalResult", &finalResult)); 751 752 QueueEntry entry; 753 entry.mOffset = 0; 754 entry.mFinalResult = finalResult; 755 756 if (audio) { 757 Mutex::Autolock autoLock(mLock); 758 if (mAudioQueue.empty() && mSyncQueues) { 759 syncQueuesDone_l(); 760 } 761 mAudioQueue.push_back(entry); 762 postDrainAudioQueue_l(); 763 } else { 764 if (mVideoQueue.empty() && mSyncQueues) { 765 Mutex::Autolock autoLock(mLock); 766 syncQueuesDone_l(); 767 } 768 mVideoQueue.push_back(entry); 769 postDrainVideoQueue(); 770 } 771} 772 773void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { 774 int32_t audio; 775 CHECK(msg->findInt32("audio", &audio)); 776 777 { 778 Mutex::Autolock autoLock(mFlushLock); 779 if (audio) { 780 mFlushingAudio = false; 781 } else { 782 mFlushingVideo = false; 783 } 784 } 785 786 // If we're currently syncing the queues, i.e. dropping audio while 787 // aligning the first audio/video buffer times and only one of the 788 // two queues has data, we may starve that queue by not requesting 789 // more buffers from the decoder. If the other source then encounters 790 // a discontinuity that leads to flushing, we'll never find the 791 // corresponding discontinuity on the other queue. 792 // Therefore we'll stop syncing the queues if at least one of them 793 // is flushed. 794 { 795 Mutex::Autolock autoLock(mLock); 796 syncQueuesDone_l(); 797 } 798 799 ALOGV("flushing %s", audio ? "audio" : "video"); 800 if (audio) { 801 { 802 Mutex::Autolock autoLock(mLock); 803 flushQueue(&mAudioQueue); 804 805 ++mAudioQueueGeneration; 806 prepareForMediaRenderingStart(); 807 808 if (offloadingAudio()) { 809 mFirstAnchorTimeMediaUs = -1; 810 } 811 } 812 813 mDrainAudioQueuePending = false; 814 815 if (offloadingAudio()) { 816 mAudioSink->pause(); 817 mAudioSink->flush(); 818 mAudioSink->start(); 819 } 820 } else { 821 flushQueue(&mVideoQueue); 822 823 mDrainVideoQueuePending = false; 824 ++mVideoQueueGeneration; 825 826 if (mVideoScheduler != NULL) { 827 mVideoScheduler->restart(); 828 } 829 830 prepareForMediaRenderingStart(); 831 } 832 833 mVideoSampleReceived = false; 834 notifyFlushComplete(audio); 835} 836 837void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) { 838 while (!queue->empty()) { 839 QueueEntry *entry = &*queue->begin(); 840 841 if (entry->mBuffer != NULL) { 842 entry->mNotifyConsumed->post(); 843 } 844 845 queue->erase(queue->begin()); 846 entry = NULL; 847 } 848} 849 850void NuPlayer::Renderer::notifyFlushComplete(bool audio) { 851 sp<AMessage> notify = mNotify->dup(); 852 notify->setInt32("what", kWhatFlushComplete); 853 notify->setInt32("audio", static_cast<int32_t>(audio)); 854 notify->post(); 855} 856 857bool NuPlayer::Renderer::dropBufferWhileFlushing( 858 bool audio, const sp<AMessage> &msg) { 859 bool flushing = false; 860 861 { 862 Mutex::Autolock autoLock(mFlushLock); 863 if (audio) { 864 flushing = mFlushingAudio; 865 } else { 866 flushing = mFlushingVideo; 867 } 868 } 869 870 if (!flushing) { 871 return false; 872 } 873 874 sp<AMessage> notifyConsumed; 875 if (msg->findMessage("notifyConsumed", ¬ifyConsumed)) { 876 notifyConsumed->post(); 877 } 878 879 return true; 880} 881 882void NuPlayer::Renderer::onAudioSinkChanged() { 883 if (offloadingAudio()) { 884 return; 885 } 886 CHECK(!mDrainAudioQueuePending); 887 mNumFramesWritten = 0; 888 uint32_t written; 889 if (mAudioSink->getFramesWritten(&written) == OK) { 890 mNumFramesWritten = written; 891 } 892} 893 894void NuPlayer::Renderer::onDisableOffloadAudio() { 895 Mutex::Autolock autoLock(mLock); 896 mFlags &= ~FLAG_OFFLOAD_AUDIO; 897 ++mAudioQueueGeneration; 898} 899 900void NuPlayer::Renderer::notifyPosition() { 901 // notifyPosition() must be called only after setting mAnchorTimeRealUs 902 // and mAnchorTimeMediaUs, and must not be paused as it extrapolates position. 903 //CHECK_GE(mAnchorTimeRealUs, 0); 904 //CHECK_GE(mAnchorTimeMediaUs, 0); 905 //CHECK(!mPaused || !mHasAudio); // video-only does display in paused mode. 906 907 int64_t nowUs = ALooper::GetNowUs(); 908 909 if (mLastPositionUpdateUs >= 0 910 && nowUs < mLastPositionUpdateUs + kMinPositionUpdateDelayUs) { 911 return; 912 } 913 mLastPositionUpdateUs = nowUs; 914 915 int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs; 916 917 //ALOGD("notifyPosition: positionUs(%lld) nowUs(%lld) mAnchorTimeRealUs(%lld)" 918 // " mAnchorTimeMediaUs(%lld) mFirstAnchorTimeMediaUs(%lld)", 919 // (long long)positionUs, (long long)nowUs, (long long)mAnchorTimeRealUs, 920 // (long long)mAnchorTimeMediaUs, (long long)mFirstAnchorTimeMediaUs); 921 922 // Due to adding the latency to mAnchorTimeRealUs in onDrainAudioQueue(), 923 // positionUs may be less than the first media time. This is avoided 924 // here to prevent potential retrograde motion of the position bar 925 // when starting up after a seek. 926 if (positionUs < mFirstAnchorTimeMediaUs) { 927 positionUs = mFirstAnchorTimeMediaUs; 928 } 929 sp<AMessage> notify = mNotify->dup(); 930 notify->setInt32("what", kWhatPosition); 931 notify->setInt64("positionUs", positionUs); 932 notify->setInt64("videoLateByUs", mVideoLateByUs); 933 notify->post(); 934} 935 936void NuPlayer::Renderer::onPause() { 937 if (mPaused) { 938 ALOGW("Renderer::onPause() called while already paused!"); 939 return; 940 } 941 { 942 Mutex::Autolock autoLock(mLock); 943 ++mAudioQueueGeneration; 944 ++mVideoQueueGeneration; 945 prepareForMediaRenderingStart(); 946 mPaused = true; 947 } 948 949 mDrainAudioQueuePending = false; 950 mDrainVideoQueuePending = false; 951 952 if (mHasAudio) { 953 mAudioSink->pause(); 954 } 955 956 ALOGV("now paused audio queue has %d entries, video has %d entries", 957 mAudioQueue.size(), mVideoQueue.size()); 958} 959 960void NuPlayer::Renderer::onResume() { 961 if (!mPaused) { 962 return; 963 } 964 965 if (mHasAudio) { 966 mAudioSink->start(); 967 } 968 969 Mutex::Autolock autoLock(mLock); 970 mPaused = false; 971 972 if (!mAudioQueue.empty()) { 973 postDrainAudioQueue_l(); 974 } 975 976 if (!mVideoQueue.empty()) { 977 postDrainVideoQueue(); 978 } 979} 980 981void NuPlayer::Renderer::onSetVideoFrameRate(float fps) { 982 if (mVideoScheduler == NULL) { 983 mVideoScheduler = new VideoFrameScheduler(); 984 } 985 mVideoScheduler->init(fps); 986} 987 988// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs() 989// as it acquires locks and may query the audio driver. 990// 991// Some calls are not needed since notifyPosition() doesn't always deliver a message. 992// Some calls could conceivably retrieve extrapolated data instead of 993// accessing getTimestamp() or getPosition() every time a data buffer with 994// a media time is received. 995// 996int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) { 997 uint32_t numFramesPlayed; 998 int64_t numFramesPlayedAt; 999 AudioTimestamp ts; 1000 static const int64_t kStaleTimestamp100ms = 100000; 1001 1002 status_t res = mAudioSink->getTimestamp(ts); 1003 if (res == OK) { // case 1: mixing audio tracks and offloaded tracks. 1004 numFramesPlayed = ts.mPosition; 1005 numFramesPlayedAt = 1006 ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000; 1007 const int64_t timestampAge = nowUs - numFramesPlayedAt; 1008 if (timestampAge > kStaleTimestamp100ms) { 1009 // This is an audio FIXME. 1010 // getTimestamp returns a timestamp which may come from audio mixing threads. 1011 // After pausing, the MixerThread may go idle, thus the mTime estimate may 1012 // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms, 1013 // the max latency should be about 25ms with an average around 12ms (to be verified). 1014 // For safety we use 100ms. 1015 ALOGW("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)", 1016 (long long)nowUs, (long long)numFramesPlayedAt); 1017 numFramesPlayedAt = nowUs - kStaleTimestamp100ms; 1018 } 1019 //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt); 1020 } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track 1021 numFramesPlayed = 0; 1022 numFramesPlayedAt = nowUs; 1023 //ALOGD("getTimestamp: WOULD_BLOCK %d %lld", 1024 // numFramesPlayed, (long long)numFramesPlayedAt); 1025 } else { // case 3: transitory at new track or audio fast tracks. 1026 res = mAudioSink->getPosition(&numFramesPlayed); 1027 CHECK_EQ(res, (status_t)OK); 1028 numFramesPlayedAt = nowUs; 1029 numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */ 1030 //ALOGD("getPosition: %d %lld", numFramesPlayed, numFramesPlayedAt); 1031 } 1032 1033 // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours. 1034 //CHECK_EQ(numFramesPlayed & (1 << 31), 0); // can't be negative until 12.4 hrs, test 1035 int64_t durationUs = (int32_t)numFramesPlayed * 1000LL * mAudioSink->msecsPerFrame() 1036 + nowUs - numFramesPlayedAt; 1037 if (durationUs < 0) { 1038 // Occurs when numFramesPlayed position is very small and the following: 1039 // (1) In case 1, the time nowUs is computed before getTimestamp() is called and 1040 // numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed. 1041 // (2) In case 3, using getPosition and adding mAudioSink->latency() to 1042 // numFramesPlayedAt, by a time amount greater than numFramesPlayed. 1043 // 1044 // Both of these are transitory conditions. 1045 ALOGW("getPlayedOutAudioDurationUs: negative timestamp %lld set to zero", (long long)durationUs); 1046 durationUs = 0; 1047 } 1048 ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)", 1049 (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt); 1050 return durationUs; 1051} 1052 1053void NuPlayer::Renderer::onAudioOffloadTearDown() { 1054 int64_t firstAudioTimeUs; 1055 { 1056 Mutex::Autolock autoLock(mLock); 1057 firstAudioTimeUs = mFirstAnchorTimeMediaUs; 1058 } 1059 1060 int64_t currentPositionUs = 1061 firstAudioTimeUs + getPlayedOutAudioDurationUs(ALooper::GetNowUs()); 1062 1063 mAudioSink->stop(); 1064 mAudioSink->flush(); 1065 1066 sp<AMessage> notify = mNotify->dup(); 1067 notify->setInt32("what", kWhatAudioOffloadTearDown); 1068 notify->setInt64("positionUs", currentPositionUs); 1069 notify->post(); 1070} 1071 1072} // namespace android 1073 1074