NuPlayerRenderer.cpp revision 274084f7bfd3f6bdf5a55cf16ceefed345aa0060
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "NuPlayerRenderer" 19#include <utils/Log.h> 20 21#include "NuPlayerRenderer.h" 22 23#include <cutils/properties.h> 24 25#include <media/stagefright/foundation/ABuffer.h> 26#include <media/stagefright/foundation/ADebug.h> 27#include <media/stagefright/foundation/AMessage.h> 28#include <media/stagefright/MediaErrors.h> 29#include <media/stagefright/MetaData.h> 30 31#include <VideoFrameScheduler.h> 32 33#include <inttypes.h> 34 35namespace android { 36 37// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink 38// is closed to allow the audio DSP to power down. 39static const int64_t kOffloadPauseMaxUs = 60000000ll; 40 41// static 42const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; 43 44static bool sFrameAccurateAVsync = false; 45 46static void readProperties() { 47 char value[PROPERTY_VALUE_MAX]; 48 if (property_get("persist.sys.media.avsync", value, NULL)) { 49 sFrameAccurateAVsync = 50 !strcmp("1", value) || !strcasecmp("true", value); 51 } 52} 53 54NuPlayer::Renderer::Renderer( 55 const sp<MediaPlayerBase::AudioSink> &sink, 56 const sp<AMessage> ¬ify, 57 uint32_t flags) 58 : mAudioSink(sink), 59 mNotify(notify), 60 mFlags(flags), 61 mNumFramesWritten(0), 62 mDrainAudioQueuePending(false), 63 mDrainVideoQueuePending(false), 64 mAudioQueueGeneration(0), 65 mVideoQueueGeneration(0), 66 mFirstAnchorTimeMediaUs(-1), 67 mAnchorTimeMediaUs(-1), 68 mAnchorTimeRealUs(-1), 69 mFlushingAudio(false), 70 mFlushingVideo(false), 71 mHasAudio(false), 72 mHasVideo(false), 73 mSyncQueues(false), 74 mPaused(false), 75 mVideoSampleReceived(false), 76 mVideoRenderingStarted(false), 77 mVideoRenderingStartGeneration(0), 78 mAudioRenderingStartGeneration(0), 79 mLastPositionUpdateUs(-1ll), 80 mVideoLateByUs(0ll), 81 mAudioOffloadPauseTimeoutGeneration(0), 82 mAudioOffloadTornDown(false) { 83 readProperties(); 84} 85 86NuPlayer::Renderer::~Renderer() { 87 if (offloadingAudio()) { 88 mAudioSink->stop(); 89 mAudioSink->flush(); 90 mAudioSink->close(); 91 } 92} 93 94void NuPlayer::Renderer::queueBuffer( 95 bool audio, 96 const sp<ABuffer> &buffer, 97 const sp<AMessage> ¬ifyConsumed) { 98 sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id()); 99 msg->setInt32("audio", static_cast<int32_t>(audio)); 100 msg->setBuffer("buffer", buffer); 101 msg->setMessage("notifyConsumed", notifyConsumed); 102 msg->post(); 103} 104 105void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) { 106 CHECK_NE(finalResult, (status_t)OK); 107 108 sp<AMessage> msg = new AMessage(kWhatQueueEOS, id()); 109 msg->setInt32("audio", static_cast<int32_t>(audio)); 110 msg->setInt32("finalResult", finalResult); 111 msg->post(); 112} 113 114void NuPlayer::Renderer::flush(bool audio) { 115 { 116 Mutex::Autolock autoLock(mFlushLock); 117 if (audio) { 118 if (mFlushingAudio) { 119 return; 120 } 121 mFlushingAudio = true; 122 } else { 123 if (mFlushingVideo) { 124 return; 125 } 126 mFlushingVideo = true; 127 } 128 } 129 130 sp<AMessage> msg = new AMessage(kWhatFlush, id()); 131 msg->setInt32("audio", static_cast<int32_t>(audio)); 132 msg->post(); 133} 134 135void NuPlayer::Renderer::signalTimeDiscontinuity() { 136 Mutex::Autolock autoLock(mLock); 137 // CHECK(mAudioQueue.empty()); 138 // CHECK(mVideoQueue.empty()); 139 mFirstAnchorTimeMediaUs = -1; 140 mAnchorTimeMediaUs = -1; 141 mAnchorTimeRealUs = -1; 142 mSyncQueues = false; 143} 144 145void NuPlayer::Renderer::signalAudioSinkChanged() { 146 (new AMessage(kWhatAudioSinkChanged, id()))->post(); 147} 148 149void NuPlayer::Renderer::signalDisableOffloadAudio() { 150 (new AMessage(kWhatDisableOffloadAudio, id()))->post(); 151} 152 153void NuPlayer::Renderer::pause() { 154 (new AMessage(kWhatPause, id()))->post(); 155} 156 157void NuPlayer::Renderer::resume() { 158 (new AMessage(kWhatResume, id()))->post(); 159} 160 161void NuPlayer::Renderer::setVideoFrameRate(float fps) { 162 sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, id()); 163 msg->setFloat("frame-rate", fps); 164 msg->post(); 165} 166 167void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { 168 switch (msg->what()) { 169 case kWhatStopAudioSink: 170 { 171 mAudioSink->stop(); 172 break; 173 } 174 175 case kWhatDrainAudioQueue: 176 { 177 int32_t generation; 178 CHECK(msg->findInt32("generation", &generation)); 179 if (generation != mAudioQueueGeneration) { 180 break; 181 } 182 183 mDrainAudioQueuePending = false; 184 185 if (onDrainAudioQueue()) { 186 uint32_t numFramesPlayed; 187 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), 188 (status_t)OK); 189 190 uint32_t numFramesPendingPlayout = 191 mNumFramesWritten - numFramesPlayed; 192 193 // This is how long the audio sink will have data to 194 // play back. 195 int64_t delayUs = 196 mAudioSink->msecsPerFrame() 197 * numFramesPendingPlayout * 1000ll; 198 199 // Let's give it more data after about half that time 200 // has elapsed. 201 // kWhatDrainAudioQueue is used for non-offloading mode, 202 // and mLock is used only for offloading mode. Therefore, 203 // no need to acquire mLock here. 204 postDrainAudioQueue_l(delayUs / 2); 205 } 206 break; 207 } 208 209 case kWhatDrainVideoQueue: 210 { 211 int32_t generation; 212 CHECK(msg->findInt32("generation", &generation)); 213 if (generation != mVideoQueueGeneration) { 214 break; 215 } 216 217 mDrainVideoQueuePending = false; 218 219 onDrainVideoQueue(); 220 221 postDrainVideoQueue(); 222 break; 223 } 224 225 case kWhatQueueBuffer: 226 { 227 onQueueBuffer(msg); 228 break; 229 } 230 231 case kWhatQueueEOS: 232 { 233 onQueueEOS(msg); 234 break; 235 } 236 237 case kWhatFlush: 238 { 239 onFlush(msg); 240 break; 241 } 242 243 case kWhatAudioSinkChanged: 244 { 245 onAudioSinkChanged(); 246 break; 247 } 248 249 case kWhatDisableOffloadAudio: 250 { 251 onDisableOffloadAudio(); 252 break; 253 } 254 255 case kWhatPause: 256 { 257 onPause(); 258 break; 259 } 260 261 case kWhatResume: 262 { 263 onResume(); 264 break; 265 } 266 267 case kWhatSetVideoFrameRate: 268 { 269 float fps; 270 CHECK(msg->findFloat("frame-rate", &fps)); 271 onSetVideoFrameRate(fps); 272 break; 273 } 274 275 case kWhatAudioOffloadTearDown: 276 { 277 onAudioOffloadTearDown(); 278 break; 279 } 280 281 case kWhatAudioOffloadPauseTimeout: 282 { 283 int32_t generation; 284 CHECK(msg->findInt32("generation", &generation)); 285 if (generation != mAudioOffloadPauseTimeoutGeneration) { 286 break; 287 } 288 onAudioOffloadTearDown(); 289 break; 290 } 291 292 default: 293 TRESPASS(); 294 break; 295 } 296} 297 298void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) { 299 if (mDrainAudioQueuePending || mSyncQueues || mPaused 300 || offloadingAudio()) { 301 return; 302 } 303 304 if (mAudioQueue.empty()) { 305 return; 306 } 307 308 mDrainAudioQueuePending = true; 309 sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id()); 310 msg->setInt32("generation", mAudioQueueGeneration); 311 msg->post(delayUs); 312} 313 314void NuPlayer::Renderer::prepareForMediaRenderingStart() { 315 mAudioRenderingStartGeneration = mAudioQueueGeneration; 316 mVideoRenderingStartGeneration = mVideoQueueGeneration; 317} 318 319void NuPlayer::Renderer::notifyIfMediaRenderingStarted() { 320 if (mVideoRenderingStartGeneration == mVideoQueueGeneration && 321 mAudioRenderingStartGeneration == mAudioQueueGeneration) { 322 mVideoRenderingStartGeneration = -1; 323 mAudioRenderingStartGeneration = -1; 324 325 sp<AMessage> notify = mNotify->dup(); 326 notify->setInt32("what", kWhatMediaRenderingStart); 327 notify->post(); 328 } 329} 330 331// static 332size_t NuPlayer::Renderer::AudioSinkCallback( 333 MediaPlayerBase::AudioSink * /* audioSink */, 334 void *buffer, 335 size_t size, 336 void *cookie, 337 MediaPlayerBase::AudioSink::cb_event_t event) { 338 NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie; 339 340 switch (event) { 341 case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER: 342 { 343 return me->fillAudioBuffer(buffer, size); 344 break; 345 } 346 347 case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END: 348 { 349 me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM); 350 break; 351 } 352 353 case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN: 354 { 355 me->notifyAudioOffloadTearDown(); 356 break; 357 } 358 } 359 360 return 0; 361} 362 363size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { 364 Mutex::Autolock autoLock(mLock); 365 366 if (!offloadingAudio() || mPaused) { 367 return 0; 368 } 369 370 bool hasEOS = false; 371 372 size_t sizeCopied = 0; 373 bool firstEntry = true; 374 while (sizeCopied < size && !mAudioQueue.empty()) { 375 QueueEntry *entry = &*mAudioQueue.begin(); 376 377 if (entry->mBuffer == NULL) { // EOS 378 hasEOS = true; 379 mAudioQueue.erase(mAudioQueue.begin()); 380 entry = NULL; 381 break; 382 } 383 384 if (firstEntry && entry->mOffset == 0) { 385 firstEntry = false; 386 int64_t mediaTimeUs; 387 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 388 ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); 389 if (mFirstAnchorTimeMediaUs == -1) { 390 mFirstAnchorTimeMediaUs = mediaTimeUs; 391 } 392 393 int64_t nowUs = ALooper::GetNowUs(); 394 mAnchorTimeMediaUs = 395 mFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs); 396 mAnchorTimeRealUs = nowUs; 397 398 notifyPosition(); 399 } 400 401 size_t copy = entry->mBuffer->size() - entry->mOffset; 402 size_t sizeRemaining = size - sizeCopied; 403 if (copy > sizeRemaining) { 404 copy = sizeRemaining; 405 } 406 407 memcpy((char *)buffer + sizeCopied, 408 entry->mBuffer->data() + entry->mOffset, 409 copy); 410 411 entry->mOffset += copy; 412 if (entry->mOffset == entry->mBuffer->size()) { 413 entry->mNotifyConsumed->post(); 414 mAudioQueue.erase(mAudioQueue.begin()); 415 entry = NULL; 416 } 417 sizeCopied += copy; 418 notifyIfMediaRenderingStarted(); 419 } 420 421 if (hasEOS) { 422 (new AMessage(kWhatStopAudioSink, id()))->post(); 423 } 424 425 return sizeCopied; 426} 427 428bool NuPlayer::Renderer::onDrainAudioQueue() { 429 uint32_t numFramesPlayed; 430 if (mAudioSink->getPosition(&numFramesPlayed) != OK) { 431 return false; 432 } 433 434 ssize_t numFramesAvailableToWrite = 435 mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed); 436 437#if 0 438 if (numFramesAvailableToWrite == mAudioSink->frameCount()) { 439 ALOGI("audio sink underrun"); 440 } else { 441 ALOGV("audio queue has %d frames left to play", 442 mAudioSink->frameCount() - numFramesAvailableToWrite); 443 } 444#endif 445 446 size_t numBytesAvailableToWrite = 447 numFramesAvailableToWrite * mAudioSink->frameSize(); 448 449 while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) { 450 QueueEntry *entry = &*mAudioQueue.begin(); 451 452 if (entry->mBuffer == NULL) { 453 // EOS 454 int64_t postEOSDelayUs = 0; 455 if (mAudioSink->needsTrailingPadding()) { 456 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs()); 457 } 458 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs); 459 460 mAudioQueue.erase(mAudioQueue.begin()); 461 entry = NULL; 462 return false; 463 } 464 465 if (entry->mOffset == 0) { 466 int64_t mediaTimeUs; 467 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 468 ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); 469 if (mFirstAnchorTimeMediaUs == -1) { 470 mFirstAnchorTimeMediaUs = mediaTimeUs; 471 } 472 mAnchorTimeMediaUs = mediaTimeUs; 473 474 int64_t nowUs = ALooper::GetNowUs(); 475 mAnchorTimeRealUs = nowUs + getPendingAudioPlayoutDurationUs(nowUs); 476 477 notifyPosition(); 478 } 479 480 size_t copy = entry->mBuffer->size() - entry->mOffset; 481 if (copy > numBytesAvailableToWrite) { 482 copy = numBytesAvailableToWrite; 483 } 484 485 ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, copy); 486 if (written < 0) { 487 // An error in AudioSink write is fatal here. 488 LOG_ALWAYS_FATAL("AudioSink write error(%zd) when writing %zu bytes", written, copy); 489 } 490 491 entry->mOffset += written; 492 if (entry->mOffset == entry->mBuffer->size()) { 493 entry->mNotifyConsumed->post(); 494 mAudioQueue.erase(mAudioQueue.begin()); 495 496 entry = NULL; 497 } 498 499 numBytesAvailableToWrite -= written; 500 size_t copiedFrames = written / mAudioSink->frameSize(); 501 mNumFramesWritten += copiedFrames; 502 503 notifyIfMediaRenderingStarted(); 504 505 if (written != (ssize_t)copy) { 506 // A short count was received from AudioSink::write() 507 // 508 // AudioSink write should block until exactly the number of bytes are delivered. 509 // But it may return with a short count (without an error) when: 510 // 511 // 1) Size to be copied is not a multiple of the frame size. We consider this fatal. 512 // 2) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded. 513 514 // (Case 1) 515 // Must be a multiple of the frame size. If it is not a multiple of a frame size, it 516 // needs to fail, as we should not carry over fractional frames between calls. 517 CHECK_EQ(copy % mAudioSink->frameSize(), 0); 518 519 // (Case 2) 520 // Return early to the caller. 521 // Beware of calling immediately again as this may busy-loop if you are not careful. 522 ALOGW("AudioSink write short frame count %zd < %zu", written, copy); 523 break; 524 } 525 } 526 return !mAudioQueue.empty(); 527} 528 529int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) { 530 int64_t writtenAudioDurationUs = 531 mNumFramesWritten * 1000LL * mAudioSink->msecsPerFrame(); 532 return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs); 533} 534 535void NuPlayer::Renderer::postDrainVideoQueue() { 536 if (mDrainVideoQueuePending 537 || mSyncQueues 538 || (mPaused && mVideoSampleReceived)) { 539 return; 540 } 541 542 if (mVideoQueue.empty()) { 543 return; 544 } 545 546 QueueEntry &entry = *mVideoQueue.begin(); 547 548 sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id()); 549 msg->setInt32("generation", mVideoQueueGeneration); 550 551 if (entry.mBuffer == NULL) { 552 // EOS doesn't carry a timestamp. 553 msg->post(); 554 mDrainVideoQueuePending = true; 555 return; 556 } 557 558 int64_t delayUs; 559 int64_t nowUs = ALooper::GetNowUs(); 560 int64_t realTimeUs; 561 if (mFlags & FLAG_REAL_TIME) { 562 int64_t mediaTimeUs; 563 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 564 realTimeUs = mediaTimeUs; 565 } else { 566 int64_t mediaTimeUs; 567 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 568 569 if (mFirstAnchorTimeMediaUs == -1 && !mHasAudio) { 570 mFirstAnchorTimeMediaUs = mediaTimeUs; 571 } 572 if (mAnchorTimeMediaUs < 0) { 573 if (!mHasAudio) { 574 mAnchorTimeMediaUs = mediaTimeUs; 575 mAnchorTimeRealUs = nowUs; 576 notifyPosition(); 577 } 578 realTimeUs = nowUs; 579 } else { 580 realTimeUs = 581 (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs; 582 } 583 } 584 585 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000; 586 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000); 587 588 delayUs = realTimeUs - nowUs; 589 590 ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs); 591 // post 2 display refreshes before rendering is due 592 // FIXME currently this increases power consumption, so unless frame-accurate 593 // AV sync is requested, post closer to required render time (at 0.63 vsyncs) 594 if (!sFrameAccurateAVsync) { 595 twoVsyncsUs >>= 4; 596 } 597 msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0); 598 599 mDrainVideoQueuePending = true; 600} 601 602void NuPlayer::Renderer::onDrainVideoQueue() { 603 if (mVideoQueue.empty()) { 604 return; 605 } 606 607 QueueEntry *entry = &*mVideoQueue.begin(); 608 609 if (entry->mBuffer == NULL) { 610 // EOS 611 612 notifyEOS(false /* audio */, entry->mFinalResult); 613 614 mVideoQueue.erase(mVideoQueue.begin()); 615 entry = NULL; 616 617 mVideoLateByUs = 0ll; 618 return; 619 } 620 621 int64_t realTimeUs; 622 if (mFlags & FLAG_REAL_TIME) { 623 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs)); 624 } else { 625 int64_t mediaTimeUs; 626 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 627 628 realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs; 629 } 630 631 bool tooLate = false; 632 633 if (!mPaused) { 634 mVideoLateByUs = ALooper::GetNowUs() - realTimeUs; 635 tooLate = (mVideoLateByUs > 40000); 636 637 if (tooLate) { 638 ALOGV("video late by %lld us (%.2f secs)", 639 mVideoLateByUs, mVideoLateByUs / 1E6); 640 } else { 641 ALOGV("rendering video at media time %.2f secs", 642 (mFlags & FLAG_REAL_TIME ? realTimeUs : 643 (realTimeUs + mAnchorTimeMediaUs - mAnchorTimeRealUs)) / 1E6); 644 } 645 } else { 646 mVideoLateByUs = 0ll; 647 } 648 649 entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll); 650 entry->mNotifyConsumed->setInt32("render", !tooLate); 651 entry->mNotifyConsumed->post(); 652 mVideoQueue.erase(mVideoQueue.begin()); 653 entry = NULL; 654 655 mVideoSampleReceived = true; 656 657 if (!mPaused) { 658 if (!mVideoRenderingStarted) { 659 mVideoRenderingStarted = true; 660 notifyVideoRenderingStart(); 661 } 662 notifyIfMediaRenderingStarted(); 663 } 664} 665 666void NuPlayer::Renderer::notifyVideoRenderingStart() { 667 sp<AMessage> notify = mNotify->dup(); 668 notify->setInt32("what", kWhatVideoRenderingStart); 669 notify->post(); 670} 671 672void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) { 673 sp<AMessage> notify = mNotify->dup(); 674 notify->setInt32("what", kWhatEOS); 675 notify->setInt32("audio", static_cast<int32_t>(audio)); 676 notify->setInt32("finalResult", finalResult); 677 notify->post(delayUs); 678} 679 680void NuPlayer::Renderer::notifyAudioOffloadTearDown() { 681 (new AMessage(kWhatAudioOffloadTearDown, id()))->post(); 682} 683 684void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { 685 int32_t audio; 686 CHECK(msg->findInt32("audio", &audio)); 687 688 if (audio) { 689 mHasAudio = true; 690 } else { 691 mHasVideo = true; 692 if (mVideoScheduler == NULL) { 693 mVideoScheduler = new VideoFrameScheduler(); 694 mVideoScheduler->init(); 695 } 696 } 697 698 if (dropBufferWhileFlushing(audio, msg)) { 699 return; 700 } 701 702 sp<ABuffer> buffer; 703 CHECK(msg->findBuffer("buffer", &buffer)); 704 705 sp<AMessage> notifyConsumed; 706 CHECK(msg->findMessage("notifyConsumed", ¬ifyConsumed)); 707 708 QueueEntry entry; 709 entry.mBuffer = buffer; 710 entry.mNotifyConsumed = notifyConsumed; 711 entry.mOffset = 0; 712 entry.mFinalResult = OK; 713 714 if (audio) { 715 Mutex::Autolock autoLock(mLock); 716 mAudioQueue.push_back(entry); 717 postDrainAudioQueue_l(); 718 } else { 719 mVideoQueue.push_back(entry); 720 postDrainVideoQueue(); 721 } 722 723 Mutex::Autolock autoLock(mLock); 724 if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) { 725 return; 726 } 727 728 sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer; 729 sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer; 730 731 if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) { 732 // EOS signalled on either queue. 733 syncQueuesDone_l(); 734 return; 735 } 736 737 int64_t firstAudioTimeUs; 738 int64_t firstVideoTimeUs; 739 CHECK(firstAudioBuffer->meta() 740 ->findInt64("timeUs", &firstAudioTimeUs)); 741 CHECK(firstVideoBuffer->meta() 742 ->findInt64("timeUs", &firstVideoTimeUs)); 743 744 int64_t diff = firstVideoTimeUs - firstAudioTimeUs; 745 746 ALOGV("queueDiff = %.2f secs", diff / 1E6); 747 748 if (diff > 100000ll) { 749 // Audio data starts More than 0.1 secs before video. 750 // Drop some audio. 751 752 (*mAudioQueue.begin()).mNotifyConsumed->post(); 753 mAudioQueue.erase(mAudioQueue.begin()); 754 return; 755 } 756 757 syncQueuesDone_l(); 758} 759 760void NuPlayer::Renderer::syncQueuesDone_l() { 761 if (!mSyncQueues) { 762 return; 763 } 764 765 mSyncQueues = false; 766 767 if (!mAudioQueue.empty()) { 768 postDrainAudioQueue_l(); 769 } 770 771 if (!mVideoQueue.empty()) { 772 postDrainVideoQueue(); 773 } 774} 775 776void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) { 777 int32_t audio; 778 CHECK(msg->findInt32("audio", &audio)); 779 780 if (dropBufferWhileFlushing(audio, msg)) { 781 return; 782 } 783 784 int32_t finalResult; 785 CHECK(msg->findInt32("finalResult", &finalResult)); 786 787 QueueEntry entry; 788 entry.mOffset = 0; 789 entry.mFinalResult = finalResult; 790 791 if (audio) { 792 Mutex::Autolock autoLock(mLock); 793 if (mAudioQueue.empty() && mSyncQueues) { 794 syncQueuesDone_l(); 795 } 796 mAudioQueue.push_back(entry); 797 postDrainAudioQueue_l(); 798 } else { 799 if (mVideoQueue.empty() && mSyncQueues) { 800 Mutex::Autolock autoLock(mLock); 801 syncQueuesDone_l(); 802 } 803 mVideoQueue.push_back(entry); 804 postDrainVideoQueue(); 805 } 806} 807 808void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { 809 int32_t audio; 810 CHECK(msg->findInt32("audio", &audio)); 811 812 { 813 Mutex::Autolock autoLock(mFlushLock); 814 if (audio) { 815 mFlushingAudio = false; 816 } else { 817 mFlushingVideo = false; 818 } 819 } 820 821 // If we're currently syncing the queues, i.e. dropping audio while 822 // aligning the first audio/video buffer times and only one of the 823 // two queues has data, we may starve that queue by not requesting 824 // more buffers from the decoder. If the other source then encounters 825 // a discontinuity that leads to flushing, we'll never find the 826 // corresponding discontinuity on the other queue. 827 // Therefore we'll stop syncing the queues if at least one of them 828 // is flushed. 829 { 830 Mutex::Autolock autoLock(mLock); 831 syncQueuesDone_l(); 832 } 833 834 ALOGV("flushing %s", audio ? "audio" : "video"); 835 if (audio) { 836 { 837 Mutex::Autolock autoLock(mLock); 838 flushQueue(&mAudioQueue); 839 840 ++mAudioQueueGeneration; 841 prepareForMediaRenderingStart(); 842 843 if (offloadingAudio()) { 844 mFirstAnchorTimeMediaUs = -1; 845 } 846 } 847 848 mDrainAudioQueuePending = false; 849 850 if (offloadingAudio()) { 851 mAudioSink->pause(); 852 mAudioSink->flush(); 853 mAudioSink->start(); 854 } 855 } else { 856 flushQueue(&mVideoQueue); 857 858 mDrainVideoQueuePending = false; 859 ++mVideoQueueGeneration; 860 861 if (mVideoScheduler != NULL) { 862 mVideoScheduler->restart(); 863 } 864 865 prepareForMediaRenderingStart(); 866 } 867 868 mVideoSampleReceived = false; 869 notifyFlushComplete(audio); 870} 871 872void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) { 873 while (!queue->empty()) { 874 QueueEntry *entry = &*queue->begin(); 875 876 if (entry->mBuffer != NULL) { 877 entry->mNotifyConsumed->post(); 878 } 879 880 queue->erase(queue->begin()); 881 entry = NULL; 882 } 883} 884 885void NuPlayer::Renderer::notifyFlushComplete(bool audio) { 886 sp<AMessage> notify = mNotify->dup(); 887 notify->setInt32("what", kWhatFlushComplete); 888 notify->setInt32("audio", static_cast<int32_t>(audio)); 889 notify->post(); 890} 891 892bool NuPlayer::Renderer::dropBufferWhileFlushing( 893 bool audio, const sp<AMessage> &msg) { 894 bool flushing = false; 895 896 { 897 Mutex::Autolock autoLock(mFlushLock); 898 if (audio) { 899 flushing = mFlushingAudio; 900 } else { 901 flushing = mFlushingVideo; 902 } 903 } 904 905 if (!flushing) { 906 return false; 907 } 908 909 sp<AMessage> notifyConsumed; 910 if (msg->findMessage("notifyConsumed", ¬ifyConsumed)) { 911 notifyConsumed->post(); 912 } 913 914 return true; 915} 916 917void NuPlayer::Renderer::onAudioSinkChanged() { 918 if (offloadingAudio()) { 919 return; 920 } 921 CHECK(!mDrainAudioQueuePending); 922 mNumFramesWritten = 0; 923 uint32_t written; 924 if (mAudioSink->getFramesWritten(&written) == OK) { 925 mNumFramesWritten = written; 926 } 927} 928 929void NuPlayer::Renderer::onDisableOffloadAudio() { 930 Mutex::Autolock autoLock(mLock); 931 mFlags &= ~FLAG_OFFLOAD_AUDIO; 932 ++mAudioQueueGeneration; 933} 934 935void NuPlayer::Renderer::notifyPosition() { 936 // notifyPosition() must be called only after setting mAnchorTimeRealUs 937 // and mAnchorTimeMediaUs, and must not be paused as it extrapolates position. 938 //CHECK_GE(mAnchorTimeRealUs, 0); 939 //CHECK_GE(mAnchorTimeMediaUs, 0); 940 //CHECK(!mPaused || !mHasAudio); // video-only does display in paused mode. 941 942 int64_t nowUs = ALooper::GetNowUs(); 943 944 if (mLastPositionUpdateUs >= 0 945 && nowUs < mLastPositionUpdateUs + kMinPositionUpdateDelayUs) { 946 return; 947 } 948 mLastPositionUpdateUs = nowUs; 949 950 int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs; 951 952 //ALOGD("notifyPosition: positionUs(%lld) nowUs(%lld) mAnchorTimeRealUs(%lld)" 953 // " mAnchorTimeMediaUs(%lld) mFirstAnchorTimeMediaUs(%lld)", 954 // (long long)positionUs, (long long)nowUs, (long long)mAnchorTimeRealUs, 955 // (long long)mAnchorTimeMediaUs, (long long)mFirstAnchorTimeMediaUs); 956 957 // Due to adding the latency to mAnchorTimeRealUs in onDrainAudioQueue(), 958 // positionUs may be less than the first media time. This is avoided 959 // here to prevent potential retrograde motion of the position bar 960 // when starting up after a seek. 961 if (positionUs < mFirstAnchorTimeMediaUs) { 962 positionUs = mFirstAnchorTimeMediaUs; 963 } 964 sp<AMessage> notify = mNotify->dup(); 965 notify->setInt32("what", kWhatPosition); 966 notify->setInt64("positionUs", positionUs); 967 notify->setInt64("videoLateByUs", mVideoLateByUs); 968 notify->post(); 969} 970 971void NuPlayer::Renderer::onPause() { 972 if (mPaused) { 973 ALOGW("Renderer::onPause() called while already paused!"); 974 return; 975 } 976 { 977 Mutex::Autolock autoLock(mLock); 978 ++mAudioQueueGeneration; 979 ++mVideoQueueGeneration; 980 prepareForMediaRenderingStart(); 981 mPaused = true; 982 } 983 984 mDrainAudioQueuePending = false; 985 mDrainVideoQueuePending = false; 986 987 if (mHasAudio) { 988 mAudioSink->pause(); 989 startAudioOffloadPauseTimeout(); 990 } 991 992 ALOGV("now paused audio queue has %d entries, video has %d entries", 993 mAudioQueue.size(), mVideoQueue.size()); 994} 995 996void NuPlayer::Renderer::onResume() { 997 readProperties(); 998 999 if (!mPaused) { 1000 return; 1001 } 1002 1003 if (mHasAudio) { 1004 cancelAudioOffloadPauseTimeout(); 1005 mAudioSink->start(); 1006 } 1007 1008 Mutex::Autolock autoLock(mLock); 1009 mPaused = false; 1010 1011 if (!mAudioQueue.empty()) { 1012 postDrainAudioQueue_l(); 1013 } 1014 1015 if (!mVideoQueue.empty()) { 1016 postDrainVideoQueue(); 1017 } 1018} 1019 1020void NuPlayer::Renderer::onSetVideoFrameRate(float fps) { 1021 if (mVideoScheduler == NULL) { 1022 mVideoScheduler = new VideoFrameScheduler(); 1023 } 1024 mVideoScheduler->init(fps); 1025} 1026 1027// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs() 1028// as it acquires locks and may query the audio driver. 1029// 1030// Some calls are not needed since notifyPosition() doesn't always deliver a message. 1031// Some calls could conceivably retrieve extrapolated data instead of 1032// accessing getTimestamp() or getPosition() every time a data buffer with 1033// a media time is received. 1034// 1035int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) { 1036 uint32_t numFramesPlayed; 1037 int64_t numFramesPlayedAt; 1038 AudioTimestamp ts; 1039 static const int64_t kStaleTimestamp100ms = 100000; 1040 1041 status_t res = mAudioSink->getTimestamp(ts); 1042 if (res == OK) { // case 1: mixing audio tracks and offloaded tracks. 1043 numFramesPlayed = ts.mPosition; 1044 numFramesPlayedAt = 1045 ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000; 1046 const int64_t timestampAge = nowUs - numFramesPlayedAt; 1047 if (timestampAge > kStaleTimestamp100ms) { 1048 // This is an audio FIXME. 1049 // getTimestamp returns a timestamp which may come from audio mixing threads. 1050 // After pausing, the MixerThread may go idle, thus the mTime estimate may 1051 // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms, 1052 // the max latency should be about 25ms with an average around 12ms (to be verified). 1053 // For safety we use 100ms. 1054 ALOGW("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)", 1055 (long long)nowUs, (long long)numFramesPlayedAt); 1056 numFramesPlayedAt = nowUs - kStaleTimestamp100ms; 1057 } 1058 //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt); 1059 } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track 1060 numFramesPlayed = 0; 1061 numFramesPlayedAt = nowUs; 1062 //ALOGD("getTimestamp: WOULD_BLOCK %d %lld", 1063 // numFramesPlayed, (long long)numFramesPlayedAt); 1064 } else { // case 3: transitory at new track or audio fast tracks. 1065 res = mAudioSink->getPosition(&numFramesPlayed); 1066 CHECK_EQ(res, (status_t)OK); 1067 numFramesPlayedAt = nowUs; 1068 numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */ 1069 //ALOGD("getPosition: %d %lld", numFramesPlayed, numFramesPlayedAt); 1070 } 1071 1072 // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours. 1073 //CHECK_EQ(numFramesPlayed & (1 << 31), 0); // can't be negative until 12.4 hrs, test 1074 int64_t durationUs = (int32_t)numFramesPlayed * 1000LL * mAudioSink->msecsPerFrame() 1075 + nowUs - numFramesPlayedAt; 1076 if (durationUs < 0) { 1077 // Occurs when numFramesPlayed position is very small and the following: 1078 // (1) In case 1, the time nowUs is computed before getTimestamp() is called and 1079 // numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed. 1080 // (2) In case 3, using getPosition and adding mAudioSink->latency() to 1081 // numFramesPlayedAt, by a time amount greater than numFramesPlayed. 1082 // 1083 // Both of these are transitory conditions. 1084 ALOGW("getPlayedOutAudioDurationUs: negative timestamp %lld set to zero", (long long)durationUs); 1085 durationUs = 0; 1086 } 1087 ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)", 1088 (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt); 1089 return durationUs; 1090} 1091 1092void NuPlayer::Renderer::onAudioOffloadTearDown() { 1093 if (mAudioOffloadTornDown) { 1094 return; 1095 } 1096 mAudioOffloadTornDown = true; 1097 1098 int64_t firstAudioTimeUs; 1099 { 1100 Mutex::Autolock autoLock(mLock); 1101 firstAudioTimeUs = mFirstAnchorTimeMediaUs; 1102 } 1103 1104 int64_t currentPositionUs = 1105 firstAudioTimeUs + getPlayedOutAudioDurationUs(ALooper::GetNowUs()); 1106 1107 mAudioSink->stop(); 1108 mAudioSink->flush(); 1109 1110 sp<AMessage> notify = mNotify->dup(); 1111 notify->setInt32("what", kWhatAudioOffloadTearDown); 1112 notify->setInt64("positionUs", currentPositionUs); 1113 notify->post(); 1114} 1115 1116void NuPlayer::Renderer::startAudioOffloadPauseTimeout() { 1117 if (offloadingAudio()) { 1118 sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, id()); 1119 msg->setInt32("generation", mAudioOffloadPauseTimeoutGeneration); 1120 msg->post(kOffloadPauseMaxUs); 1121 } 1122} 1123 1124void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() { 1125 if (offloadingAudio()) { 1126 ++mAudioOffloadPauseTimeoutGeneration; 1127 } 1128} 1129 1130} // namespace android 1131 1132