NuPlayerRenderer.cpp revision a73d9e0b3d171d2bfcd9eb07df9d6d36ae74df57
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "NuPlayerRenderer" 19#include <utils/Log.h> 20 21#include "NuPlayerRenderer.h" 22 23#include <cutils/properties.h> 24 25#include <media/stagefright/foundation/ABuffer.h> 26#include <media/stagefright/foundation/ADebug.h> 27#include <media/stagefright/foundation/AMessage.h> 28#include <media/stagefright/foundation/AUtils.h> 29#include <media/stagefright/MediaErrors.h> 30#include <media/stagefright/MetaData.h> 31 32#include <VideoFrameScheduler.h> 33 34#include <inttypes.h> 35 36namespace android { 37 38// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink 39// is closed to allow the audio DSP to power down. 40static const int64_t kOffloadPauseMaxUs = 60000000ll; 41 42// static 43const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; 44 45static bool sFrameAccurateAVsync = false; 46 47static void readProperties() { 48 char value[PROPERTY_VALUE_MAX]; 49 if (property_get("persist.sys.media.avsync", value, NULL)) { 50 sFrameAccurateAVsync = 51 !strcmp("1", value) || !strcasecmp("true", value); 52 } 53} 54 55NuPlayer::Renderer::Renderer( 56 const sp<MediaPlayerBase::AudioSink> &sink, 57 const sp<AMessage> ¬ify, 58 uint32_t flags) 59 : mAudioSink(sink), 60 mNotify(notify), 61 mFlags(flags), 62 mNumFramesWritten(0), 63 mDrainAudioQueuePending(false), 64 mDrainVideoQueuePending(false), 65 mAudioQueueGeneration(0), 66 mVideoQueueGeneration(0), 67 mAudioFirstAnchorTimeMediaUs(-1), 68 mVideoAnchorTimeMediaUs(-1), 69 mVideoAnchorTimeRealUs(-1), 70 mVideoLateByUs(0ll), 71 mHasAudio(false), 72 mHasVideo(false), 73 mPauseStartedTimeRealUs(-1), 74 mFlushingAudio(false), 75 mFlushingVideo(false), 76 mSyncQueues(false), 77 mPaused(false), 78 mVideoSampleReceived(false), 79 mVideoRenderingStarted(false), 80 mVideoRenderingStartGeneration(0), 81 mAudioRenderingStartGeneration(0), 82 mAudioOffloadPauseTimeoutGeneration(0), 83 mAudioOffloadTornDown(false) { 84 readProperties(); 85} 86 87NuPlayer::Renderer::~Renderer() { 88 if (offloadingAudio()) { 89 mAudioSink->stop(); 90 mAudioSink->flush(); 91 mAudioSink->close(); 92 } 93} 94 95void NuPlayer::Renderer::queueBuffer( 96 bool audio, 97 const sp<ABuffer> &buffer, 98 const sp<AMessage> ¬ifyConsumed) { 99 sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id()); 100 msg->setInt32("audio", static_cast<int32_t>(audio)); 101 msg->setBuffer("buffer", buffer); 102 msg->setMessage("notifyConsumed", notifyConsumed); 103 msg->post(); 104} 105 106void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) { 107 CHECK_NE(finalResult, (status_t)OK); 108 109 sp<AMessage> msg = new AMessage(kWhatQueueEOS, id()); 110 msg->setInt32("audio", static_cast<int32_t>(audio)); 111 msg->setInt32("finalResult", finalResult); 112 msg->post(); 113} 114 115void NuPlayer::Renderer::flush(bool audio) { 116 { 117 Mutex::Autolock autoLock(mFlushLock); 118 if (audio) { 119 if (mFlushingAudio) { 120 return; 121 } 122 mFlushingAudio = true; 123 } else { 124 if (mFlushingVideo) { 125 return; 126 } 127 mFlushingVideo = true; 128 } 129 } 130 131 sp<AMessage> msg = new AMessage(kWhatFlush, id()); 132 msg->setInt32("audio", static_cast<int32_t>(audio)); 133 msg->post(); 134} 135 136void NuPlayer::Renderer::signalTimeDiscontinuity() { 137 Mutex::Autolock autoLock(mLock); 138 // CHECK(mAudioQueue.empty()); 139 // CHECK(mVideoQueue.empty()); 140 setAudioFirstAnchorTime(-1); 141 setVideoAnchorTime(-1, -1); 142 setVideoLateByUs(0); 143 mSyncQueues = false; 144} 145 146void NuPlayer::Renderer::signalAudioSinkChanged() { 147 (new AMessage(kWhatAudioSinkChanged, id()))->post(); 148} 149 150void NuPlayer::Renderer::signalDisableOffloadAudio() { 151 (new AMessage(kWhatDisableOffloadAudio, id()))->post(); 152} 153 154void NuPlayer::Renderer::pause() { 155 (new AMessage(kWhatPause, id()))->post(); 156} 157 158void NuPlayer::Renderer::resume() { 159 (new AMessage(kWhatResume, id()))->post(); 160} 161 162void NuPlayer::Renderer::setVideoFrameRate(float fps) { 163 sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, id()); 164 msg->setFloat("frame-rate", fps); 165 msg->post(); 166} 167 168status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) { 169 return getCurrentPosition(mediaUs, ALooper::GetNowUs()); 170} 171 172status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs, int64_t nowUs) { 173 Mutex::Autolock autoLock(mTimeLock); 174 if (!mHasAudio && !mHasVideo) { 175 return NO_INIT; 176 } 177 178 int64_t positionUs = 0; 179 if (!mHasAudio) { 180 if (mVideoAnchorTimeMediaUs < 0) { 181 return NO_INIT; 182 } 183 positionUs = (nowUs - mVideoAnchorTimeRealUs) + mVideoAnchorTimeMediaUs; 184 185 if (mPauseStartedTimeRealUs != -1) { 186 positionUs -= (nowUs - mPauseStartedTimeRealUs); 187 } 188 } else { 189 if (mAudioFirstAnchorTimeMediaUs < 0) { 190 return NO_INIT; 191 } 192 positionUs = mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs); 193 } 194 *mediaUs = (positionUs <= 0) ? 0 : positionUs; 195 return OK; 196} 197 198void NuPlayer::Renderer::setHasMedia(bool audio) { 199 Mutex::Autolock autoLock(mTimeLock); 200 if (audio) { 201 mHasAudio = true; 202 } else { 203 mHasVideo = true; 204 } 205} 206 207void NuPlayer::Renderer::setAudioFirstAnchorTime(int64_t mediaUs) { 208 Mutex::Autolock autoLock(mTimeLock); 209 mAudioFirstAnchorTimeMediaUs = mediaUs; 210} 211 212void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded(int64_t mediaUs) { 213 Mutex::Autolock autoLock(mTimeLock); 214 if (mAudioFirstAnchorTimeMediaUs == -1) { 215 mAudioFirstAnchorTimeMediaUs = mediaUs; 216 } 217} 218 219void NuPlayer::Renderer::setVideoAnchorTime(int64_t mediaUs, int64_t realUs) { 220 Mutex::Autolock autoLock(mTimeLock); 221 mVideoAnchorTimeMediaUs = mediaUs; 222 mVideoAnchorTimeRealUs = realUs; 223} 224 225void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) { 226 Mutex::Autolock autoLock(mTimeLock); 227 mVideoLateByUs = lateUs; 228} 229 230int64_t NuPlayer::Renderer::getVideoLateByUs() { 231 Mutex::Autolock autoLock(mTimeLock); 232 return mVideoLateByUs; 233} 234 235void NuPlayer::Renderer::setPauseStartedTimeRealUs(int64_t realUs) { 236 Mutex::Autolock autoLock(mTimeLock); 237 mPauseStartedTimeRealUs = realUs; 238} 239 240void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { 241 switch (msg->what()) { 242 case kWhatStopAudioSink: 243 { 244 mAudioSink->stop(); 245 break; 246 } 247 248 case kWhatDrainAudioQueue: 249 { 250 int32_t generation; 251 CHECK(msg->findInt32("generation", &generation)); 252 if (generation != mAudioQueueGeneration) { 253 break; 254 } 255 256 mDrainAudioQueuePending = false; 257 258 if (onDrainAudioQueue()) { 259 uint32_t numFramesPlayed; 260 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), 261 (status_t)OK); 262 263 uint32_t numFramesPendingPlayout = 264 mNumFramesWritten - numFramesPlayed; 265 266 // This is how long the audio sink will have data to 267 // play back. 268 int64_t delayUs = 269 mAudioSink->msecsPerFrame() 270 * numFramesPendingPlayout * 1000ll; 271 272 // Let's give it more data after about half that time 273 // has elapsed. 274 // kWhatDrainAudioQueue is used for non-offloading mode, 275 // and mLock is used only for offloading mode. Therefore, 276 // no need to acquire mLock here. 277 postDrainAudioQueue_l(delayUs / 2); 278 } 279 break; 280 } 281 282 case kWhatDrainVideoQueue: 283 { 284 int32_t generation; 285 CHECK(msg->findInt32("generation", &generation)); 286 if (generation != mVideoQueueGeneration) { 287 break; 288 } 289 290 mDrainVideoQueuePending = false; 291 292 onDrainVideoQueue(); 293 294 postDrainVideoQueue(); 295 break; 296 } 297 298 case kWhatQueueBuffer: 299 { 300 onQueueBuffer(msg); 301 break; 302 } 303 304 case kWhatQueueEOS: 305 { 306 onQueueEOS(msg); 307 break; 308 } 309 310 case kWhatFlush: 311 { 312 onFlush(msg); 313 break; 314 } 315 316 case kWhatAudioSinkChanged: 317 { 318 onAudioSinkChanged(); 319 break; 320 } 321 322 case kWhatDisableOffloadAudio: 323 { 324 onDisableOffloadAudio(); 325 break; 326 } 327 328 case kWhatPause: 329 { 330 onPause(); 331 break; 332 } 333 334 case kWhatResume: 335 { 336 onResume(); 337 break; 338 } 339 340 case kWhatSetVideoFrameRate: 341 { 342 float fps; 343 CHECK(msg->findFloat("frame-rate", &fps)); 344 onSetVideoFrameRate(fps); 345 break; 346 } 347 348 case kWhatAudioOffloadTearDown: 349 { 350 onAudioOffloadTearDown(kDueToError); 351 break; 352 } 353 354 case kWhatAudioOffloadPauseTimeout: 355 { 356 int32_t generation; 357 CHECK(msg->findInt32("generation", &generation)); 358 if (generation != mAudioOffloadPauseTimeoutGeneration) { 359 break; 360 } 361 ALOGV("Audio Offload tear down due to pause timeout."); 362 onAudioOffloadTearDown(kDueToTimeout); 363 break; 364 } 365 366 default: 367 TRESPASS(); 368 break; 369 } 370} 371 372void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) { 373 if (mDrainAudioQueuePending || mSyncQueues || mPaused 374 || offloadingAudio()) { 375 return; 376 } 377 378 if (mAudioQueue.empty()) { 379 return; 380 } 381 382 mDrainAudioQueuePending = true; 383 sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id()); 384 msg->setInt32("generation", mAudioQueueGeneration); 385 msg->post(delayUs); 386} 387 388void NuPlayer::Renderer::prepareForMediaRenderingStart() { 389 mAudioRenderingStartGeneration = mAudioQueueGeneration; 390 mVideoRenderingStartGeneration = mVideoQueueGeneration; 391} 392 393void NuPlayer::Renderer::notifyIfMediaRenderingStarted() { 394 if (mVideoRenderingStartGeneration == mVideoQueueGeneration && 395 mAudioRenderingStartGeneration == mAudioQueueGeneration) { 396 mVideoRenderingStartGeneration = -1; 397 mAudioRenderingStartGeneration = -1; 398 399 sp<AMessage> notify = mNotify->dup(); 400 notify->setInt32("what", kWhatMediaRenderingStart); 401 notify->post(); 402 } 403} 404 405// static 406size_t NuPlayer::Renderer::AudioSinkCallback( 407 MediaPlayerBase::AudioSink * /* audioSink */, 408 void *buffer, 409 size_t size, 410 void *cookie, 411 MediaPlayerBase::AudioSink::cb_event_t event) { 412 NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie; 413 414 switch (event) { 415 case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER: 416 { 417 return me->fillAudioBuffer(buffer, size); 418 break; 419 } 420 421 case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END: 422 { 423 me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM); 424 break; 425 } 426 427 case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN: 428 { 429 me->notifyAudioOffloadTearDown(); 430 break; 431 } 432 } 433 434 return 0; 435} 436 437size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { 438 Mutex::Autolock autoLock(mLock); 439 440 if (!offloadingAudio() || mPaused) { 441 return 0; 442 } 443 444 bool hasEOS = false; 445 446 size_t sizeCopied = 0; 447 bool firstEntry = true; 448 while (sizeCopied < size && !mAudioQueue.empty()) { 449 QueueEntry *entry = &*mAudioQueue.begin(); 450 451 if (entry->mBuffer == NULL) { // EOS 452 hasEOS = true; 453 mAudioQueue.erase(mAudioQueue.begin()); 454 entry = NULL; 455 break; 456 } 457 458 if (firstEntry && entry->mOffset == 0) { 459 firstEntry = false; 460 int64_t mediaTimeUs; 461 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 462 ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); 463 setAudioFirstAnchorTimeIfNeeded(mediaTimeUs); 464 } 465 466 size_t copy = entry->mBuffer->size() - entry->mOffset; 467 size_t sizeRemaining = size - sizeCopied; 468 if (copy > sizeRemaining) { 469 copy = sizeRemaining; 470 } 471 472 memcpy((char *)buffer + sizeCopied, 473 entry->mBuffer->data() + entry->mOffset, 474 copy); 475 476 entry->mOffset += copy; 477 if (entry->mOffset == entry->mBuffer->size()) { 478 entry->mNotifyConsumed->post(); 479 mAudioQueue.erase(mAudioQueue.begin()); 480 entry = NULL; 481 } 482 sizeCopied += copy; 483 notifyIfMediaRenderingStarted(); 484 } 485 486 if (hasEOS) { 487 (new AMessage(kWhatStopAudioSink, id()))->post(); 488 } 489 490 return sizeCopied; 491} 492 493bool NuPlayer::Renderer::onDrainAudioQueue() { 494 uint32_t numFramesPlayed; 495 if (mAudioSink->getPosition(&numFramesPlayed) != OK) { 496 return false; 497 } 498 499 ssize_t numFramesAvailableToWrite = 500 mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed); 501 502#if 0 503 if (numFramesAvailableToWrite == mAudioSink->frameCount()) { 504 ALOGI("audio sink underrun"); 505 } else { 506 ALOGV("audio queue has %d frames left to play", 507 mAudioSink->frameCount() - numFramesAvailableToWrite); 508 } 509#endif 510 511 size_t numBytesAvailableToWrite = 512 numFramesAvailableToWrite * mAudioSink->frameSize(); 513 514 while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) { 515 QueueEntry *entry = &*mAudioQueue.begin(); 516 517 if (entry->mBuffer == NULL) { 518 // EOS 519 int64_t postEOSDelayUs = 0; 520 if (mAudioSink->needsTrailingPadding()) { 521 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs()); 522 } 523 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs); 524 525 mAudioQueue.erase(mAudioQueue.begin()); 526 entry = NULL; 527 return false; 528 } 529 530 if (entry->mOffset == 0) { 531 int64_t mediaTimeUs; 532 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 533 ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); 534 535 setAudioFirstAnchorTimeIfNeeded(mediaTimeUs); 536 } 537 538 size_t copy = entry->mBuffer->size() - entry->mOffset; 539 if (copy > numBytesAvailableToWrite) { 540 copy = numBytesAvailableToWrite; 541 } 542 543 ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, copy); 544 if (written < 0) { 545 // An error in AudioSink write is fatal here. 546 LOG_ALWAYS_FATAL("AudioSink write error(%zd) when writing %zu bytes", written, copy); 547 } 548 549 entry->mOffset += written; 550 if (entry->mOffset == entry->mBuffer->size()) { 551 entry->mNotifyConsumed->post(); 552 mAudioQueue.erase(mAudioQueue.begin()); 553 554 entry = NULL; 555 } 556 557 numBytesAvailableToWrite -= written; 558 size_t copiedFrames = written / mAudioSink->frameSize(); 559 mNumFramesWritten += copiedFrames; 560 561 notifyIfMediaRenderingStarted(); 562 563 if (written != (ssize_t)copy) { 564 // A short count was received from AudioSink::write() 565 // 566 // AudioSink write should block until exactly the number of bytes are delivered. 567 // But it may return with a short count (without an error) when: 568 // 569 // 1) Size to be copied is not a multiple of the frame size. We consider this fatal. 570 // 2) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded. 571 572 // (Case 1) 573 // Must be a multiple of the frame size. If it is not a multiple of a frame size, it 574 // needs to fail, as we should not carry over fractional frames between calls. 575 CHECK_EQ(copy % mAudioSink->frameSize(), 0); 576 577 // (Case 2) 578 // Return early to the caller. 579 // Beware of calling immediately again as this may busy-loop if you are not careful. 580 ALOGW("AudioSink write short frame count %zd < %zu", written, copy); 581 break; 582 } 583 } 584 return !mAudioQueue.empty(); 585} 586 587int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) { 588 int64_t writtenAudioDurationUs = 589 mNumFramesWritten * 1000LL * mAudioSink->msecsPerFrame(); 590 return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs); 591} 592 593int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) { 594 int64_t currentPositionUs; 595 if (getCurrentPosition(¤tPositionUs, nowUs) != OK) { 596 currentPositionUs = 0; 597 } 598 return (mediaTimeUs - currentPositionUs) + nowUs; 599} 600 601void NuPlayer::Renderer::postDrainVideoQueue() { 602 if (mDrainVideoQueuePending 603 || mSyncQueues 604 || (mPaused && mVideoSampleReceived)) { 605 return; 606 } 607 608 if (mVideoQueue.empty()) { 609 return; 610 } 611 612 QueueEntry &entry = *mVideoQueue.begin(); 613 614 sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id()); 615 msg->setInt32("generation", mVideoQueueGeneration); 616 617 if (entry.mBuffer == NULL) { 618 // EOS doesn't carry a timestamp. 619 msg->post(); 620 mDrainVideoQueuePending = true; 621 return; 622 } 623 624 int64_t delayUs; 625 int64_t nowUs = ALooper::GetNowUs(); 626 int64_t realTimeUs; 627 if (mFlags & FLAG_REAL_TIME) { 628 int64_t mediaTimeUs; 629 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 630 realTimeUs = mediaTimeUs; 631 } else { 632 int64_t mediaTimeUs; 633 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 634 635 if (mVideoAnchorTimeMediaUs < 0) { 636 setVideoAnchorTime(mediaTimeUs, nowUs); 637 realTimeUs = nowUs; 638 } else { 639 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); 640 } 641 } 642 643 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000; 644 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000); 645 646 delayUs = realTimeUs - nowUs; 647 648 ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs); 649 // post 2 display refreshes before rendering is due 650 // FIXME currently this increases power consumption, so unless frame-accurate 651 // AV sync is requested, post closer to required render time (at 0.63 vsyncs) 652 if (!sFrameAccurateAVsync) { 653 twoVsyncsUs >>= 4; 654 } 655 msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0); 656 657 mDrainVideoQueuePending = true; 658} 659 660void NuPlayer::Renderer::onDrainVideoQueue() { 661 if (mVideoQueue.empty()) { 662 return; 663 } 664 665 QueueEntry *entry = &*mVideoQueue.begin(); 666 667 if (entry->mBuffer == NULL) { 668 // EOS 669 670 notifyEOS(false /* audio */, entry->mFinalResult); 671 672 mVideoQueue.erase(mVideoQueue.begin()); 673 entry = NULL; 674 675 setVideoLateByUs(0); 676 return; 677 } 678 679 int64_t nowUs = -1; 680 int64_t realTimeUs; 681 if (mFlags & FLAG_REAL_TIME) { 682 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs)); 683 } else { 684 int64_t mediaTimeUs; 685 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 686 687 nowUs = ALooper::GetNowUs(); 688 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); 689 } 690 691 bool tooLate = false; 692 693 if (!mPaused) { 694 if (nowUs == -1) { 695 nowUs = ALooper::GetNowUs(); 696 } 697 setVideoLateByUs(nowUs - realTimeUs); 698 tooLate = (mVideoLateByUs > 40000); 699 700 if (tooLate) { 701 ALOGV("video late by %lld us (%.2f secs)", 702 mVideoLateByUs, mVideoLateByUs / 1E6); 703 } else { 704 ALOGV("rendering video at media time %.2f secs", 705 (mFlags & FLAG_REAL_TIME ? realTimeUs : 706 (realTimeUs + mVideoAnchorTimeMediaUs - mVideoAnchorTimeRealUs)) / 1E6); 707 } 708 } else { 709 setVideoLateByUs(0); 710 if (!mVideoSampleReceived) { 711 // This will ensure that the first frame after a flush won't be used as anchor 712 // when renderer is in paused state, because resume can happen any time after seek. 713 setVideoAnchorTime(-1, -1); 714 } 715 } 716 717 entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll); 718 entry->mNotifyConsumed->setInt32("render", !tooLate); 719 entry->mNotifyConsumed->post(); 720 mVideoQueue.erase(mVideoQueue.begin()); 721 entry = NULL; 722 723 mVideoSampleReceived = true; 724 725 if (!mPaused) { 726 if (!mVideoRenderingStarted) { 727 mVideoRenderingStarted = true; 728 notifyVideoRenderingStart(); 729 } 730 notifyIfMediaRenderingStarted(); 731 } 732} 733 734void NuPlayer::Renderer::notifyVideoRenderingStart() { 735 sp<AMessage> notify = mNotify->dup(); 736 notify->setInt32("what", kWhatVideoRenderingStart); 737 notify->post(); 738} 739 740void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) { 741 sp<AMessage> notify = mNotify->dup(); 742 notify->setInt32("what", kWhatEOS); 743 notify->setInt32("audio", static_cast<int32_t>(audio)); 744 notify->setInt32("finalResult", finalResult); 745 notify->post(delayUs); 746} 747 748void NuPlayer::Renderer::notifyAudioOffloadTearDown() { 749 (new AMessage(kWhatAudioOffloadTearDown, id()))->post(); 750} 751 752void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { 753 int32_t audio; 754 CHECK(msg->findInt32("audio", &audio)); 755 756 setHasMedia(audio); 757 758 if (mHasVideo) { 759 if (mVideoScheduler == NULL) { 760 mVideoScheduler = new VideoFrameScheduler(); 761 mVideoScheduler->init(); 762 } 763 } 764 765 if (dropBufferWhileFlushing(audio, msg)) { 766 return; 767 } 768 769 sp<ABuffer> buffer; 770 CHECK(msg->findBuffer("buffer", &buffer)); 771 772 sp<AMessage> notifyConsumed; 773 CHECK(msg->findMessage("notifyConsumed", ¬ifyConsumed)); 774 775 QueueEntry entry; 776 entry.mBuffer = buffer; 777 entry.mNotifyConsumed = notifyConsumed; 778 entry.mOffset = 0; 779 entry.mFinalResult = OK; 780 781 if (audio) { 782 Mutex::Autolock autoLock(mLock); 783 mAudioQueue.push_back(entry); 784 postDrainAudioQueue_l(); 785 } else { 786 mVideoQueue.push_back(entry); 787 postDrainVideoQueue(); 788 } 789 790 Mutex::Autolock autoLock(mLock); 791 if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) { 792 return; 793 } 794 795 sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer; 796 sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer; 797 798 if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) { 799 // EOS signalled on either queue. 800 syncQueuesDone_l(); 801 return; 802 } 803 804 int64_t firstAudioTimeUs; 805 int64_t firstVideoTimeUs; 806 CHECK(firstAudioBuffer->meta() 807 ->findInt64("timeUs", &firstAudioTimeUs)); 808 CHECK(firstVideoBuffer->meta() 809 ->findInt64("timeUs", &firstVideoTimeUs)); 810 811 int64_t diff = firstVideoTimeUs - firstAudioTimeUs; 812 813 ALOGV("queueDiff = %.2f secs", diff / 1E6); 814 815 if (diff > 100000ll) { 816 // Audio data starts More than 0.1 secs before video. 817 // Drop some audio. 818 819 (*mAudioQueue.begin()).mNotifyConsumed->post(); 820 mAudioQueue.erase(mAudioQueue.begin()); 821 return; 822 } 823 824 syncQueuesDone_l(); 825} 826 827void NuPlayer::Renderer::syncQueuesDone_l() { 828 if (!mSyncQueues) { 829 return; 830 } 831 832 mSyncQueues = false; 833 834 if (!mAudioQueue.empty()) { 835 postDrainAudioQueue_l(); 836 } 837 838 if (!mVideoQueue.empty()) { 839 postDrainVideoQueue(); 840 } 841} 842 843void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) { 844 int32_t audio; 845 CHECK(msg->findInt32("audio", &audio)); 846 847 if (dropBufferWhileFlushing(audio, msg)) { 848 return; 849 } 850 851 int32_t finalResult; 852 CHECK(msg->findInt32("finalResult", &finalResult)); 853 854 QueueEntry entry; 855 entry.mOffset = 0; 856 entry.mFinalResult = finalResult; 857 858 if (audio) { 859 Mutex::Autolock autoLock(mLock); 860 if (mAudioQueue.empty() && mSyncQueues) { 861 syncQueuesDone_l(); 862 } 863 mAudioQueue.push_back(entry); 864 postDrainAudioQueue_l(); 865 } else { 866 if (mVideoQueue.empty() && mSyncQueues) { 867 Mutex::Autolock autoLock(mLock); 868 syncQueuesDone_l(); 869 } 870 mVideoQueue.push_back(entry); 871 postDrainVideoQueue(); 872 } 873} 874 875void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { 876 int32_t audio; 877 CHECK(msg->findInt32("audio", &audio)); 878 879 { 880 Mutex::Autolock autoLock(mFlushLock); 881 if (audio) { 882 mFlushingAudio = false; 883 } else { 884 mFlushingVideo = false; 885 } 886 } 887 888 // If we're currently syncing the queues, i.e. dropping audio while 889 // aligning the first audio/video buffer times and only one of the 890 // two queues has data, we may starve that queue by not requesting 891 // more buffers from the decoder. If the other source then encounters 892 // a discontinuity that leads to flushing, we'll never find the 893 // corresponding discontinuity on the other queue. 894 // Therefore we'll stop syncing the queues if at least one of them 895 // is flushed. 896 { 897 Mutex::Autolock autoLock(mLock); 898 syncQueuesDone_l(); 899 setPauseStartedTimeRealUs(-1); 900 } 901 902 ALOGV("flushing %s", audio ? "audio" : "video"); 903 if (audio) { 904 { 905 Mutex::Autolock autoLock(mLock); 906 flushQueue(&mAudioQueue); 907 908 ++mAudioQueueGeneration; 909 prepareForMediaRenderingStart(); 910 911 if (offloadingAudio()) { 912 setAudioFirstAnchorTime(-1); 913 } 914 } 915 916 mDrainAudioQueuePending = false; 917 918 if (offloadingAudio()) { 919 mAudioSink->pause(); 920 mAudioSink->flush(); 921 mAudioSink->start(); 922 } 923 } else { 924 flushQueue(&mVideoQueue); 925 926 mDrainVideoQueuePending = false; 927 ++mVideoQueueGeneration; 928 929 if (mVideoScheduler != NULL) { 930 mVideoScheduler->restart(); 931 } 932 933 prepareForMediaRenderingStart(); 934 } 935 936 mVideoSampleReceived = false; 937 notifyFlushComplete(audio); 938} 939 940void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) { 941 while (!queue->empty()) { 942 QueueEntry *entry = &*queue->begin(); 943 944 if (entry->mBuffer != NULL) { 945 entry->mNotifyConsumed->post(); 946 } 947 948 queue->erase(queue->begin()); 949 entry = NULL; 950 } 951} 952 953void NuPlayer::Renderer::notifyFlushComplete(bool audio) { 954 sp<AMessage> notify = mNotify->dup(); 955 notify->setInt32("what", kWhatFlushComplete); 956 notify->setInt32("audio", static_cast<int32_t>(audio)); 957 notify->post(); 958} 959 960bool NuPlayer::Renderer::dropBufferWhileFlushing( 961 bool audio, const sp<AMessage> &msg) { 962 bool flushing = false; 963 964 { 965 Mutex::Autolock autoLock(mFlushLock); 966 if (audio) { 967 flushing = mFlushingAudio; 968 } else { 969 flushing = mFlushingVideo; 970 } 971 } 972 973 if (!flushing) { 974 return false; 975 } 976 977 sp<AMessage> notifyConsumed; 978 if (msg->findMessage("notifyConsumed", ¬ifyConsumed)) { 979 notifyConsumed->post(); 980 } 981 982 return true; 983} 984 985void NuPlayer::Renderer::onAudioSinkChanged() { 986 if (offloadingAudio()) { 987 return; 988 } 989 CHECK(!mDrainAudioQueuePending); 990 mNumFramesWritten = 0; 991 uint32_t written; 992 if (mAudioSink->getFramesWritten(&written) == OK) { 993 mNumFramesWritten = written; 994 } 995} 996 997void NuPlayer::Renderer::onDisableOffloadAudio() { 998 Mutex::Autolock autoLock(mLock); 999 mFlags &= ~FLAG_OFFLOAD_AUDIO; 1000 ++mAudioQueueGeneration; 1001} 1002 1003void NuPlayer::Renderer::onPause() { 1004 if (mPaused) { 1005 ALOGW("Renderer::onPause() called while already paused!"); 1006 return; 1007 } 1008 { 1009 Mutex::Autolock autoLock(mLock); 1010 ++mAudioQueueGeneration; 1011 ++mVideoQueueGeneration; 1012 prepareForMediaRenderingStart(); 1013 mPaused = true; 1014 setPauseStartedTimeRealUs(ALooper::GetNowUs()); 1015 } 1016 1017 mDrainAudioQueuePending = false; 1018 mDrainVideoQueuePending = false; 1019 1020 if (mHasAudio) { 1021 mAudioSink->pause(); 1022 startAudioOffloadPauseTimeout(); 1023 } 1024 1025 ALOGV("now paused audio queue has %d entries, video has %d entries", 1026 mAudioQueue.size(), mVideoQueue.size()); 1027} 1028 1029void NuPlayer::Renderer::onResume() { 1030 readProperties(); 1031 1032 if (!mPaused) { 1033 return; 1034 } 1035 1036 if (mHasAudio) { 1037 cancelAudioOffloadPauseTimeout(); 1038 mAudioSink->start(); 1039 } 1040 1041 Mutex::Autolock autoLock(mLock); 1042 mPaused = false; 1043 if (mPauseStartedTimeRealUs != -1) { 1044 int64_t newAnchorRealUs = 1045 mVideoAnchorTimeRealUs + ALooper::GetNowUs() - mPauseStartedTimeRealUs; 1046 setVideoAnchorTime(mVideoAnchorTimeMediaUs, newAnchorRealUs); 1047 setPauseStartedTimeRealUs(-1); 1048 } 1049 1050 if (!mAudioQueue.empty()) { 1051 postDrainAudioQueue_l(); 1052 } 1053 1054 if (!mVideoQueue.empty()) { 1055 postDrainVideoQueue(); 1056 } 1057} 1058 1059void NuPlayer::Renderer::onSetVideoFrameRate(float fps) { 1060 if (mVideoScheduler == NULL) { 1061 mVideoScheduler = new VideoFrameScheduler(); 1062 } 1063 mVideoScheduler->init(fps); 1064} 1065 1066// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs() 1067// as it acquires locks and may query the audio driver. 1068// 1069// Some calls could conceivably retrieve extrapolated data instead of 1070// accessing getTimestamp() or getPosition() every time a data buffer with 1071// a media time is received. 1072// 1073int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) { 1074 uint32_t numFramesPlayed; 1075 int64_t numFramesPlayedAt; 1076 AudioTimestamp ts; 1077 static const int64_t kStaleTimestamp100ms = 100000; 1078 1079 status_t res = mAudioSink->getTimestamp(ts); 1080 if (res == OK) { // case 1: mixing audio tracks and offloaded tracks. 1081 numFramesPlayed = ts.mPosition; 1082 numFramesPlayedAt = 1083 ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000; 1084 const int64_t timestampAge = nowUs - numFramesPlayedAt; 1085 if (timestampAge > kStaleTimestamp100ms) { 1086 // This is an audio FIXME. 1087 // getTimestamp returns a timestamp which may come from audio mixing threads. 1088 // After pausing, the MixerThread may go idle, thus the mTime estimate may 1089 // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms, 1090 // the max latency should be about 25ms with an average around 12ms (to be verified). 1091 // For safety we use 100ms. 1092 ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)", 1093 (long long)nowUs, (long long)numFramesPlayedAt); 1094 numFramesPlayedAt = nowUs - kStaleTimestamp100ms; 1095 } 1096 //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt); 1097 } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track 1098 numFramesPlayed = 0; 1099 numFramesPlayedAt = nowUs; 1100 //ALOGD("getTimestamp: WOULD_BLOCK %d %lld", 1101 // numFramesPlayed, (long long)numFramesPlayedAt); 1102 } else { // case 3: transitory at new track or audio fast tracks. 1103 res = mAudioSink->getPosition(&numFramesPlayed); 1104 CHECK_EQ(res, (status_t)OK); 1105 numFramesPlayedAt = nowUs; 1106 numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */ 1107 //ALOGD("getPosition: %d %lld", numFramesPlayed, numFramesPlayedAt); 1108 } 1109 1110 // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours. 1111 //CHECK_EQ(numFramesPlayed & (1 << 31), 0); // can't be negative until 12.4 hrs, test 1112 int64_t durationUs = (int32_t)numFramesPlayed * 1000LL * mAudioSink->msecsPerFrame() 1113 + nowUs - numFramesPlayedAt; 1114 if (durationUs < 0) { 1115 // Occurs when numFramesPlayed position is very small and the following: 1116 // (1) In case 1, the time nowUs is computed before getTimestamp() is called and 1117 // numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed. 1118 // (2) In case 3, using getPosition and adding mAudioSink->latency() to 1119 // numFramesPlayedAt, by a time amount greater than numFramesPlayed. 1120 // 1121 // Both of these are transitory conditions. 1122 ALOGV("getPlayedOutAudioDurationUs: negative duration %lld set to zero", (long long)durationUs); 1123 durationUs = 0; 1124 } 1125 ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)", 1126 (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt); 1127 return durationUs; 1128} 1129 1130void NuPlayer::Renderer::onAudioOffloadTearDown(AudioOffloadTearDownReason reason) { 1131 if (mAudioOffloadTornDown) { 1132 return; 1133 } 1134 mAudioOffloadTornDown = true; 1135 1136 int64_t currentPositionUs; 1137 if (getCurrentPosition(¤tPositionUs) != OK) { 1138 currentPositionUs = 0; 1139 } 1140 1141 mAudioSink->stop(); 1142 mAudioSink->flush(); 1143 1144 sp<AMessage> notify = mNotify->dup(); 1145 notify->setInt32("what", kWhatAudioOffloadTearDown); 1146 notify->setInt64("positionUs", currentPositionUs); 1147 notify->setInt32("reason", reason); 1148 notify->post(); 1149} 1150 1151void NuPlayer::Renderer::startAudioOffloadPauseTimeout() { 1152 if (offloadingAudio()) { 1153 sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, id()); 1154 msg->setInt32("generation", mAudioOffloadPauseTimeoutGeneration); 1155 msg->post(kOffloadPauseMaxUs); 1156 } 1157} 1158 1159void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() { 1160 if (offloadingAudio()) { 1161 ++mAudioOffloadPauseTimeoutGeneration; 1162 } 1163} 1164 1165} // namespace android 1166 1167