PlaybackSession.cpp revision 4646d5756c06e896f9f8489fa07cd14f958b2f84
1/* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "PlaybackSession" 19#include <utils/Log.h> 20 21#include "PlaybackSession.h" 22 23#include "Converter.h" 24#include "MediaPuller.h" 25#include "RepeaterSource.h" 26#include "include/avc_utils.h" 27#include "WifiDisplaySource.h" 28 29#include <binder/IServiceManager.h> 30#include <cutils/properties.h> 31#include <media/IHDCP.h> 32#include <media/stagefright/foundation/ABitReader.h> 33#include <media/stagefright/foundation/ABuffer.h> 34#include <media/stagefright/foundation/ADebug.h> 35#include <media/stagefright/foundation/AMessage.h> 36#include <media/stagefright/foundation/hexdump.h> 37#include <media/stagefright/AudioSource.h> 38#include <media/stagefright/DataSource.h> 39#include <media/stagefright/MediaDefs.h> 40#include <media/stagefright/MediaErrors.h> 41#include <media/stagefright/MediaSource.h> 42#include <media/stagefright/MetaData.h> 43#include <media/stagefright/NuMediaExtractor.h> 44#include <media/stagefright/SurfaceMediaSource.h> 45#include <media/stagefright/Utils.h> 46 47#include <OMX_IVCommon.h> 48 49namespace android { 50 51struct WifiDisplaySource::PlaybackSession::Track : public AHandler { 52 enum { 53 kWhatStopped, 54 }; 55 56 Track(const sp<AMessage> ¬ify, 57 const sp<ALooper> &pullLooper, 58 const sp<ALooper> &codecLooper, 59 const sp<MediaPuller> &mediaPuller, 60 const sp<Converter> &converter); 61 62 Track(const sp<AMessage> ¬ify, const sp<AMessage> &format); 63 64 void setRepeaterSource(const sp<RepeaterSource> &source); 65 66 sp<AMessage> getFormat(); 67 bool isAudio() const; 68 69 const sp<Converter> &converter() const; 70 const sp<RepeaterSource> &repeaterSource() const; 71 72 ssize_t mediaSenderTrackIndex() const; 73 void setMediaSenderTrackIndex(size_t index); 74 75 status_t start(); 76 void stopAsync(); 77 78 void pause(); 79 void resume(); 80 81 void queueAccessUnit(const sp<ABuffer> &accessUnit); 82 sp<ABuffer> dequeueAccessUnit(); 83 84 bool hasOutputBuffer(int64_t *timeUs) const; 85 void queueOutputBuffer(const sp<ABuffer> &accessUnit); 86 sp<ABuffer> dequeueOutputBuffer(); 87 88#if SUSPEND_VIDEO_IF_IDLE 89 bool isSuspended() const; 90#endif 91 92 size_t countQueuedOutputBuffers() const { 93 return mQueuedOutputBuffers.size(); 94 } 95 96 void requestIDRFrame(); 97 98protected: 99 virtual void onMessageReceived(const sp<AMessage> &msg); 100 virtual ~Track(); 101 102private: 103 enum { 104 kWhatMediaPullerStopped, 105 }; 106 107 sp<AMessage> mNotify; 108 sp<ALooper> mPullLooper; 109 sp<ALooper> mCodecLooper; 110 sp<MediaPuller> mMediaPuller; 111 sp<Converter> mConverter; 112 sp<AMessage> mFormat; 113 bool mStarted; 114 ssize_t mMediaSenderTrackIndex; 115 bool mIsAudio; 116 List<sp<ABuffer> > mQueuedAccessUnits; 117 sp<RepeaterSource> mRepeaterSource; 118 List<sp<ABuffer> > mQueuedOutputBuffers; 119 int64_t mLastOutputBufferQueuedTimeUs; 120 121 static bool IsAudioFormat(const sp<AMessage> &format); 122 123 DISALLOW_EVIL_CONSTRUCTORS(Track); 124}; 125 126WifiDisplaySource::PlaybackSession::Track::Track( 127 const sp<AMessage> ¬ify, 128 const sp<ALooper> &pullLooper, 129 const sp<ALooper> &codecLooper, 130 const sp<MediaPuller> &mediaPuller, 131 const sp<Converter> &converter) 132 : mNotify(notify), 133 mPullLooper(pullLooper), 134 mCodecLooper(codecLooper), 135 mMediaPuller(mediaPuller), 136 mConverter(converter), 137 mStarted(false), 138 mIsAudio(IsAudioFormat(mConverter->getOutputFormat())), 139 mLastOutputBufferQueuedTimeUs(-1ll) { 140} 141 142WifiDisplaySource::PlaybackSession::Track::Track( 143 const sp<AMessage> ¬ify, const sp<AMessage> &format) 144 : mNotify(notify), 145 mFormat(format), 146 mStarted(false), 147 mIsAudio(IsAudioFormat(format)), 148 mLastOutputBufferQueuedTimeUs(-1ll) { 149} 150 151WifiDisplaySource::PlaybackSession::Track::~Track() { 152 CHECK(!mStarted); 153} 154 155// static 156bool WifiDisplaySource::PlaybackSession::Track::IsAudioFormat( 157 const sp<AMessage> &format) { 158 AString mime; 159 CHECK(format->findString("mime", &mime)); 160 161 return !strncasecmp(mime.c_str(), "audio/", 6); 162} 163 164sp<AMessage> WifiDisplaySource::PlaybackSession::Track::getFormat() { 165 return mFormat != NULL ? mFormat : mConverter->getOutputFormat(); 166} 167 168bool WifiDisplaySource::PlaybackSession::Track::isAudio() const { 169 return mIsAudio; 170} 171 172const sp<Converter> &WifiDisplaySource::PlaybackSession::Track::converter() const { 173 return mConverter; 174} 175 176const sp<RepeaterSource> & 177WifiDisplaySource::PlaybackSession::Track::repeaterSource() const { 178 return mRepeaterSource; 179} 180 181ssize_t WifiDisplaySource::PlaybackSession::Track::mediaSenderTrackIndex() const { 182 CHECK_GE(mMediaSenderTrackIndex, 0); 183 return mMediaSenderTrackIndex; 184} 185 186void WifiDisplaySource::PlaybackSession::Track::setMediaSenderTrackIndex( 187 size_t index) { 188 mMediaSenderTrackIndex = index; 189} 190 191status_t WifiDisplaySource::PlaybackSession::Track::start() { 192 ALOGV("Track::start isAudio=%d", mIsAudio); 193 194 CHECK(!mStarted); 195 196 status_t err = OK; 197 198 if (mMediaPuller != NULL) { 199 err = mMediaPuller->start(); 200 } 201 202 if (err == OK) { 203 mStarted = true; 204 } 205 206 return err; 207} 208 209void WifiDisplaySource::PlaybackSession::Track::stopAsync() { 210 ALOGV("Track::stopAsync isAudio=%d", mIsAudio); 211 212 if (mConverter != NULL) { 213 mConverter->shutdownAsync(); 214 } 215 216 sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, id()); 217 218 if (mStarted && mMediaPuller != NULL) { 219 if (mRepeaterSource != NULL) { 220 // Let's unblock MediaPuller's MediaSource::read(). 221 mRepeaterSource->wakeUp(); 222 } 223 224 mMediaPuller->stopAsync(msg); 225 } else { 226 mStarted = false; 227 msg->post(); 228 } 229} 230 231void WifiDisplaySource::PlaybackSession::Track::pause() { 232 mMediaPuller->pause(); 233} 234 235void WifiDisplaySource::PlaybackSession::Track::resume() { 236 mMediaPuller->resume(); 237} 238 239void WifiDisplaySource::PlaybackSession::Track::onMessageReceived( 240 const sp<AMessage> &msg) { 241 switch (msg->what()) { 242 case kWhatMediaPullerStopped: 243 { 244 mConverter.clear(); 245 246 mStarted = false; 247 248 sp<AMessage> notify = mNotify->dup(); 249 notify->setInt32("what", kWhatStopped); 250 notify->post(); 251 252 ALOGI("kWhatStopped %s posted", mIsAudio ? "audio" : "video"); 253 break; 254 } 255 256 default: 257 TRESPASS(); 258 } 259} 260 261void WifiDisplaySource::PlaybackSession::Track::queueAccessUnit( 262 const sp<ABuffer> &accessUnit) { 263 mQueuedAccessUnits.push_back(accessUnit); 264} 265 266sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueAccessUnit() { 267 if (mQueuedAccessUnits.empty()) { 268 return NULL; 269 } 270 271 sp<ABuffer> accessUnit = *mQueuedAccessUnits.begin(); 272 CHECK(accessUnit != NULL); 273 274 mQueuedAccessUnits.erase(mQueuedAccessUnits.begin()); 275 276 return accessUnit; 277} 278 279void WifiDisplaySource::PlaybackSession::Track::setRepeaterSource( 280 const sp<RepeaterSource> &source) { 281 mRepeaterSource = source; 282} 283 284void WifiDisplaySource::PlaybackSession::Track::requestIDRFrame() { 285 if (mIsAudio) { 286 return; 287 } 288 289 if (mRepeaterSource != NULL) { 290 mRepeaterSource->wakeUp(); 291 } 292 293 mConverter->requestIDRFrame(); 294} 295 296bool WifiDisplaySource::PlaybackSession::Track::hasOutputBuffer( 297 int64_t *timeUs) const { 298 *timeUs = 0ll; 299 300 if (mQueuedOutputBuffers.empty()) { 301 return false; 302 } 303 304 const sp<ABuffer> &outputBuffer = *mQueuedOutputBuffers.begin(); 305 306 CHECK(outputBuffer->meta()->findInt64("timeUs", timeUs)); 307 308 return true; 309} 310 311void WifiDisplaySource::PlaybackSession::Track::queueOutputBuffer( 312 const sp<ABuffer> &accessUnit) { 313 mQueuedOutputBuffers.push_back(accessUnit); 314 mLastOutputBufferQueuedTimeUs = ALooper::GetNowUs(); 315} 316 317sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueOutputBuffer() { 318 CHECK(!mQueuedOutputBuffers.empty()); 319 320 sp<ABuffer> outputBuffer = *mQueuedOutputBuffers.begin(); 321 mQueuedOutputBuffers.erase(mQueuedOutputBuffers.begin()); 322 323 return outputBuffer; 324} 325 326#if SUSPEND_VIDEO_IF_IDLE 327bool WifiDisplaySource::PlaybackSession::Track::isSuspended() const { 328 if (!mQueuedOutputBuffers.empty()) { 329 return false; 330 } 331 332 if (mLastOutputBufferQueuedTimeUs < 0ll) { 333 // We've never seen an output buffer queued, but tracks start 334 // out live, not suspended. 335 return false; 336 } 337 338 // If we've not seen new output data for 60ms or more, we consider 339 // this track suspended for the time being. 340 return (ALooper::GetNowUs() - mLastOutputBufferQueuedTimeUs) > 60000ll; 341} 342#endif 343 344//////////////////////////////////////////////////////////////////////////////// 345 346WifiDisplaySource::PlaybackSession::PlaybackSession( 347 const sp<ANetworkSession> &netSession, 348 const sp<AMessage> ¬ify, 349 const in_addr &interfaceAddr, 350 const sp<IHDCP> &hdcp, 351 const char *path) 352 : mNetSession(netSession), 353 mNotify(notify), 354 mInterfaceAddr(interfaceAddr), 355 mHDCP(hdcp), 356 mLocalRTPPort(-1), 357 mWeAreDead(false), 358 mPaused(false), 359 mLastLifesignUs(), 360 mVideoTrackIndex(-1), 361 mPrevTimeUs(-1ll), 362 mPullExtractorPending(false), 363 mPullExtractorGeneration(0), 364 mFirstSampleTimeRealUs(-1ll), 365 mFirstSampleTimeUs(-1ll) { 366 if (path != NULL) { 367 mMediaPath.setTo(path); 368 } 369} 370 371status_t WifiDisplaySource::PlaybackSession::init( 372 const char *clientIP, 373 int32_t clientRtp, 374 RTPSender::TransportMode rtpMode, 375 int32_t clientRtcp, 376 RTPSender::TransportMode rtcpMode, 377 bool enableAudio, 378 bool usePCMAudio, 379 bool enableVideo, 380 VideoFormats::ResolutionType videoResolutionType, 381 size_t videoResolutionIndex) { 382 sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, id()); 383 mMediaSender = new MediaSender(mNetSession, notify); 384 looper()->registerHandler(mMediaSender); 385 386 mMediaSender->setHDCP(mHDCP); 387 388 status_t err = setupPacketizer( 389 enableAudio, 390 usePCMAudio, 391 enableVideo, 392 videoResolutionType, 393 videoResolutionIndex); 394 395 if (err == OK) { 396 err = mMediaSender->initAsync( 397 -1 /* trackIndex */, 398 clientIP, 399 clientRtp, 400 rtpMode, 401 clientRtcp, 402 rtcpMode, 403 &mLocalRTPPort); 404 } 405 406 if (err != OK) { 407 mLocalRTPPort = -1; 408 409 looper()->unregisterHandler(mMediaSender->id()); 410 mMediaSender.clear(); 411 412 return err; 413 } 414 415 updateLiveness(); 416 417 return OK; 418} 419 420WifiDisplaySource::PlaybackSession::~PlaybackSession() { 421} 422 423int32_t WifiDisplaySource::PlaybackSession::getRTPPort() const { 424 return mLocalRTPPort; 425} 426 427int64_t WifiDisplaySource::PlaybackSession::getLastLifesignUs() const { 428 return mLastLifesignUs; 429} 430 431void WifiDisplaySource::PlaybackSession::updateLiveness() { 432 mLastLifesignUs = ALooper::GetNowUs(); 433} 434 435status_t WifiDisplaySource::PlaybackSession::play() { 436 updateLiveness(); 437 438 (new AMessage(kWhatResume, id()))->post(); 439 440 return OK; 441} 442 443status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() { 444 for (size_t i = 0; i < mTracks.size(); ++i) { 445 CHECK_EQ((status_t)OK, mTracks.editValueAt(i)->start()); 446 } 447 448 sp<AMessage> notify = mNotify->dup(); 449 notify->setInt32("what", kWhatSessionEstablished); 450 notify->post(); 451 452 return OK; 453} 454 455status_t WifiDisplaySource::PlaybackSession::pause() { 456 updateLiveness(); 457 458 (new AMessage(kWhatPause, id()))->post(); 459 460 return OK; 461} 462 463void WifiDisplaySource::PlaybackSession::destroyAsync() { 464 ALOGI("destroyAsync"); 465 466 for (size_t i = 0; i < mTracks.size(); ++i) { 467 mTracks.valueAt(i)->stopAsync(); 468 } 469} 470 471void WifiDisplaySource::PlaybackSession::onMessageReceived( 472 const sp<AMessage> &msg) { 473 switch (msg->what()) { 474 case kWhatConverterNotify: 475 { 476 if (mWeAreDead) { 477 ALOGV("dropping msg '%s' because we're dead", 478 msg->debugString().c_str()); 479 480 break; 481 } 482 483 int32_t what; 484 CHECK(msg->findInt32("what", &what)); 485 486 size_t trackIndex; 487 CHECK(msg->findSize("trackIndex", &trackIndex)); 488 489 if (what == Converter::kWhatAccessUnit) { 490 sp<ABuffer> accessUnit; 491 CHECK(msg->findBuffer("accessUnit", &accessUnit)); 492 493 const sp<Track> &track = mTracks.valueFor(trackIndex); 494 495 status_t err = mMediaSender->queueAccessUnit( 496 track->mediaSenderTrackIndex(), 497 accessUnit); 498 499 if (err != OK) { 500 notifySessionDead(); 501 } 502 break; 503 } else if (what == Converter::kWhatEOS) { 504 CHECK_EQ(what, Converter::kWhatEOS); 505 506 ALOGI("output EOS on track %d", trackIndex); 507 508 ssize_t index = mTracks.indexOfKey(trackIndex); 509 CHECK_GE(index, 0); 510 511 const sp<Converter> &converter = 512 mTracks.valueAt(index)->converter(); 513 looper()->unregisterHandler(converter->id()); 514 515 mTracks.removeItemsAt(index); 516 517 if (mTracks.isEmpty()) { 518 ALOGI("Reached EOS"); 519 } 520 } else { 521 CHECK_EQ(what, Converter::kWhatError); 522 523 status_t err; 524 CHECK(msg->findInt32("err", &err)); 525 526 ALOGE("converter signaled error %d", err); 527 528 notifySessionDead(); 529 } 530 break; 531 } 532 533 case kWhatMediaSenderNotify: 534 { 535 int32_t what; 536 CHECK(msg->findInt32("what", &what)); 537 538 if (what == MediaSender::kWhatInitDone) { 539 status_t err; 540 CHECK(msg->findInt32("err", &err)); 541 542 if (err == OK) { 543 onMediaSenderInitialized(); 544 } else { 545 notifySessionDead(); 546 } 547 } else if (what == MediaSender::kWhatError) { 548 notifySessionDead(); 549 } else if (what == MediaSender::kWhatNetworkStall) { 550 size_t numBytesQueued; 551 CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); 552 553 if (mVideoTrackIndex >= 0) { 554 const sp<Track> &videoTrack = 555 mTracks.valueFor(mVideoTrackIndex); 556 557 sp<Converter> converter = videoTrack->converter(); 558 if (converter != NULL) { 559 converter->dropAFrame(); 560 } 561 } 562 } else if (what == MediaSender::kWhatInformSender) { 563 onSinkFeedback(msg); 564 } else { 565 TRESPASS(); 566 } 567 break; 568 } 569 570 case kWhatTrackNotify: 571 { 572 int32_t what; 573 CHECK(msg->findInt32("what", &what)); 574 575 size_t trackIndex; 576 CHECK(msg->findSize("trackIndex", &trackIndex)); 577 578 if (what == Track::kWhatStopped) { 579 ALOGI("Track %d stopped", trackIndex); 580 581 sp<Track> track = mTracks.valueFor(trackIndex); 582 looper()->unregisterHandler(track->id()); 583 mTracks.removeItem(trackIndex); 584 track.clear(); 585 586 if (!mTracks.isEmpty()) { 587 ALOGI("not all tracks are stopped yet"); 588 break; 589 } 590 591 looper()->unregisterHandler(mMediaSender->id()); 592 mMediaSender.clear(); 593 594 sp<AMessage> notify = mNotify->dup(); 595 notify->setInt32("what", kWhatSessionDestroyed); 596 notify->post(); 597 } 598 break; 599 } 600 601 case kWhatPause: 602 { 603 if (mExtractor != NULL) { 604 ++mPullExtractorGeneration; 605 mFirstSampleTimeRealUs = -1ll; 606 mFirstSampleTimeUs = -1ll; 607 } 608 609 if (mPaused) { 610 break; 611 } 612 613 for (size_t i = 0; i < mTracks.size(); ++i) { 614 mTracks.editValueAt(i)->pause(); 615 } 616 617 mPaused = true; 618 break; 619 } 620 621 case kWhatResume: 622 { 623 if (mExtractor != NULL) { 624 schedulePullExtractor(); 625 } 626 627 if (!mPaused) { 628 break; 629 } 630 631 for (size_t i = 0; i < mTracks.size(); ++i) { 632 mTracks.editValueAt(i)->resume(); 633 } 634 635 mPaused = false; 636 break; 637 } 638 639 case kWhatPullExtractorSample: 640 { 641 int32_t generation; 642 CHECK(msg->findInt32("generation", &generation)); 643 644 if (generation != mPullExtractorGeneration) { 645 break; 646 } 647 648 mPullExtractorPending = false; 649 650 onPullExtractor(); 651 break; 652 } 653 654 default: 655 TRESPASS(); 656 } 657} 658 659void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) { 660 int64_t avgLatencyUs; 661 CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs)); 662 663 int64_t maxLatencyUs; 664 CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs)); 665 666 ALOGI("sink reports avg. latency of %lld ms (max %lld ms)", 667 avgLatencyUs / 1000ll, 668 maxLatencyUs / 1000ll); 669 670 if (mVideoTrackIndex >= 0) { 671 const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex); 672 sp<Converter> converter = videoTrack->converter(); 673 674 if (converter != NULL) { 675 int32_t videoBitrate = 676 Converter::GetInt32Property("media.wfd.video-bitrate", -1); 677 678 char val[PROPERTY_VALUE_MAX]; 679 if (videoBitrate < 0 680 && property_get("media.wfd.video-bitrate", val, NULL) 681 && !strcasecmp("adaptive", val)) { 682 videoBitrate = converter->getVideoBitrate(); 683 684 if (avgLatencyUs > 300000ll) { 685 videoBitrate *= 0.6; 686 } else if (avgLatencyUs < 100000ll) { 687 videoBitrate *= 1.1; 688 } 689 } 690 691 if (videoBitrate > 0) { 692 if (videoBitrate < 500000) { 693 videoBitrate = 500000; 694 } else if (videoBitrate > 10000000) { 695 videoBitrate = 10000000; 696 } 697 698 if (videoBitrate != converter->getVideoBitrate()) { 699 ALOGI("setting video bitrate to %d bps", videoBitrate); 700 701 converter->setVideoBitrate(videoBitrate); 702 } 703 } 704 } 705 706 sp<RepeaterSource> repeaterSource = videoTrack->repeaterSource(); 707 if (repeaterSource != NULL) { 708 double rateHz = 709 Converter::GetInt32Property( 710 "media.wfd.video-framerate", -1); 711 712 if (rateHz < 0.0) { 713 rateHz = repeaterSource->getFrameRate(); 714 715 if (avgLatencyUs > 300000ll) { 716 rateHz *= 0.9; 717 } else if (avgLatencyUs < 200000ll) { 718 rateHz *= 1.1; 719 } 720 } 721 722 if (rateHz > 0) { 723 if (rateHz < 5.0) { 724 rateHz = 5.0; 725 } else if (rateHz > 30.0) { 726 rateHz = 30.0; 727 } 728 729 if (rateHz != repeaterSource->getFrameRate()) { 730 ALOGI("setting frame rate to %.2f Hz", rateHz); 731 732 repeaterSource->setFrameRate(rateHz); 733 } 734 } 735 } 736 } 737} 738 739status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer( 740 bool enableAudio, bool enableVideo) { 741 DataSource::RegisterDefaultSniffers(); 742 743 mExtractor = new NuMediaExtractor; 744 745 status_t err = mExtractor->setDataSource(mMediaPath.c_str()); 746 747 if (err != OK) { 748 return err; 749 } 750 751 size_t n = mExtractor->countTracks(); 752 bool haveAudio = false; 753 bool haveVideo = false; 754 for (size_t i = 0; i < n; ++i) { 755 sp<AMessage> format; 756 err = mExtractor->getTrackFormat(i, &format); 757 758 if (err != OK) { 759 continue; 760 } 761 762 AString mime; 763 CHECK(format->findString("mime", &mime)); 764 765 bool isAudio = !strncasecmp(mime.c_str(), "audio/", 6); 766 bool isVideo = !strncasecmp(mime.c_str(), "video/", 6); 767 768 if (isAudio && enableAudio && !haveAudio) { 769 haveAudio = true; 770 } else if (isVideo && enableVideo && !haveVideo) { 771 haveVideo = true; 772 } else { 773 continue; 774 } 775 776 err = mExtractor->selectTrack(i); 777 778 size_t trackIndex = mTracks.size(); 779 780 sp<AMessage> notify = new AMessage(kWhatTrackNotify, id()); 781 notify->setSize("trackIndex", trackIndex); 782 783 sp<Track> track = new Track(notify, format); 784 looper()->registerHandler(track); 785 786 mTracks.add(trackIndex, track); 787 788 mExtractorTrackToInternalTrack.add(i, trackIndex); 789 790 if (isVideo) { 791 mVideoTrackIndex = trackIndex; 792 } 793 794 uint32_t flags = MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS; 795 796 ssize_t mediaSenderTrackIndex = 797 mMediaSender->addTrack(format, flags); 798 CHECK_GE(mediaSenderTrackIndex, 0); 799 800 track->setMediaSenderTrackIndex(mediaSenderTrackIndex); 801 802 if ((haveAudio || !enableAudio) && (haveVideo || !enableVideo)) { 803 break; 804 } 805 } 806 807 return OK; 808} 809 810void WifiDisplaySource::PlaybackSession::schedulePullExtractor() { 811 if (mPullExtractorPending) { 812 return; 813 } 814 815 int64_t sampleTimeUs; 816 status_t err = mExtractor->getSampleTime(&sampleTimeUs); 817 818 int64_t nowUs = ALooper::GetNowUs(); 819 820 if (mFirstSampleTimeRealUs < 0ll) { 821 mFirstSampleTimeRealUs = nowUs; 822 mFirstSampleTimeUs = sampleTimeUs; 823 } 824 825 int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs; 826 827 sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, id()); 828 msg->setInt32("generation", mPullExtractorGeneration); 829 msg->post(whenUs - nowUs); 830 831 mPullExtractorPending = true; 832} 833 834void WifiDisplaySource::PlaybackSession::onPullExtractor() { 835 sp<ABuffer> accessUnit = new ABuffer(1024 * 1024); 836 status_t err = mExtractor->readSampleData(accessUnit); 837 if (err != OK) { 838 // EOS. 839 return; 840 } 841 842 int64_t timeUs; 843 CHECK_EQ((status_t)OK, mExtractor->getSampleTime(&timeUs)); 844 845 accessUnit->meta()->setInt64( 846 "timeUs", mFirstSampleTimeRealUs + timeUs - mFirstSampleTimeUs); 847 848 size_t trackIndex; 849 CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex)); 850 851 sp<AMessage> msg = new AMessage(kWhatConverterNotify, id()); 852 853 msg->setSize( 854 "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex)); 855 856 msg->setInt32("what", Converter::kWhatAccessUnit); 857 msg->setBuffer("accessUnit", accessUnit); 858 msg->post(); 859 860 mExtractor->advance(); 861 862 schedulePullExtractor(); 863} 864 865status_t WifiDisplaySource::PlaybackSession::setupPacketizer( 866 bool enableAudio, 867 bool usePCMAudio, 868 bool enableVideo, 869 VideoFormats::ResolutionType videoResolutionType, 870 size_t videoResolutionIndex) { 871 CHECK(enableAudio || enableVideo); 872 873 if (!mMediaPath.empty()) { 874 return setupMediaPacketizer(enableAudio, enableVideo); 875 } 876 877 if (enableVideo) { 878 status_t err = addVideoSource( 879 videoResolutionType, videoResolutionIndex); 880 881 if (err != OK) { 882 return err; 883 } 884 } 885 886 if (!enableAudio) { 887 return OK; 888 } 889 890 return addAudioSource(usePCMAudio); 891} 892 893status_t WifiDisplaySource::PlaybackSession::addSource( 894 bool isVideo, const sp<MediaSource> &source, bool isRepeaterSource, 895 bool usePCMAudio, size_t *numInputBuffers) { 896 CHECK(!usePCMAudio || !isVideo); 897 CHECK(!isRepeaterSource || isVideo); 898 899 sp<ALooper> pullLooper = new ALooper; 900 pullLooper->setName("pull_looper"); 901 902 pullLooper->start( 903 false /* runOnCallingThread */, 904 false /* canCallJava */, 905 PRIORITY_AUDIO); 906 907 sp<ALooper> codecLooper = new ALooper; 908 codecLooper->setName("codec_looper"); 909 910 codecLooper->start( 911 false /* runOnCallingThread */, 912 false /* canCallJava */, 913 PRIORITY_AUDIO); 914 915 size_t trackIndex; 916 917 sp<AMessage> notify; 918 919 trackIndex = mTracks.size(); 920 921 sp<AMessage> format; 922 status_t err = convertMetaDataToMessage(source->getFormat(), &format); 923 CHECK_EQ(err, (status_t)OK); 924 925 if (isVideo) { 926 format->setInt32("store-metadata-in-buffers", true); 927 928 format->setInt32( 929 "color-format", OMX_COLOR_FormatAndroidOpaque); 930 } 931 932 notify = new AMessage(kWhatConverterNotify, id()); 933 notify->setSize("trackIndex", trackIndex); 934 935 sp<Converter> converter = 936 new Converter(notify, codecLooper, format, usePCMAudio); 937 938 err = converter->initCheck(); 939 if (err != OK) { 940 ALOGE("%s converter returned err %d", isVideo ? "video" : "audio", err); 941 return err; 942 } 943 944 looper()->registerHandler(converter); 945 946 notify = new AMessage(Converter::kWhatMediaPullerNotify, converter->id()); 947 notify->setSize("trackIndex", trackIndex); 948 949 sp<MediaPuller> puller = new MediaPuller(source, notify); 950 pullLooper->registerHandler(puller); 951 952 if (numInputBuffers != NULL) { 953 *numInputBuffers = converter->getInputBufferCount(); 954 } 955 956 notify = new AMessage(kWhatTrackNotify, id()); 957 notify->setSize("trackIndex", trackIndex); 958 959 sp<Track> track = new Track( 960 notify, pullLooper, codecLooper, puller, converter); 961 962 if (isRepeaterSource) { 963 track->setRepeaterSource(static_cast<RepeaterSource *>(source.get())); 964 } 965 966 looper()->registerHandler(track); 967 968 mTracks.add(trackIndex, track); 969 970 if (isVideo) { 971 mVideoTrackIndex = trackIndex; 972 } 973 974 uint32_t flags = 0; 975 if (converter->needToManuallyPrependSPSPPS()) { 976 flags |= MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS; 977 } 978 979 ssize_t mediaSenderTrackIndex = 980 mMediaSender->addTrack(converter->getOutputFormat(), flags); 981 CHECK_GE(mediaSenderTrackIndex, 0); 982 983 track->setMediaSenderTrackIndex(mediaSenderTrackIndex); 984 985 return OK; 986} 987 988status_t WifiDisplaySource::PlaybackSession::addVideoSource( 989 VideoFormats::ResolutionType videoResolutionType, 990 size_t videoResolutionIndex) { 991 size_t width, height, framesPerSecond; 992 bool interlaced; 993 CHECK(VideoFormats::GetConfiguration( 994 videoResolutionType, 995 videoResolutionIndex, 996 &width, 997 &height, 998 &framesPerSecond, 999 &interlaced)); 1000 1001 sp<SurfaceMediaSource> source = new SurfaceMediaSource(width, height); 1002 1003 source->setUseAbsoluteTimestamps(); 1004 1005 sp<RepeaterSource> videoSource = 1006 new RepeaterSource(source, framesPerSecond); 1007 1008 size_t numInputBuffers; 1009 status_t err = addSource( 1010 true /* isVideo */, videoSource, true /* isRepeaterSource */, 1011 false /* usePCMAudio */, &numInputBuffers); 1012 1013 if (err != OK) { 1014 return err; 1015 } 1016 1017 err = source->setMaxAcquiredBufferCount(numInputBuffers); 1018 CHECK_EQ(err, (status_t)OK); 1019 1020 mBufferQueue = source->getBufferQueue(); 1021 1022 return OK; 1023} 1024 1025status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) { 1026 sp<AudioSource> audioSource = new AudioSource( 1027 AUDIO_SOURCE_REMOTE_SUBMIX, 1028 48000 /* sampleRate */, 1029 2 /* channelCount */); 1030 1031 if (audioSource->initCheck() == OK) { 1032 return addSource( 1033 false /* isVideo */, audioSource, false /* isRepeaterSource */, 1034 usePCMAudio, NULL /* numInputBuffers */); 1035 } 1036 1037 ALOGW("Unable to instantiate audio source"); 1038 1039 return OK; 1040} 1041 1042sp<IGraphicBufferProducer> WifiDisplaySource::PlaybackSession::getSurfaceTexture() { 1043 return mBufferQueue; 1044} 1045 1046void WifiDisplaySource::PlaybackSession::requestIDRFrame() { 1047 for (size_t i = 0; i < mTracks.size(); ++i) { 1048 const sp<Track> &track = mTracks.valueAt(i); 1049 1050 track->requestIDRFrame(); 1051 } 1052} 1053 1054void WifiDisplaySource::PlaybackSession::notifySessionDead() { 1055 // Inform WifiDisplaySource of our premature death (wish). 1056 sp<AMessage> notify = mNotify->dup(); 1057 notify->setInt32("what", kWhatSessionDead); 1058 notify->post(); 1059 1060 mWeAreDead = true; 1061} 1062 1063} // namespace android 1064 1065