1/* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17//#define LOG_NDEBUG 0 18#define LOG_TAG "PlaybackSession" 19#include <utils/Log.h> 20 21#include "PlaybackSession.h" 22 23#include "Converter.h" 24#include "MediaPuller.h" 25#include "RepeaterSource.h" 26#include "include/avc_utils.h" 27#include "WifiDisplaySource.h" 28 29#include <binder/IServiceManager.h> 30#include <cutils/properties.h> 31#include <media/IHDCP.h> 32#include <media/IMediaHTTPService.h> 33#include <media/stagefright/foundation/ABitReader.h> 34#include <media/stagefright/foundation/ABuffer.h> 35#include <media/stagefright/foundation/ADebug.h> 36#include <media/stagefright/foundation/AMessage.h> 37#include <media/stagefright/foundation/hexdump.h> 38#include <media/stagefright/AudioSource.h> 39#include <media/stagefright/DataSource.h> 40#include <media/stagefright/MediaDefs.h> 41#include <media/stagefright/MediaErrors.h> 42#include <media/stagefright/MediaSource.h> 43#include <media/stagefright/MetaData.h> 44#include <media/stagefright/NuMediaExtractor.h> 45#include <media/stagefright/SurfaceMediaSource.h> 46#include <media/stagefright/Utils.h> 47 48#include <OMX_IVCommon.h> 49 50namespace android { 51 52struct WifiDisplaySource::PlaybackSession::Track : public AHandler { 53 enum { 54 kWhatStopped, 55 }; 56 57 Track(const sp<AMessage> ¬ify, 58 const sp<ALooper> &pullLooper, 59 const sp<ALooper> &codecLooper, 60 const sp<MediaPuller> &mediaPuller, 61 const sp<Converter> &converter); 62 63 Track(const sp<AMessage> ¬ify, const sp<AMessage> &format); 64 65 void setRepeaterSource(const sp<RepeaterSource> &source); 66 67 sp<AMessage> getFormat(); 68 bool isAudio() const; 69 70 const sp<Converter> &converter() const; 71 const sp<RepeaterSource> &repeaterSource() const; 72 73 ssize_t mediaSenderTrackIndex() const; 74 void setMediaSenderTrackIndex(size_t index); 75 76 status_t start(); 77 void stopAsync(); 78 79 void pause(); 80 void resume(); 81 82 void queueAccessUnit(const sp<ABuffer> &accessUnit); 83 sp<ABuffer> dequeueAccessUnit(); 84 85 bool hasOutputBuffer(int64_t *timeUs) const; 86 void queueOutputBuffer(const sp<ABuffer> &accessUnit); 87 sp<ABuffer> dequeueOutputBuffer(); 88 89#if SUSPEND_VIDEO_IF_IDLE 90 bool isSuspended() const; 91#endif 92 93 size_t countQueuedOutputBuffers() const { 94 return mQueuedOutputBuffers.size(); 95 } 96 97 void requestIDRFrame(); 98 99protected: 100 virtual void onMessageReceived(const sp<AMessage> &msg); 101 virtual ~Track(); 102 103private: 104 enum { 105 kWhatMediaPullerStopped, 106 }; 107 108 sp<AMessage> mNotify; 109 sp<ALooper> mPullLooper; 110 sp<ALooper> mCodecLooper; 111 sp<MediaPuller> mMediaPuller; 112 sp<Converter> mConverter; 113 sp<AMessage> mFormat; 114 bool mStarted; 115 ssize_t mMediaSenderTrackIndex; 116 bool mIsAudio; 117 List<sp<ABuffer> > mQueuedAccessUnits; 118 sp<RepeaterSource> mRepeaterSource; 119 List<sp<ABuffer> > mQueuedOutputBuffers; 120 int64_t mLastOutputBufferQueuedTimeUs; 121 122 static bool IsAudioFormat(const sp<AMessage> &format); 123 124 DISALLOW_EVIL_CONSTRUCTORS(Track); 125}; 126 127WifiDisplaySource::PlaybackSession::Track::Track( 128 const sp<AMessage> ¬ify, 129 const sp<ALooper> &pullLooper, 130 const sp<ALooper> &codecLooper, 131 const sp<MediaPuller> &mediaPuller, 132 const sp<Converter> &converter) 133 : mNotify(notify), 134 mPullLooper(pullLooper), 135 mCodecLooper(codecLooper), 136 mMediaPuller(mediaPuller), 137 mConverter(converter), 138 mStarted(false), 139 mIsAudio(IsAudioFormat(mConverter->getOutputFormat())), 140 mLastOutputBufferQueuedTimeUs(-1ll) { 141} 142 143WifiDisplaySource::PlaybackSession::Track::Track( 144 const sp<AMessage> ¬ify, const sp<AMessage> &format) 145 : mNotify(notify), 146 mFormat(format), 147 mStarted(false), 148 mIsAudio(IsAudioFormat(format)), 149 mLastOutputBufferQueuedTimeUs(-1ll) { 150} 151 152WifiDisplaySource::PlaybackSession::Track::~Track() { 153 CHECK(!mStarted); 154} 155 156// static 157bool WifiDisplaySource::PlaybackSession::Track::IsAudioFormat( 158 const sp<AMessage> &format) { 159 AString mime; 160 CHECK(format->findString("mime", &mime)); 161 162 return !strncasecmp(mime.c_str(), "audio/", 6); 163} 164 165sp<AMessage> WifiDisplaySource::PlaybackSession::Track::getFormat() { 166 return mFormat != NULL ? mFormat : mConverter->getOutputFormat(); 167} 168 169bool WifiDisplaySource::PlaybackSession::Track::isAudio() const { 170 return mIsAudio; 171} 172 173const sp<Converter> &WifiDisplaySource::PlaybackSession::Track::converter() const { 174 return mConverter; 175} 176 177const sp<RepeaterSource> & 178WifiDisplaySource::PlaybackSession::Track::repeaterSource() const { 179 return mRepeaterSource; 180} 181 182ssize_t WifiDisplaySource::PlaybackSession::Track::mediaSenderTrackIndex() const { 183 CHECK_GE(mMediaSenderTrackIndex, 0); 184 return mMediaSenderTrackIndex; 185} 186 187void WifiDisplaySource::PlaybackSession::Track::setMediaSenderTrackIndex( 188 size_t index) { 189 mMediaSenderTrackIndex = index; 190} 191 192status_t WifiDisplaySource::PlaybackSession::Track::start() { 193 ALOGV("Track::start isAudio=%d", mIsAudio); 194 195 CHECK(!mStarted); 196 197 status_t err = OK; 198 199 if (mMediaPuller != NULL) { 200 err = mMediaPuller->start(); 201 } 202 203 if (err == OK) { 204 mStarted = true; 205 } 206 207 return err; 208} 209 210void WifiDisplaySource::PlaybackSession::Track::stopAsync() { 211 ALOGV("Track::stopAsync isAudio=%d", mIsAudio); 212 213 if (mConverter != NULL) { 214 mConverter->shutdownAsync(); 215 } 216 217 sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, this); 218 219 if (mStarted && mMediaPuller != NULL) { 220 if (mRepeaterSource != NULL) { 221 // Let's unblock MediaPuller's MediaSource::read(). 222 mRepeaterSource->wakeUp(); 223 } 224 225 mMediaPuller->stopAsync(msg); 226 } else { 227 mStarted = false; 228 msg->post(); 229 } 230} 231 232void WifiDisplaySource::PlaybackSession::Track::pause() { 233 mMediaPuller->pause(); 234} 235 236void WifiDisplaySource::PlaybackSession::Track::resume() { 237 mMediaPuller->resume(); 238} 239 240void WifiDisplaySource::PlaybackSession::Track::onMessageReceived( 241 const sp<AMessage> &msg) { 242 switch (msg->what()) { 243 case kWhatMediaPullerStopped: 244 { 245 mConverter.clear(); 246 247 mStarted = false; 248 249 sp<AMessage> notify = mNotify->dup(); 250 notify->setInt32("what", kWhatStopped); 251 notify->post(); 252 253 ALOGI("kWhatStopped %s posted", mIsAudio ? "audio" : "video"); 254 break; 255 } 256 257 default: 258 TRESPASS(); 259 } 260} 261 262void WifiDisplaySource::PlaybackSession::Track::queueAccessUnit( 263 const sp<ABuffer> &accessUnit) { 264 mQueuedAccessUnits.push_back(accessUnit); 265} 266 267sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueAccessUnit() { 268 if (mQueuedAccessUnits.empty()) { 269 return NULL; 270 } 271 272 sp<ABuffer> accessUnit = *mQueuedAccessUnits.begin(); 273 CHECK(accessUnit != NULL); 274 275 mQueuedAccessUnits.erase(mQueuedAccessUnits.begin()); 276 277 return accessUnit; 278} 279 280void WifiDisplaySource::PlaybackSession::Track::setRepeaterSource( 281 const sp<RepeaterSource> &source) { 282 mRepeaterSource = source; 283} 284 285void WifiDisplaySource::PlaybackSession::Track::requestIDRFrame() { 286 if (mIsAudio) { 287 return; 288 } 289 290 if (mRepeaterSource != NULL) { 291 mRepeaterSource->wakeUp(); 292 } 293 294 mConverter->requestIDRFrame(); 295} 296 297bool WifiDisplaySource::PlaybackSession::Track::hasOutputBuffer( 298 int64_t *timeUs) const { 299 *timeUs = 0ll; 300 301 if (mQueuedOutputBuffers.empty()) { 302 return false; 303 } 304 305 const sp<ABuffer> &outputBuffer = *mQueuedOutputBuffers.begin(); 306 307 CHECK(outputBuffer->meta()->findInt64("timeUs", timeUs)); 308 309 return true; 310} 311 312void WifiDisplaySource::PlaybackSession::Track::queueOutputBuffer( 313 const sp<ABuffer> &accessUnit) { 314 mQueuedOutputBuffers.push_back(accessUnit); 315 mLastOutputBufferQueuedTimeUs = ALooper::GetNowUs(); 316} 317 318sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueOutputBuffer() { 319 CHECK(!mQueuedOutputBuffers.empty()); 320 321 sp<ABuffer> outputBuffer = *mQueuedOutputBuffers.begin(); 322 mQueuedOutputBuffers.erase(mQueuedOutputBuffers.begin()); 323 324 return outputBuffer; 325} 326 327#if SUSPEND_VIDEO_IF_IDLE 328bool WifiDisplaySource::PlaybackSession::Track::isSuspended() const { 329 if (!mQueuedOutputBuffers.empty()) { 330 return false; 331 } 332 333 if (mLastOutputBufferQueuedTimeUs < 0ll) { 334 // We've never seen an output buffer queued, but tracks start 335 // out live, not suspended. 336 return false; 337 } 338 339 // If we've not seen new output data for 60ms or more, we consider 340 // this track suspended for the time being. 341 return (ALooper::GetNowUs() - mLastOutputBufferQueuedTimeUs) > 60000ll; 342} 343#endif 344 345//////////////////////////////////////////////////////////////////////////////// 346 347WifiDisplaySource::PlaybackSession::PlaybackSession( 348 const String16 &opPackageName, 349 const sp<ANetworkSession> &netSession, 350 const sp<AMessage> ¬ify, 351 const in_addr &interfaceAddr, 352 const sp<IHDCP> &hdcp, 353 const char *path) 354 : mOpPackageName(opPackageName), 355 mNetSession(netSession), 356 mNotify(notify), 357 mInterfaceAddr(interfaceAddr), 358 mHDCP(hdcp), 359 mLocalRTPPort(-1), 360 mWeAreDead(false), 361 mPaused(false), 362 mLastLifesignUs(), 363 mVideoTrackIndex(-1), 364 mPrevTimeUs(-1ll), 365 mPullExtractorPending(false), 366 mPullExtractorGeneration(0), 367 mFirstSampleTimeRealUs(-1ll), 368 mFirstSampleTimeUs(-1ll) { 369 if (path != NULL) { 370 mMediaPath.setTo(path); 371 } 372} 373 374status_t WifiDisplaySource::PlaybackSession::init( 375 const char *clientIP, 376 int32_t clientRtp, 377 RTPSender::TransportMode rtpMode, 378 int32_t clientRtcp, 379 RTPSender::TransportMode rtcpMode, 380 bool enableAudio, 381 bool usePCMAudio, 382 bool enableVideo, 383 VideoFormats::ResolutionType videoResolutionType, 384 size_t videoResolutionIndex, 385 VideoFormats::ProfileType videoProfileType, 386 VideoFormats::LevelType videoLevelType) { 387 sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, this); 388 mMediaSender = new MediaSender(mNetSession, notify); 389 looper()->registerHandler(mMediaSender); 390 391 mMediaSender->setHDCP(mHDCP); 392 393 status_t err = setupPacketizer( 394 enableAudio, 395 usePCMAudio, 396 enableVideo, 397 videoResolutionType, 398 videoResolutionIndex, 399 videoProfileType, 400 videoLevelType); 401 402 if (err == OK) { 403 err = mMediaSender->initAsync( 404 -1 /* trackIndex */, 405 clientIP, 406 clientRtp, 407 rtpMode, 408 clientRtcp, 409 rtcpMode, 410 &mLocalRTPPort); 411 } 412 413 if (err != OK) { 414 mLocalRTPPort = -1; 415 416 looper()->unregisterHandler(mMediaSender->id()); 417 mMediaSender.clear(); 418 419 return err; 420 } 421 422 updateLiveness(); 423 424 return OK; 425} 426 427WifiDisplaySource::PlaybackSession::~PlaybackSession() { 428} 429 430int32_t WifiDisplaySource::PlaybackSession::getRTPPort() const { 431 return mLocalRTPPort; 432} 433 434int64_t WifiDisplaySource::PlaybackSession::getLastLifesignUs() const { 435 return mLastLifesignUs; 436} 437 438void WifiDisplaySource::PlaybackSession::updateLiveness() { 439 mLastLifesignUs = ALooper::GetNowUs(); 440} 441 442status_t WifiDisplaySource::PlaybackSession::play() { 443 updateLiveness(); 444 445 (new AMessage(kWhatResume, this))->post(); 446 447 return OK; 448} 449 450status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() { 451 for (size_t i = 0; i < mTracks.size(); ++i) { 452 CHECK_EQ((status_t)OK, mTracks.editValueAt(i)->start()); 453 } 454 455 sp<AMessage> notify = mNotify->dup(); 456 notify->setInt32("what", kWhatSessionEstablished); 457 notify->post(); 458 459 return OK; 460} 461 462status_t WifiDisplaySource::PlaybackSession::pause() { 463 updateLiveness(); 464 465 (new AMessage(kWhatPause, this))->post(); 466 467 return OK; 468} 469 470void WifiDisplaySource::PlaybackSession::destroyAsync() { 471 ALOGI("destroyAsync"); 472 473 for (size_t i = 0; i < mTracks.size(); ++i) { 474 mTracks.valueAt(i)->stopAsync(); 475 } 476} 477 478void WifiDisplaySource::PlaybackSession::onMessageReceived( 479 const sp<AMessage> &msg) { 480 switch (msg->what()) { 481 case kWhatConverterNotify: 482 { 483 if (mWeAreDead) { 484 ALOGV("dropping msg '%s' because we're dead", 485 msg->debugString().c_str()); 486 487 break; 488 } 489 490 int32_t what; 491 CHECK(msg->findInt32("what", &what)); 492 493 size_t trackIndex; 494 CHECK(msg->findSize("trackIndex", &trackIndex)); 495 496 if (what == Converter::kWhatAccessUnit) { 497 sp<ABuffer> accessUnit; 498 CHECK(msg->findBuffer("accessUnit", &accessUnit)); 499 500 const sp<Track> &track = mTracks.valueFor(trackIndex); 501 502 status_t err = mMediaSender->queueAccessUnit( 503 track->mediaSenderTrackIndex(), 504 accessUnit); 505 506 if (err != OK) { 507 notifySessionDead(); 508 } 509 break; 510 } else if (what == Converter::kWhatEOS) { 511 CHECK_EQ(what, Converter::kWhatEOS); 512 513 ALOGI("output EOS on track %zu", trackIndex); 514 515 ssize_t index = mTracks.indexOfKey(trackIndex); 516 CHECK_GE(index, 0); 517 518 const sp<Converter> &converter = 519 mTracks.valueAt(index)->converter(); 520 looper()->unregisterHandler(converter->id()); 521 522 mTracks.removeItemsAt(index); 523 524 if (mTracks.isEmpty()) { 525 ALOGI("Reached EOS"); 526 } 527 } else if (what != Converter::kWhatShutdownCompleted) { 528 CHECK_EQ(what, Converter::kWhatError); 529 530 status_t err; 531 CHECK(msg->findInt32("err", &err)); 532 533 ALOGE("converter signaled error %d", err); 534 535 notifySessionDead(); 536 } 537 break; 538 } 539 540 case kWhatMediaSenderNotify: 541 { 542 int32_t what; 543 CHECK(msg->findInt32("what", &what)); 544 545 if (what == MediaSender::kWhatInitDone) { 546 status_t err; 547 CHECK(msg->findInt32("err", &err)); 548 549 if (err == OK) { 550 onMediaSenderInitialized(); 551 } else { 552 notifySessionDead(); 553 } 554 } else if (what == MediaSender::kWhatError) { 555 notifySessionDead(); 556 } else if (what == MediaSender::kWhatNetworkStall) { 557 size_t numBytesQueued; 558 CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); 559 560 if (mVideoTrackIndex >= 0) { 561 const sp<Track> &videoTrack = 562 mTracks.valueFor(mVideoTrackIndex); 563 564 sp<Converter> converter = videoTrack->converter(); 565 if (converter != NULL) { 566 converter->dropAFrame(); 567 } 568 } 569 } else if (what == MediaSender::kWhatInformSender) { 570 onSinkFeedback(msg); 571 } else { 572 TRESPASS(); 573 } 574 break; 575 } 576 577 case kWhatTrackNotify: 578 { 579 int32_t what; 580 CHECK(msg->findInt32("what", &what)); 581 582 size_t trackIndex; 583 CHECK(msg->findSize("trackIndex", &trackIndex)); 584 585 if (what == Track::kWhatStopped) { 586 ALOGI("Track %zu stopped", trackIndex); 587 588 sp<Track> track = mTracks.valueFor(trackIndex); 589 looper()->unregisterHandler(track->id()); 590 mTracks.removeItem(trackIndex); 591 track.clear(); 592 593 if (!mTracks.isEmpty()) { 594 ALOGI("not all tracks are stopped yet"); 595 break; 596 } 597 598 looper()->unregisterHandler(mMediaSender->id()); 599 mMediaSender.clear(); 600 601 sp<AMessage> notify = mNotify->dup(); 602 notify->setInt32("what", kWhatSessionDestroyed); 603 notify->post(); 604 } 605 break; 606 } 607 608 case kWhatPause: 609 { 610 if (mExtractor != NULL) { 611 ++mPullExtractorGeneration; 612 mFirstSampleTimeRealUs = -1ll; 613 mFirstSampleTimeUs = -1ll; 614 } 615 616 if (mPaused) { 617 break; 618 } 619 620 for (size_t i = 0; i < mTracks.size(); ++i) { 621 mTracks.editValueAt(i)->pause(); 622 } 623 624 mPaused = true; 625 break; 626 } 627 628 case kWhatResume: 629 { 630 if (mExtractor != NULL) { 631 schedulePullExtractor(); 632 } 633 634 if (!mPaused) { 635 break; 636 } 637 638 for (size_t i = 0; i < mTracks.size(); ++i) { 639 mTracks.editValueAt(i)->resume(); 640 } 641 642 mPaused = false; 643 break; 644 } 645 646 case kWhatPullExtractorSample: 647 { 648 int32_t generation; 649 CHECK(msg->findInt32("generation", &generation)); 650 651 if (generation != mPullExtractorGeneration) { 652 break; 653 } 654 655 mPullExtractorPending = false; 656 657 onPullExtractor(); 658 break; 659 } 660 661 default: 662 TRESPASS(); 663 } 664} 665 666void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) { 667 int64_t avgLatencyUs; 668 CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs)); 669 670 int64_t maxLatencyUs; 671 CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs)); 672 673 ALOGI("sink reports avg. latency of %lld ms (max %lld ms)", 674 avgLatencyUs / 1000ll, 675 maxLatencyUs / 1000ll); 676 677 if (mVideoTrackIndex >= 0) { 678 const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex); 679 sp<Converter> converter = videoTrack->converter(); 680 681 if (converter != NULL) { 682 int32_t videoBitrate = 683 Converter::GetInt32Property("media.wfd.video-bitrate", -1); 684 685 char val[PROPERTY_VALUE_MAX]; 686 if (videoBitrate < 0 687 && property_get("media.wfd.video-bitrate", val, NULL) 688 && !strcasecmp("adaptive", val)) { 689 videoBitrate = converter->getVideoBitrate(); 690 691 if (avgLatencyUs > 300000ll) { 692 videoBitrate *= 0.6; 693 } else if (avgLatencyUs < 100000ll) { 694 videoBitrate *= 1.1; 695 } 696 } 697 698 if (videoBitrate > 0) { 699 if (videoBitrate < 500000) { 700 videoBitrate = 500000; 701 } else if (videoBitrate > 10000000) { 702 videoBitrate = 10000000; 703 } 704 705 if (videoBitrate != converter->getVideoBitrate()) { 706 ALOGI("setting video bitrate to %d bps", videoBitrate); 707 708 converter->setVideoBitrate(videoBitrate); 709 } 710 } 711 } 712 713 sp<RepeaterSource> repeaterSource = videoTrack->repeaterSource(); 714 if (repeaterSource != NULL) { 715 double rateHz = 716 Converter::GetInt32Property( 717 "media.wfd.video-framerate", -1); 718 719 char val[PROPERTY_VALUE_MAX]; 720 if (rateHz < 0.0 721 && property_get("media.wfd.video-framerate", val, NULL) 722 && !strcasecmp("adaptive", val)) { 723 rateHz = repeaterSource->getFrameRate(); 724 725 if (avgLatencyUs > 300000ll) { 726 rateHz *= 0.9; 727 } else if (avgLatencyUs < 200000ll) { 728 rateHz *= 1.1; 729 } 730 } 731 732 if (rateHz > 0) { 733 if (rateHz < 5.0) { 734 rateHz = 5.0; 735 } else if (rateHz > 30.0) { 736 rateHz = 30.0; 737 } 738 739 if (rateHz != repeaterSource->getFrameRate()) { 740 ALOGI("setting frame rate to %.2f Hz", rateHz); 741 742 repeaterSource->setFrameRate(rateHz); 743 } 744 } 745 } 746 } 747} 748 749status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer( 750 bool enableAudio, bool enableVideo) { 751 DataSource::RegisterDefaultSniffers(); 752 753 mExtractor = new NuMediaExtractor; 754 755 status_t err = mExtractor->setDataSource( 756 NULL /* httpService */, mMediaPath.c_str()); 757 758 if (err != OK) { 759 return err; 760 } 761 762 size_t n = mExtractor->countTracks(); 763 bool haveAudio = false; 764 bool haveVideo = false; 765 for (size_t i = 0; i < n; ++i) { 766 sp<AMessage> format; 767 err = mExtractor->getTrackFormat(i, &format); 768 769 if (err != OK) { 770 continue; 771 } 772 773 AString mime; 774 CHECK(format->findString("mime", &mime)); 775 776 bool isAudio = !strncasecmp(mime.c_str(), "audio/", 6); 777 bool isVideo = !strncasecmp(mime.c_str(), "video/", 6); 778 779 if (isAudio && enableAudio && !haveAudio) { 780 haveAudio = true; 781 } else if (isVideo && enableVideo && !haveVideo) { 782 haveVideo = true; 783 } else { 784 continue; 785 } 786 787 err = mExtractor->selectTrack(i); 788 789 size_t trackIndex = mTracks.size(); 790 791 sp<AMessage> notify = new AMessage(kWhatTrackNotify, this); 792 notify->setSize("trackIndex", trackIndex); 793 794 sp<Track> track = new Track(notify, format); 795 looper()->registerHandler(track); 796 797 mTracks.add(trackIndex, track); 798 799 mExtractorTrackToInternalTrack.add(i, trackIndex); 800 801 if (isVideo) { 802 mVideoTrackIndex = trackIndex; 803 } 804 805 uint32_t flags = MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS; 806 807 ssize_t mediaSenderTrackIndex = 808 mMediaSender->addTrack(format, flags); 809 CHECK_GE(mediaSenderTrackIndex, 0); 810 811 track->setMediaSenderTrackIndex(mediaSenderTrackIndex); 812 813 if ((haveAudio || !enableAudio) && (haveVideo || !enableVideo)) { 814 break; 815 } 816 } 817 818 return OK; 819} 820 821void WifiDisplaySource::PlaybackSession::schedulePullExtractor() { 822 if (mPullExtractorPending) { 823 return; 824 } 825 826 int64_t delayUs = 1000000; // default delay is 1 sec 827 int64_t sampleTimeUs; 828 status_t err = mExtractor->getSampleTime(&sampleTimeUs); 829 830 if (err == OK) { 831 int64_t nowUs = ALooper::GetNowUs(); 832 833 if (mFirstSampleTimeRealUs < 0ll) { 834 mFirstSampleTimeRealUs = nowUs; 835 mFirstSampleTimeUs = sampleTimeUs; 836 } 837 838 int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs; 839 delayUs = whenUs - nowUs; 840 } else { 841 ALOGW("could not get sample time (%d)", err); 842 } 843 844 sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, this); 845 msg->setInt32("generation", mPullExtractorGeneration); 846 msg->post(delayUs); 847 848 mPullExtractorPending = true; 849} 850 851void WifiDisplaySource::PlaybackSession::onPullExtractor() { 852 sp<ABuffer> accessUnit = new ABuffer(1024 * 1024); 853 status_t err = mExtractor->readSampleData(accessUnit); 854 if (err != OK) { 855 // EOS. 856 return; 857 } 858 859 int64_t timeUs; 860 CHECK_EQ((status_t)OK, mExtractor->getSampleTime(&timeUs)); 861 862 accessUnit->meta()->setInt64( 863 "timeUs", mFirstSampleTimeRealUs + timeUs - mFirstSampleTimeUs); 864 865 size_t trackIndex; 866 CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex)); 867 868 sp<AMessage> msg = new AMessage(kWhatConverterNotify, this); 869 870 msg->setSize( 871 "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex)); 872 873 msg->setInt32("what", Converter::kWhatAccessUnit); 874 msg->setBuffer("accessUnit", accessUnit); 875 msg->post(); 876 877 mExtractor->advance(); 878 879 schedulePullExtractor(); 880} 881 882status_t WifiDisplaySource::PlaybackSession::setupPacketizer( 883 bool enableAudio, 884 bool usePCMAudio, 885 bool enableVideo, 886 VideoFormats::ResolutionType videoResolutionType, 887 size_t videoResolutionIndex, 888 VideoFormats::ProfileType videoProfileType, 889 VideoFormats::LevelType videoLevelType) { 890 CHECK(enableAudio || enableVideo); 891 892 if (!mMediaPath.empty()) { 893 return setupMediaPacketizer(enableAudio, enableVideo); 894 } 895 896 if (enableVideo) { 897 status_t err = addVideoSource( 898 videoResolutionType, videoResolutionIndex, videoProfileType, 899 videoLevelType); 900 901 if (err != OK) { 902 return err; 903 } 904 } 905 906 if (!enableAudio) { 907 return OK; 908 } 909 910 return addAudioSource(usePCMAudio); 911} 912 913status_t WifiDisplaySource::PlaybackSession::addSource( 914 bool isVideo, const sp<MediaSource> &source, bool isRepeaterSource, 915 bool usePCMAudio, unsigned profileIdc, unsigned levelIdc, 916 unsigned constraintSet, size_t *numInputBuffers) { 917 CHECK(!usePCMAudio || !isVideo); 918 CHECK(!isRepeaterSource || isVideo); 919 CHECK(!profileIdc || isVideo); 920 CHECK(!levelIdc || isVideo); 921 CHECK(!constraintSet || isVideo); 922 923 sp<ALooper> pullLooper = new ALooper; 924 pullLooper->setName("pull_looper"); 925 926 pullLooper->start( 927 false /* runOnCallingThread */, 928 false /* canCallJava */, 929 PRIORITY_AUDIO); 930 931 sp<ALooper> codecLooper = new ALooper; 932 codecLooper->setName("codec_looper"); 933 934 codecLooper->start( 935 false /* runOnCallingThread */, 936 false /* canCallJava */, 937 PRIORITY_AUDIO); 938 939 size_t trackIndex; 940 941 sp<AMessage> notify; 942 943 trackIndex = mTracks.size(); 944 945 sp<AMessage> format; 946 status_t err = convertMetaDataToMessage(source->getFormat(), &format); 947 CHECK_EQ(err, (status_t)OK); 948 949 if (isVideo) { 950 format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC); 951 format->setInt32( 952 "android._input-metadata-buffer-type", kMetadataBufferTypeANWBuffer); 953 format->setInt32("android._store-metadata-in-buffers-output", (mHDCP != NULL) 954 && (mHDCP->getCaps() & HDCPModule::HDCP_CAPS_ENCRYPT_NATIVE)); 955 format->setInt32( 956 "color-format", OMX_COLOR_FormatAndroidOpaque); 957 format->setInt32("profile-idc", profileIdc); 958 format->setInt32("level-idc", levelIdc); 959 format->setInt32("constraint-set", constraintSet); 960 } else { 961 if (usePCMAudio) { 962 format->setInt32("pcm-encoding", kAudioEncodingPcm16bit); 963 format->setString("mime", MEDIA_MIMETYPE_AUDIO_RAW); 964 } else { 965 format->setString("mime", MEDIA_MIMETYPE_AUDIO_AAC); 966 } 967 } 968 969 notify = new AMessage(kWhatConverterNotify, this); 970 notify->setSize("trackIndex", trackIndex); 971 972 sp<Converter> converter = new Converter(notify, codecLooper, format); 973 974 looper()->registerHandler(converter); 975 976 err = converter->init(); 977 if (err != OK) { 978 ALOGE("%s converter returned err %d", isVideo ? "video" : "audio", err); 979 980 looper()->unregisterHandler(converter->id()); 981 return err; 982 } 983 984 notify = new AMessage(Converter::kWhatMediaPullerNotify, converter); 985 notify->setSize("trackIndex", trackIndex); 986 987 sp<MediaPuller> puller = new MediaPuller(source, notify); 988 pullLooper->registerHandler(puller); 989 990 if (numInputBuffers != NULL) { 991 *numInputBuffers = converter->getInputBufferCount(); 992 } 993 994 notify = new AMessage(kWhatTrackNotify, this); 995 notify->setSize("trackIndex", trackIndex); 996 997 sp<Track> track = new Track( 998 notify, pullLooper, codecLooper, puller, converter); 999 1000 if (isRepeaterSource) { 1001 track->setRepeaterSource(static_cast<RepeaterSource *>(source.get())); 1002 } 1003 1004 looper()->registerHandler(track); 1005 1006 mTracks.add(trackIndex, track); 1007 1008 if (isVideo) { 1009 mVideoTrackIndex = trackIndex; 1010 } 1011 1012 uint32_t flags = 0; 1013 if (converter->needToManuallyPrependSPSPPS()) { 1014 flags |= MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS; 1015 } 1016 1017 ssize_t mediaSenderTrackIndex = 1018 mMediaSender->addTrack(converter->getOutputFormat(), flags); 1019 CHECK_GE(mediaSenderTrackIndex, 0); 1020 1021 track->setMediaSenderTrackIndex(mediaSenderTrackIndex); 1022 1023 return OK; 1024} 1025 1026status_t WifiDisplaySource::PlaybackSession::addVideoSource( 1027 VideoFormats::ResolutionType videoResolutionType, 1028 size_t videoResolutionIndex, 1029 VideoFormats::ProfileType videoProfileType, 1030 VideoFormats::LevelType videoLevelType) { 1031 size_t width, height, framesPerSecond; 1032 bool interlaced; 1033 CHECK(VideoFormats::GetConfiguration( 1034 videoResolutionType, 1035 videoResolutionIndex, 1036 &width, 1037 &height, 1038 &framesPerSecond, 1039 &interlaced)); 1040 1041 unsigned profileIdc, levelIdc, constraintSet; 1042 CHECK(VideoFormats::GetProfileLevel( 1043 videoProfileType, 1044 videoLevelType, 1045 &profileIdc, 1046 &levelIdc, 1047 &constraintSet)); 1048 1049 sp<SurfaceMediaSource> source = new SurfaceMediaSource(width, height); 1050 1051 source->setUseAbsoluteTimestamps(); 1052 1053 sp<RepeaterSource> videoSource = 1054 new RepeaterSource(source, framesPerSecond); 1055 1056 size_t numInputBuffers; 1057 status_t err = addSource( 1058 true /* isVideo */, videoSource, true /* isRepeaterSource */, 1059 false /* usePCMAudio */, profileIdc, levelIdc, constraintSet, 1060 &numInputBuffers); 1061 1062 if (err != OK) { 1063 return err; 1064 } 1065 1066 err = source->setMaxAcquiredBufferCount(numInputBuffers); 1067 CHECK_EQ(err, (status_t)OK); 1068 1069 mProducer = source->getProducer(); 1070 1071 return OK; 1072} 1073 1074status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) { 1075 sp<AudioSource> audioSource = new AudioSource( 1076 AUDIO_SOURCE_REMOTE_SUBMIX, 1077 mOpPackageName, 1078 48000 /* sampleRate */, 1079 2 /* channelCount */); 1080 1081 if (audioSource->initCheck() == OK) { 1082 return addSource( 1083 false /* isVideo */, audioSource, false /* isRepeaterSource */, 1084 usePCMAudio, 0 /* profileIdc */, 0 /* levelIdc */, 1085 0 /* constraintSet */, NULL /* numInputBuffers */); 1086 } 1087 1088 ALOGW("Unable to instantiate audio source"); 1089 1090 return OK; 1091} 1092 1093sp<IGraphicBufferProducer> WifiDisplaySource::PlaybackSession::getSurfaceTexture() { 1094 return mProducer; 1095} 1096 1097void WifiDisplaySource::PlaybackSession::requestIDRFrame() { 1098 for (size_t i = 0; i < mTracks.size(); ++i) { 1099 const sp<Track> &track = mTracks.valueAt(i); 1100 1101 track->requestIDRFrame(); 1102 } 1103} 1104 1105void WifiDisplaySource::PlaybackSession::notifySessionDead() { 1106 // Inform WifiDisplaySource of our premature death (wish). 1107 sp<AMessage> notify = mNotify->dup(); 1108 notify->setInt32("what", kWhatSessionDead); 1109 notify->post(); 1110 1111 mWeAreDead = true; 1112} 1113 1114} // namespace android 1115 1116