1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "PlaybackSession"
19#include <utils/Log.h>
20
21#include "PlaybackSession.h"
22
23#include "Converter.h"
24#include "MediaPuller.h"
25#include "RepeaterSource.h"
26#include "include/avc_utils.h"
27#include "WifiDisplaySource.h"
28
29#include <binder/IServiceManager.h>
30#include <cutils/properties.h>
31#include <media/IHDCP.h>
32#include <media/stagefright/foundation/ABitReader.h>
33#include <media/stagefright/foundation/ABuffer.h>
34#include <media/stagefright/foundation/ADebug.h>
35#include <media/stagefright/foundation/AMessage.h>
36#include <media/stagefright/foundation/hexdump.h>
37#include <media/stagefright/AudioSource.h>
38#include <media/stagefright/DataSource.h>
39#include <media/stagefright/MediaDefs.h>
40#include <media/stagefright/MediaErrors.h>
41#include <media/stagefright/MediaSource.h>
42#include <media/stagefright/MetaData.h>
43#include <media/stagefright/NuMediaExtractor.h>
44#include <media/stagefright/SurfaceMediaSource.h>
45#include <media/stagefright/Utils.h>
46
47#include <OMX_IVCommon.h>
48
49namespace android {
50
51struct WifiDisplaySource::PlaybackSession::Track : public AHandler {
52    enum {
53        kWhatStopped,
54    };
55
56    Track(const sp<AMessage> &notify,
57          const sp<ALooper> &pullLooper,
58          const sp<ALooper> &codecLooper,
59          const sp<MediaPuller> &mediaPuller,
60          const sp<Converter> &converter);
61
62    Track(const sp<AMessage> &notify, const sp<AMessage> &format);
63
64    void setRepeaterSource(const sp<RepeaterSource> &source);
65
66    sp<AMessage> getFormat();
67    bool isAudio() const;
68
69    const sp<Converter> &converter() const;
70    const sp<RepeaterSource> &repeaterSource() const;
71
72    ssize_t mediaSenderTrackIndex() const;
73    void setMediaSenderTrackIndex(size_t index);
74
75    status_t start();
76    void stopAsync();
77
78    void pause();
79    void resume();
80
81    void queueAccessUnit(const sp<ABuffer> &accessUnit);
82    sp<ABuffer> dequeueAccessUnit();
83
84    bool hasOutputBuffer(int64_t *timeUs) const;
85    void queueOutputBuffer(const sp<ABuffer> &accessUnit);
86    sp<ABuffer> dequeueOutputBuffer();
87
88#if SUSPEND_VIDEO_IF_IDLE
89    bool isSuspended() const;
90#endif
91
92    size_t countQueuedOutputBuffers() const {
93        return mQueuedOutputBuffers.size();
94    }
95
96    void requestIDRFrame();
97
98protected:
99    virtual void onMessageReceived(const sp<AMessage> &msg);
100    virtual ~Track();
101
102private:
103    enum {
104        kWhatMediaPullerStopped,
105    };
106
107    sp<AMessage> mNotify;
108    sp<ALooper> mPullLooper;
109    sp<ALooper> mCodecLooper;
110    sp<MediaPuller> mMediaPuller;
111    sp<Converter> mConverter;
112    sp<AMessage> mFormat;
113    bool mStarted;
114    ssize_t mMediaSenderTrackIndex;
115    bool mIsAudio;
116    List<sp<ABuffer> > mQueuedAccessUnits;
117    sp<RepeaterSource> mRepeaterSource;
118    List<sp<ABuffer> > mQueuedOutputBuffers;
119    int64_t mLastOutputBufferQueuedTimeUs;
120
121    static bool IsAudioFormat(const sp<AMessage> &format);
122
123    DISALLOW_EVIL_CONSTRUCTORS(Track);
124};
125
126WifiDisplaySource::PlaybackSession::Track::Track(
127        const sp<AMessage> &notify,
128        const sp<ALooper> &pullLooper,
129        const sp<ALooper> &codecLooper,
130        const sp<MediaPuller> &mediaPuller,
131        const sp<Converter> &converter)
132    : mNotify(notify),
133      mPullLooper(pullLooper),
134      mCodecLooper(codecLooper),
135      mMediaPuller(mediaPuller),
136      mConverter(converter),
137      mStarted(false),
138      mIsAudio(IsAudioFormat(mConverter->getOutputFormat())),
139      mLastOutputBufferQueuedTimeUs(-1ll) {
140}
141
142WifiDisplaySource::PlaybackSession::Track::Track(
143        const sp<AMessage> &notify, const sp<AMessage> &format)
144    : mNotify(notify),
145      mFormat(format),
146      mStarted(false),
147      mIsAudio(IsAudioFormat(format)),
148      mLastOutputBufferQueuedTimeUs(-1ll) {
149}
150
151WifiDisplaySource::PlaybackSession::Track::~Track() {
152    CHECK(!mStarted);
153}
154
155// static
156bool WifiDisplaySource::PlaybackSession::Track::IsAudioFormat(
157        const sp<AMessage> &format) {
158    AString mime;
159    CHECK(format->findString("mime", &mime));
160
161    return !strncasecmp(mime.c_str(), "audio/", 6);
162}
163
164sp<AMessage> WifiDisplaySource::PlaybackSession::Track::getFormat() {
165    return mFormat != NULL ? mFormat : mConverter->getOutputFormat();
166}
167
168bool WifiDisplaySource::PlaybackSession::Track::isAudio() const {
169    return mIsAudio;
170}
171
172const sp<Converter> &WifiDisplaySource::PlaybackSession::Track::converter() const {
173    return mConverter;
174}
175
176const sp<RepeaterSource> &
177WifiDisplaySource::PlaybackSession::Track::repeaterSource() const {
178    return mRepeaterSource;
179}
180
181ssize_t WifiDisplaySource::PlaybackSession::Track::mediaSenderTrackIndex() const {
182    CHECK_GE(mMediaSenderTrackIndex, 0);
183    return mMediaSenderTrackIndex;
184}
185
186void WifiDisplaySource::PlaybackSession::Track::setMediaSenderTrackIndex(
187        size_t index) {
188    mMediaSenderTrackIndex = index;
189}
190
191status_t WifiDisplaySource::PlaybackSession::Track::start() {
192    ALOGV("Track::start isAudio=%d", mIsAudio);
193
194    CHECK(!mStarted);
195
196    status_t err = OK;
197
198    if (mMediaPuller != NULL) {
199        err = mMediaPuller->start();
200    }
201
202    if (err == OK) {
203        mStarted = true;
204    }
205
206    return err;
207}
208
209void WifiDisplaySource::PlaybackSession::Track::stopAsync() {
210    ALOGV("Track::stopAsync isAudio=%d", mIsAudio);
211
212    if (mConverter != NULL) {
213        mConverter->shutdownAsync();
214    }
215
216    sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, id());
217
218    if (mStarted && mMediaPuller != NULL) {
219        if (mRepeaterSource != NULL) {
220            // Let's unblock MediaPuller's MediaSource::read().
221            mRepeaterSource->wakeUp();
222        }
223
224        mMediaPuller->stopAsync(msg);
225    } else {
226        mStarted = false;
227        msg->post();
228    }
229}
230
231void WifiDisplaySource::PlaybackSession::Track::pause() {
232    mMediaPuller->pause();
233}
234
235void WifiDisplaySource::PlaybackSession::Track::resume() {
236    mMediaPuller->resume();
237}
238
239void WifiDisplaySource::PlaybackSession::Track::onMessageReceived(
240        const sp<AMessage> &msg) {
241    switch (msg->what()) {
242        case kWhatMediaPullerStopped:
243        {
244            mConverter.clear();
245
246            mStarted = false;
247
248            sp<AMessage> notify = mNotify->dup();
249            notify->setInt32("what", kWhatStopped);
250            notify->post();
251
252            ALOGI("kWhatStopped %s posted", mIsAudio ? "audio" : "video");
253            break;
254        }
255
256        default:
257            TRESPASS();
258    }
259}
260
261void WifiDisplaySource::PlaybackSession::Track::queueAccessUnit(
262        const sp<ABuffer> &accessUnit) {
263    mQueuedAccessUnits.push_back(accessUnit);
264}
265
266sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueAccessUnit() {
267    if (mQueuedAccessUnits.empty()) {
268        return NULL;
269    }
270
271    sp<ABuffer> accessUnit = *mQueuedAccessUnits.begin();
272    CHECK(accessUnit != NULL);
273
274    mQueuedAccessUnits.erase(mQueuedAccessUnits.begin());
275
276    return accessUnit;
277}
278
279void WifiDisplaySource::PlaybackSession::Track::setRepeaterSource(
280        const sp<RepeaterSource> &source) {
281    mRepeaterSource = source;
282}
283
284void WifiDisplaySource::PlaybackSession::Track::requestIDRFrame() {
285    if (mIsAudio) {
286        return;
287    }
288
289    if (mRepeaterSource != NULL) {
290        mRepeaterSource->wakeUp();
291    }
292
293    mConverter->requestIDRFrame();
294}
295
296bool WifiDisplaySource::PlaybackSession::Track::hasOutputBuffer(
297        int64_t *timeUs) const {
298    *timeUs = 0ll;
299
300    if (mQueuedOutputBuffers.empty()) {
301        return false;
302    }
303
304    const sp<ABuffer> &outputBuffer = *mQueuedOutputBuffers.begin();
305
306    CHECK(outputBuffer->meta()->findInt64("timeUs", timeUs));
307
308    return true;
309}
310
311void WifiDisplaySource::PlaybackSession::Track::queueOutputBuffer(
312        const sp<ABuffer> &accessUnit) {
313    mQueuedOutputBuffers.push_back(accessUnit);
314    mLastOutputBufferQueuedTimeUs = ALooper::GetNowUs();
315}
316
317sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueOutputBuffer() {
318    CHECK(!mQueuedOutputBuffers.empty());
319
320    sp<ABuffer> outputBuffer = *mQueuedOutputBuffers.begin();
321    mQueuedOutputBuffers.erase(mQueuedOutputBuffers.begin());
322
323    return outputBuffer;
324}
325
326#if SUSPEND_VIDEO_IF_IDLE
327bool WifiDisplaySource::PlaybackSession::Track::isSuspended() const {
328    if (!mQueuedOutputBuffers.empty()) {
329        return false;
330    }
331
332    if (mLastOutputBufferQueuedTimeUs < 0ll) {
333        // We've never seen an output buffer queued, but tracks start
334        // out live, not suspended.
335        return false;
336    }
337
338    // If we've not seen new output data for 60ms or more, we consider
339    // this track suspended for the time being.
340    return (ALooper::GetNowUs() - mLastOutputBufferQueuedTimeUs) > 60000ll;
341}
342#endif
343
344////////////////////////////////////////////////////////////////////////////////
345
346WifiDisplaySource::PlaybackSession::PlaybackSession(
347        const sp<ANetworkSession> &netSession,
348        const sp<AMessage> &notify,
349        const in_addr &interfaceAddr,
350        const sp<IHDCP> &hdcp,
351        const char *path)
352    : mNetSession(netSession),
353      mNotify(notify),
354      mInterfaceAddr(interfaceAddr),
355      mHDCP(hdcp),
356      mLocalRTPPort(-1),
357      mWeAreDead(false),
358      mPaused(false),
359      mLastLifesignUs(),
360      mVideoTrackIndex(-1),
361      mPrevTimeUs(-1ll),
362      mPullExtractorPending(false),
363      mPullExtractorGeneration(0),
364      mFirstSampleTimeRealUs(-1ll),
365      mFirstSampleTimeUs(-1ll) {
366    if (path != NULL) {
367        mMediaPath.setTo(path);
368    }
369}
370
371status_t WifiDisplaySource::PlaybackSession::init(
372        const char *clientIP,
373        int32_t clientRtp,
374        RTPSender::TransportMode rtpMode,
375        int32_t clientRtcp,
376        RTPSender::TransportMode rtcpMode,
377        bool enableAudio,
378        bool usePCMAudio,
379        bool enableVideo,
380        VideoFormats::ResolutionType videoResolutionType,
381        size_t videoResolutionIndex,
382        VideoFormats::ProfileType videoProfileType,
383        VideoFormats::LevelType videoLevelType) {
384    sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, id());
385    mMediaSender = new MediaSender(mNetSession, notify);
386    looper()->registerHandler(mMediaSender);
387
388    mMediaSender->setHDCP(mHDCP);
389
390    status_t err = setupPacketizer(
391            enableAudio,
392            usePCMAudio,
393            enableVideo,
394            videoResolutionType,
395            videoResolutionIndex,
396            videoProfileType,
397            videoLevelType);
398
399    if (err == OK) {
400        err = mMediaSender->initAsync(
401                -1 /* trackIndex */,
402                clientIP,
403                clientRtp,
404                rtpMode,
405                clientRtcp,
406                rtcpMode,
407                &mLocalRTPPort);
408    }
409
410    if (err != OK) {
411        mLocalRTPPort = -1;
412
413        looper()->unregisterHandler(mMediaSender->id());
414        mMediaSender.clear();
415
416        return err;
417    }
418
419    updateLiveness();
420
421    return OK;
422}
423
424WifiDisplaySource::PlaybackSession::~PlaybackSession() {
425}
426
427int32_t WifiDisplaySource::PlaybackSession::getRTPPort() const {
428    return mLocalRTPPort;
429}
430
431int64_t WifiDisplaySource::PlaybackSession::getLastLifesignUs() const {
432    return mLastLifesignUs;
433}
434
435void WifiDisplaySource::PlaybackSession::updateLiveness() {
436    mLastLifesignUs = ALooper::GetNowUs();
437}
438
439status_t WifiDisplaySource::PlaybackSession::play() {
440    updateLiveness();
441
442    (new AMessage(kWhatResume, id()))->post();
443
444    return OK;
445}
446
447status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() {
448    for (size_t i = 0; i < mTracks.size(); ++i) {
449        CHECK_EQ((status_t)OK, mTracks.editValueAt(i)->start());
450    }
451
452    sp<AMessage> notify = mNotify->dup();
453    notify->setInt32("what", kWhatSessionEstablished);
454    notify->post();
455
456    return OK;
457}
458
459status_t WifiDisplaySource::PlaybackSession::pause() {
460    updateLiveness();
461
462    (new AMessage(kWhatPause, id()))->post();
463
464    return OK;
465}
466
467void WifiDisplaySource::PlaybackSession::destroyAsync() {
468    ALOGI("destroyAsync");
469
470    for (size_t i = 0; i < mTracks.size(); ++i) {
471        mTracks.valueAt(i)->stopAsync();
472    }
473}
474
475void WifiDisplaySource::PlaybackSession::onMessageReceived(
476        const sp<AMessage> &msg) {
477    switch (msg->what()) {
478        case kWhatConverterNotify:
479        {
480            if (mWeAreDead) {
481                ALOGV("dropping msg '%s' because we're dead",
482                      msg->debugString().c_str());
483
484                break;
485            }
486
487            int32_t what;
488            CHECK(msg->findInt32("what", &what));
489
490            size_t trackIndex;
491            CHECK(msg->findSize("trackIndex", &trackIndex));
492
493            if (what == Converter::kWhatAccessUnit) {
494                sp<ABuffer> accessUnit;
495                CHECK(msg->findBuffer("accessUnit", &accessUnit));
496
497                const sp<Track> &track = mTracks.valueFor(trackIndex);
498
499                status_t err = mMediaSender->queueAccessUnit(
500                        track->mediaSenderTrackIndex(),
501                        accessUnit);
502
503                if (err != OK) {
504                    notifySessionDead();
505                }
506                break;
507            } else if (what == Converter::kWhatEOS) {
508                CHECK_EQ(what, Converter::kWhatEOS);
509
510                ALOGI("output EOS on track %d", trackIndex);
511
512                ssize_t index = mTracks.indexOfKey(trackIndex);
513                CHECK_GE(index, 0);
514
515                const sp<Converter> &converter =
516                    mTracks.valueAt(index)->converter();
517                looper()->unregisterHandler(converter->id());
518
519                mTracks.removeItemsAt(index);
520
521                if (mTracks.isEmpty()) {
522                    ALOGI("Reached EOS");
523                }
524            } else if (what != Converter::kWhatShutdownCompleted) {
525                CHECK_EQ(what, Converter::kWhatError);
526
527                status_t err;
528                CHECK(msg->findInt32("err", &err));
529
530                ALOGE("converter signaled error %d", err);
531
532                notifySessionDead();
533            }
534            break;
535        }
536
537        case kWhatMediaSenderNotify:
538        {
539            int32_t what;
540            CHECK(msg->findInt32("what", &what));
541
542            if (what == MediaSender::kWhatInitDone) {
543                status_t err;
544                CHECK(msg->findInt32("err", &err));
545
546                if (err == OK) {
547                    onMediaSenderInitialized();
548                } else {
549                    notifySessionDead();
550                }
551            } else if (what == MediaSender::kWhatError) {
552                notifySessionDead();
553            } else if (what == MediaSender::kWhatNetworkStall) {
554                size_t numBytesQueued;
555                CHECK(msg->findSize("numBytesQueued", &numBytesQueued));
556
557                if (mVideoTrackIndex >= 0) {
558                    const sp<Track> &videoTrack =
559                        mTracks.valueFor(mVideoTrackIndex);
560
561                    sp<Converter> converter = videoTrack->converter();
562                    if (converter != NULL) {
563                        converter->dropAFrame();
564                    }
565                }
566            } else if (what == MediaSender::kWhatInformSender) {
567                onSinkFeedback(msg);
568            } else {
569                TRESPASS();
570            }
571            break;
572        }
573
574        case kWhatTrackNotify:
575        {
576            int32_t what;
577            CHECK(msg->findInt32("what", &what));
578
579            size_t trackIndex;
580            CHECK(msg->findSize("trackIndex", &trackIndex));
581
582            if (what == Track::kWhatStopped) {
583                ALOGI("Track %d stopped", trackIndex);
584
585                sp<Track> track = mTracks.valueFor(trackIndex);
586                looper()->unregisterHandler(track->id());
587                mTracks.removeItem(trackIndex);
588                track.clear();
589
590                if (!mTracks.isEmpty()) {
591                    ALOGI("not all tracks are stopped yet");
592                    break;
593                }
594
595                looper()->unregisterHandler(mMediaSender->id());
596                mMediaSender.clear();
597
598                sp<AMessage> notify = mNotify->dup();
599                notify->setInt32("what", kWhatSessionDestroyed);
600                notify->post();
601            }
602            break;
603        }
604
605        case kWhatPause:
606        {
607            if (mExtractor != NULL) {
608                ++mPullExtractorGeneration;
609                mFirstSampleTimeRealUs = -1ll;
610                mFirstSampleTimeUs = -1ll;
611            }
612
613            if (mPaused) {
614                break;
615            }
616
617            for (size_t i = 0; i < mTracks.size(); ++i) {
618                mTracks.editValueAt(i)->pause();
619            }
620
621            mPaused = true;
622            break;
623        }
624
625        case kWhatResume:
626        {
627            if (mExtractor != NULL) {
628                schedulePullExtractor();
629            }
630
631            if (!mPaused) {
632                break;
633            }
634
635            for (size_t i = 0; i < mTracks.size(); ++i) {
636                mTracks.editValueAt(i)->resume();
637            }
638
639            mPaused = false;
640            break;
641        }
642
643        case kWhatPullExtractorSample:
644        {
645            int32_t generation;
646            CHECK(msg->findInt32("generation", &generation));
647
648            if (generation != mPullExtractorGeneration) {
649                break;
650            }
651
652            mPullExtractorPending = false;
653
654            onPullExtractor();
655            break;
656        }
657
658        default:
659            TRESPASS();
660    }
661}
662
663void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) {
664    int64_t avgLatencyUs;
665    CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs));
666
667    int64_t maxLatencyUs;
668    CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs));
669
670    ALOGI("sink reports avg. latency of %lld ms (max %lld ms)",
671          avgLatencyUs / 1000ll,
672          maxLatencyUs / 1000ll);
673
674    if (mVideoTrackIndex >= 0) {
675        const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex);
676        sp<Converter> converter = videoTrack->converter();
677
678        if (converter != NULL) {
679            int32_t videoBitrate =
680                Converter::GetInt32Property("media.wfd.video-bitrate", -1);
681
682            char val[PROPERTY_VALUE_MAX];
683            if (videoBitrate < 0
684                    && property_get("media.wfd.video-bitrate", val, NULL)
685                    && !strcasecmp("adaptive", val)) {
686                videoBitrate = converter->getVideoBitrate();
687
688                if (avgLatencyUs > 300000ll) {
689                    videoBitrate *= 0.6;
690                } else if (avgLatencyUs < 100000ll) {
691                    videoBitrate *= 1.1;
692                }
693            }
694
695            if (videoBitrate > 0) {
696                if (videoBitrate < 500000) {
697                    videoBitrate = 500000;
698                } else if (videoBitrate > 10000000) {
699                    videoBitrate = 10000000;
700                }
701
702                if (videoBitrate != converter->getVideoBitrate()) {
703                    ALOGI("setting video bitrate to %d bps", videoBitrate);
704
705                    converter->setVideoBitrate(videoBitrate);
706                }
707            }
708        }
709
710        sp<RepeaterSource> repeaterSource = videoTrack->repeaterSource();
711        if (repeaterSource != NULL) {
712            double rateHz =
713                Converter::GetInt32Property(
714                        "media.wfd.video-framerate", -1);
715
716            char val[PROPERTY_VALUE_MAX];
717            if (rateHz < 0.0
718                    && property_get("media.wfd.video-framerate", val, NULL)
719                    && !strcasecmp("adaptive", val)) {
720                 rateHz = repeaterSource->getFrameRate();
721
722                if (avgLatencyUs > 300000ll) {
723                    rateHz *= 0.9;
724                } else if (avgLatencyUs < 200000ll) {
725                    rateHz *= 1.1;
726                }
727            }
728
729            if (rateHz > 0) {
730                if (rateHz < 5.0) {
731                    rateHz = 5.0;
732                } else if (rateHz > 30.0) {
733                    rateHz = 30.0;
734                }
735
736                if (rateHz != repeaterSource->getFrameRate()) {
737                    ALOGI("setting frame rate to %.2f Hz", rateHz);
738
739                    repeaterSource->setFrameRate(rateHz);
740                }
741            }
742        }
743    }
744}
745
746status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer(
747        bool enableAudio, bool enableVideo) {
748    DataSource::RegisterDefaultSniffers();
749
750    mExtractor = new NuMediaExtractor;
751
752    status_t err = mExtractor->setDataSource(mMediaPath.c_str());
753
754    if (err != OK) {
755        return err;
756    }
757
758    size_t n = mExtractor->countTracks();
759    bool haveAudio = false;
760    bool haveVideo = false;
761    for (size_t i = 0; i < n; ++i) {
762        sp<AMessage> format;
763        err = mExtractor->getTrackFormat(i, &format);
764
765        if (err != OK) {
766            continue;
767        }
768
769        AString mime;
770        CHECK(format->findString("mime", &mime));
771
772        bool isAudio = !strncasecmp(mime.c_str(), "audio/", 6);
773        bool isVideo = !strncasecmp(mime.c_str(), "video/", 6);
774
775        if (isAudio && enableAudio && !haveAudio) {
776            haveAudio = true;
777        } else if (isVideo && enableVideo && !haveVideo) {
778            haveVideo = true;
779        } else {
780            continue;
781        }
782
783        err = mExtractor->selectTrack(i);
784
785        size_t trackIndex = mTracks.size();
786
787        sp<AMessage> notify = new AMessage(kWhatTrackNotify, id());
788        notify->setSize("trackIndex", trackIndex);
789
790        sp<Track> track = new Track(notify, format);
791        looper()->registerHandler(track);
792
793        mTracks.add(trackIndex, track);
794
795        mExtractorTrackToInternalTrack.add(i, trackIndex);
796
797        if (isVideo) {
798            mVideoTrackIndex = trackIndex;
799        }
800
801        uint32_t flags = MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
802
803        ssize_t mediaSenderTrackIndex =
804            mMediaSender->addTrack(format, flags);
805        CHECK_GE(mediaSenderTrackIndex, 0);
806
807        track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
808
809        if ((haveAudio || !enableAudio) && (haveVideo || !enableVideo)) {
810            break;
811        }
812    }
813
814    return OK;
815}
816
817void WifiDisplaySource::PlaybackSession::schedulePullExtractor() {
818    if (mPullExtractorPending) {
819        return;
820    }
821
822    int64_t sampleTimeUs;
823    status_t err = mExtractor->getSampleTime(&sampleTimeUs);
824
825    int64_t nowUs = ALooper::GetNowUs();
826
827    if (mFirstSampleTimeRealUs < 0ll) {
828        mFirstSampleTimeRealUs = nowUs;
829        mFirstSampleTimeUs = sampleTimeUs;
830    }
831
832    int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs;
833
834    sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, id());
835    msg->setInt32("generation", mPullExtractorGeneration);
836    msg->post(whenUs - nowUs);
837
838    mPullExtractorPending = true;
839}
840
841void WifiDisplaySource::PlaybackSession::onPullExtractor() {
842    sp<ABuffer> accessUnit = new ABuffer(1024 * 1024);
843    status_t err = mExtractor->readSampleData(accessUnit);
844    if (err != OK) {
845        // EOS.
846        return;
847    }
848
849    int64_t timeUs;
850    CHECK_EQ((status_t)OK, mExtractor->getSampleTime(&timeUs));
851
852    accessUnit->meta()->setInt64(
853            "timeUs", mFirstSampleTimeRealUs + timeUs - mFirstSampleTimeUs);
854
855    size_t trackIndex;
856    CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex));
857
858    sp<AMessage> msg = new AMessage(kWhatConverterNotify, id());
859
860    msg->setSize(
861            "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex));
862
863    msg->setInt32("what", Converter::kWhatAccessUnit);
864    msg->setBuffer("accessUnit", accessUnit);
865    msg->post();
866
867    mExtractor->advance();
868
869    schedulePullExtractor();
870}
871
872status_t WifiDisplaySource::PlaybackSession::setupPacketizer(
873        bool enableAudio,
874        bool usePCMAudio,
875        bool enableVideo,
876        VideoFormats::ResolutionType videoResolutionType,
877        size_t videoResolutionIndex,
878        VideoFormats::ProfileType videoProfileType,
879        VideoFormats::LevelType videoLevelType) {
880    CHECK(enableAudio || enableVideo);
881
882    if (!mMediaPath.empty()) {
883        return setupMediaPacketizer(enableAudio, enableVideo);
884    }
885
886    if (enableVideo) {
887        status_t err = addVideoSource(
888                videoResolutionType, videoResolutionIndex, videoProfileType,
889                videoLevelType);
890
891        if (err != OK) {
892            return err;
893        }
894    }
895
896    if (!enableAudio) {
897        return OK;
898    }
899
900    return addAudioSource(usePCMAudio);
901}
902
903status_t WifiDisplaySource::PlaybackSession::addSource(
904        bool isVideo, const sp<MediaSource> &source, bool isRepeaterSource,
905        bool usePCMAudio, unsigned profileIdc, unsigned levelIdc,
906        unsigned constraintSet, size_t *numInputBuffers) {
907    CHECK(!usePCMAudio || !isVideo);
908    CHECK(!isRepeaterSource || isVideo);
909    CHECK(!profileIdc || isVideo);
910    CHECK(!levelIdc || isVideo);
911    CHECK(!constraintSet || isVideo);
912
913    sp<ALooper> pullLooper = new ALooper;
914    pullLooper->setName("pull_looper");
915
916    pullLooper->start(
917            false /* runOnCallingThread */,
918            false /* canCallJava */,
919            PRIORITY_AUDIO);
920
921    sp<ALooper> codecLooper = new ALooper;
922    codecLooper->setName("codec_looper");
923
924    codecLooper->start(
925            false /* runOnCallingThread */,
926            false /* canCallJava */,
927            PRIORITY_AUDIO);
928
929    size_t trackIndex;
930
931    sp<AMessage> notify;
932
933    trackIndex = mTracks.size();
934
935    sp<AMessage> format;
936    status_t err = convertMetaDataToMessage(source->getFormat(), &format);
937    CHECK_EQ(err, (status_t)OK);
938
939    if (isVideo) {
940        format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC);
941        format->setInt32("store-metadata-in-buffers", true);
942        format->setInt32("store-metadata-in-buffers-output", (mHDCP != NULL)
943                && (mHDCP->getCaps() & HDCPModule::HDCP_CAPS_ENCRYPT_NATIVE));
944        format->setInt32(
945                "color-format", OMX_COLOR_FormatAndroidOpaque);
946        format->setInt32("profile-idc", profileIdc);
947        format->setInt32("level-idc", levelIdc);
948        format->setInt32("constraint-set", constraintSet);
949    } else {
950        format->setString(
951                "mime",
952                usePCMAudio
953                    ? MEDIA_MIMETYPE_AUDIO_RAW : MEDIA_MIMETYPE_AUDIO_AAC);
954    }
955
956    notify = new AMessage(kWhatConverterNotify, id());
957    notify->setSize("trackIndex", trackIndex);
958
959    sp<Converter> converter = new Converter(notify, codecLooper, format);
960
961    looper()->registerHandler(converter);
962
963    err = converter->init();
964    if (err != OK) {
965        ALOGE("%s converter returned err %d", isVideo ? "video" : "audio", err);
966
967        looper()->unregisterHandler(converter->id());
968        return err;
969    }
970
971    notify = new AMessage(Converter::kWhatMediaPullerNotify, converter->id());
972    notify->setSize("trackIndex", trackIndex);
973
974    sp<MediaPuller> puller = new MediaPuller(source, notify);
975    pullLooper->registerHandler(puller);
976
977    if (numInputBuffers != NULL) {
978        *numInputBuffers = converter->getInputBufferCount();
979    }
980
981    notify = new AMessage(kWhatTrackNotify, id());
982    notify->setSize("trackIndex", trackIndex);
983
984    sp<Track> track = new Track(
985            notify, pullLooper, codecLooper, puller, converter);
986
987    if (isRepeaterSource) {
988        track->setRepeaterSource(static_cast<RepeaterSource *>(source.get()));
989    }
990
991    looper()->registerHandler(track);
992
993    mTracks.add(trackIndex, track);
994
995    if (isVideo) {
996        mVideoTrackIndex = trackIndex;
997    }
998
999    uint32_t flags = 0;
1000    if (converter->needToManuallyPrependSPSPPS()) {
1001        flags |= MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
1002    }
1003
1004    ssize_t mediaSenderTrackIndex =
1005        mMediaSender->addTrack(converter->getOutputFormat(), flags);
1006    CHECK_GE(mediaSenderTrackIndex, 0);
1007
1008    track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
1009
1010    return OK;
1011}
1012
1013status_t WifiDisplaySource::PlaybackSession::addVideoSource(
1014        VideoFormats::ResolutionType videoResolutionType,
1015        size_t videoResolutionIndex,
1016        VideoFormats::ProfileType videoProfileType,
1017        VideoFormats::LevelType videoLevelType) {
1018    size_t width, height, framesPerSecond;
1019    bool interlaced;
1020    CHECK(VideoFormats::GetConfiguration(
1021                videoResolutionType,
1022                videoResolutionIndex,
1023                &width,
1024                &height,
1025                &framesPerSecond,
1026                &interlaced));
1027
1028    unsigned profileIdc, levelIdc, constraintSet;
1029    CHECK(VideoFormats::GetProfileLevel(
1030                videoProfileType,
1031                videoLevelType,
1032                &profileIdc,
1033                &levelIdc,
1034                &constraintSet));
1035
1036    sp<SurfaceMediaSource> source = new SurfaceMediaSource(width, height);
1037
1038    source->setUseAbsoluteTimestamps();
1039
1040    sp<RepeaterSource> videoSource =
1041        new RepeaterSource(source, framesPerSecond);
1042
1043    size_t numInputBuffers;
1044    status_t err = addSource(
1045            true /* isVideo */, videoSource, true /* isRepeaterSource */,
1046            false /* usePCMAudio */, profileIdc, levelIdc, constraintSet,
1047            &numInputBuffers);
1048
1049    if (err != OK) {
1050        return err;
1051    }
1052
1053    err = source->setMaxAcquiredBufferCount(numInputBuffers);
1054    CHECK_EQ(err, (status_t)OK);
1055
1056    mBufferQueue = source->getBufferQueue();
1057
1058    return OK;
1059}
1060
1061status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) {
1062    sp<AudioSource> audioSource = new AudioSource(
1063            AUDIO_SOURCE_REMOTE_SUBMIX,
1064            48000 /* sampleRate */,
1065            2 /* channelCount */);
1066
1067    if (audioSource->initCheck() == OK) {
1068        return addSource(
1069                false /* isVideo */, audioSource, false /* isRepeaterSource */,
1070                usePCMAudio, 0 /* profileIdc */, 0 /* levelIdc */,
1071                0 /* constraintSet */, NULL /* numInputBuffers */);
1072    }
1073
1074    ALOGW("Unable to instantiate audio source");
1075
1076    return OK;
1077}
1078
1079sp<IGraphicBufferProducer> WifiDisplaySource::PlaybackSession::getSurfaceTexture() {
1080    return mBufferQueue;
1081}
1082
1083void WifiDisplaySource::PlaybackSession::requestIDRFrame() {
1084    for (size_t i = 0; i < mTracks.size(); ++i) {
1085        const sp<Track> &track = mTracks.valueAt(i);
1086
1087        track->requestIDRFrame();
1088    }
1089}
1090
1091void WifiDisplaySource::PlaybackSession::notifySessionDead() {
1092    // Inform WifiDisplaySource of our premature death (wish).
1093    sp<AMessage> notify = mNotify->dup();
1094    notify->setInt32("what", kWhatSessionDead);
1095    notify->post();
1096
1097    mWeAreDead = true;
1098}
1099
1100}  // namespace android
1101
1102