1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "PlaybackSession"
19#include <utils/Log.h>
20
21#include "PlaybackSession.h"
22
23#include "Converter.h"
24#include "MediaPuller.h"
25#include "RepeaterSource.h"
26#include "include/avc_utils.h"
27#include "WifiDisplaySource.h"
28
29#include <binder/IServiceManager.h>
30#include <cutils/properties.h>
31#include <media/IHDCP.h>
32#include <media/IMediaHTTPService.h>
33#include <media/stagefright/foundation/ABitReader.h>
34#include <media/stagefright/foundation/ABuffer.h>
35#include <media/stagefright/foundation/ADebug.h>
36#include <media/stagefright/foundation/AMessage.h>
37#include <media/stagefright/foundation/hexdump.h>
38#include <media/stagefright/AudioSource.h>
39#include <media/stagefright/DataSource.h>
40#include <media/stagefright/MediaDefs.h>
41#include <media/stagefright/MediaErrors.h>
42#include <media/stagefright/MediaSource.h>
43#include <media/stagefright/MetaData.h>
44#include <media/stagefright/NuMediaExtractor.h>
45#include <media/stagefright/SurfaceMediaSource.h>
46#include <media/stagefright/Utils.h>
47
48#include <OMX_IVCommon.h>
49
50namespace android {
51
52struct WifiDisplaySource::PlaybackSession::Track : public AHandler {
53    enum {
54        kWhatStopped,
55    };
56
57    Track(const sp<AMessage> &notify,
58          const sp<ALooper> &pullLooper,
59          const sp<ALooper> &codecLooper,
60          const sp<MediaPuller> &mediaPuller,
61          const sp<Converter> &converter);
62
63    Track(const sp<AMessage> &notify, const sp<AMessage> &format);
64
65    void setRepeaterSource(const sp<RepeaterSource> &source);
66
67    sp<AMessage> getFormat();
68    bool isAudio() const;
69
70    const sp<Converter> &converter() const;
71    const sp<RepeaterSource> &repeaterSource() const;
72
73    ssize_t mediaSenderTrackIndex() const;
74    void setMediaSenderTrackIndex(size_t index);
75
76    status_t start();
77    void stopAsync();
78
79    void pause();
80    void resume();
81
82    void queueAccessUnit(const sp<ABuffer> &accessUnit);
83    sp<ABuffer> dequeueAccessUnit();
84
85    bool hasOutputBuffer(int64_t *timeUs) const;
86    void queueOutputBuffer(const sp<ABuffer> &accessUnit);
87    sp<ABuffer> dequeueOutputBuffer();
88
89#if SUSPEND_VIDEO_IF_IDLE
90    bool isSuspended() const;
91#endif
92
93    size_t countQueuedOutputBuffers() const {
94        return mQueuedOutputBuffers.size();
95    }
96
97    void requestIDRFrame();
98
99protected:
100    virtual void onMessageReceived(const sp<AMessage> &msg);
101    virtual ~Track();
102
103private:
104    enum {
105        kWhatMediaPullerStopped,
106    };
107
108    sp<AMessage> mNotify;
109    sp<ALooper> mPullLooper;
110    sp<ALooper> mCodecLooper;
111    sp<MediaPuller> mMediaPuller;
112    sp<Converter> mConverter;
113    sp<AMessage> mFormat;
114    bool mStarted;
115    ssize_t mMediaSenderTrackIndex;
116    bool mIsAudio;
117    List<sp<ABuffer> > mQueuedAccessUnits;
118    sp<RepeaterSource> mRepeaterSource;
119    List<sp<ABuffer> > mQueuedOutputBuffers;
120    int64_t mLastOutputBufferQueuedTimeUs;
121
122    static bool IsAudioFormat(const sp<AMessage> &format);
123
124    DISALLOW_EVIL_CONSTRUCTORS(Track);
125};
126
127WifiDisplaySource::PlaybackSession::Track::Track(
128        const sp<AMessage> &notify,
129        const sp<ALooper> &pullLooper,
130        const sp<ALooper> &codecLooper,
131        const sp<MediaPuller> &mediaPuller,
132        const sp<Converter> &converter)
133    : mNotify(notify),
134      mPullLooper(pullLooper),
135      mCodecLooper(codecLooper),
136      mMediaPuller(mediaPuller),
137      mConverter(converter),
138      mStarted(false),
139      mIsAudio(IsAudioFormat(mConverter->getOutputFormat())),
140      mLastOutputBufferQueuedTimeUs(-1ll) {
141}
142
143WifiDisplaySource::PlaybackSession::Track::Track(
144        const sp<AMessage> &notify, const sp<AMessage> &format)
145    : mNotify(notify),
146      mFormat(format),
147      mStarted(false),
148      mIsAudio(IsAudioFormat(format)),
149      mLastOutputBufferQueuedTimeUs(-1ll) {
150}
151
152WifiDisplaySource::PlaybackSession::Track::~Track() {
153    CHECK(!mStarted);
154}
155
156// static
157bool WifiDisplaySource::PlaybackSession::Track::IsAudioFormat(
158        const sp<AMessage> &format) {
159    AString mime;
160    CHECK(format->findString("mime", &mime));
161
162    return !strncasecmp(mime.c_str(), "audio/", 6);
163}
164
165sp<AMessage> WifiDisplaySource::PlaybackSession::Track::getFormat() {
166    return mFormat != NULL ? mFormat : mConverter->getOutputFormat();
167}
168
169bool WifiDisplaySource::PlaybackSession::Track::isAudio() const {
170    return mIsAudio;
171}
172
173const sp<Converter> &WifiDisplaySource::PlaybackSession::Track::converter() const {
174    return mConverter;
175}
176
177const sp<RepeaterSource> &
178WifiDisplaySource::PlaybackSession::Track::repeaterSource() const {
179    return mRepeaterSource;
180}
181
182ssize_t WifiDisplaySource::PlaybackSession::Track::mediaSenderTrackIndex() const {
183    CHECK_GE(mMediaSenderTrackIndex, 0);
184    return mMediaSenderTrackIndex;
185}
186
187void WifiDisplaySource::PlaybackSession::Track::setMediaSenderTrackIndex(
188        size_t index) {
189    mMediaSenderTrackIndex = index;
190}
191
192status_t WifiDisplaySource::PlaybackSession::Track::start() {
193    ALOGV("Track::start isAudio=%d", mIsAudio);
194
195    CHECK(!mStarted);
196
197    status_t err = OK;
198
199    if (mMediaPuller != NULL) {
200        err = mMediaPuller->start();
201    }
202
203    if (err == OK) {
204        mStarted = true;
205    }
206
207    return err;
208}
209
210void WifiDisplaySource::PlaybackSession::Track::stopAsync() {
211    ALOGV("Track::stopAsync isAudio=%d", mIsAudio);
212
213    if (mConverter != NULL) {
214        mConverter->shutdownAsync();
215    }
216
217    sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, id());
218
219    if (mStarted && mMediaPuller != NULL) {
220        if (mRepeaterSource != NULL) {
221            // Let's unblock MediaPuller's MediaSource::read().
222            mRepeaterSource->wakeUp();
223        }
224
225        mMediaPuller->stopAsync(msg);
226    } else {
227        mStarted = false;
228        msg->post();
229    }
230}
231
232void WifiDisplaySource::PlaybackSession::Track::pause() {
233    mMediaPuller->pause();
234}
235
236void WifiDisplaySource::PlaybackSession::Track::resume() {
237    mMediaPuller->resume();
238}
239
240void WifiDisplaySource::PlaybackSession::Track::onMessageReceived(
241        const sp<AMessage> &msg) {
242    switch (msg->what()) {
243        case kWhatMediaPullerStopped:
244        {
245            mConverter.clear();
246
247            mStarted = false;
248
249            sp<AMessage> notify = mNotify->dup();
250            notify->setInt32("what", kWhatStopped);
251            notify->post();
252
253            ALOGI("kWhatStopped %s posted", mIsAudio ? "audio" : "video");
254            break;
255        }
256
257        default:
258            TRESPASS();
259    }
260}
261
262void WifiDisplaySource::PlaybackSession::Track::queueAccessUnit(
263        const sp<ABuffer> &accessUnit) {
264    mQueuedAccessUnits.push_back(accessUnit);
265}
266
267sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueAccessUnit() {
268    if (mQueuedAccessUnits.empty()) {
269        return NULL;
270    }
271
272    sp<ABuffer> accessUnit = *mQueuedAccessUnits.begin();
273    CHECK(accessUnit != NULL);
274
275    mQueuedAccessUnits.erase(mQueuedAccessUnits.begin());
276
277    return accessUnit;
278}
279
280void WifiDisplaySource::PlaybackSession::Track::setRepeaterSource(
281        const sp<RepeaterSource> &source) {
282    mRepeaterSource = source;
283}
284
285void WifiDisplaySource::PlaybackSession::Track::requestIDRFrame() {
286    if (mIsAudio) {
287        return;
288    }
289
290    if (mRepeaterSource != NULL) {
291        mRepeaterSource->wakeUp();
292    }
293
294    mConverter->requestIDRFrame();
295}
296
297bool WifiDisplaySource::PlaybackSession::Track::hasOutputBuffer(
298        int64_t *timeUs) const {
299    *timeUs = 0ll;
300
301    if (mQueuedOutputBuffers.empty()) {
302        return false;
303    }
304
305    const sp<ABuffer> &outputBuffer = *mQueuedOutputBuffers.begin();
306
307    CHECK(outputBuffer->meta()->findInt64("timeUs", timeUs));
308
309    return true;
310}
311
312void WifiDisplaySource::PlaybackSession::Track::queueOutputBuffer(
313        const sp<ABuffer> &accessUnit) {
314    mQueuedOutputBuffers.push_back(accessUnit);
315    mLastOutputBufferQueuedTimeUs = ALooper::GetNowUs();
316}
317
318sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueOutputBuffer() {
319    CHECK(!mQueuedOutputBuffers.empty());
320
321    sp<ABuffer> outputBuffer = *mQueuedOutputBuffers.begin();
322    mQueuedOutputBuffers.erase(mQueuedOutputBuffers.begin());
323
324    return outputBuffer;
325}
326
327#if SUSPEND_VIDEO_IF_IDLE
328bool WifiDisplaySource::PlaybackSession::Track::isSuspended() const {
329    if (!mQueuedOutputBuffers.empty()) {
330        return false;
331    }
332
333    if (mLastOutputBufferQueuedTimeUs < 0ll) {
334        // We've never seen an output buffer queued, but tracks start
335        // out live, not suspended.
336        return false;
337    }
338
339    // If we've not seen new output data for 60ms or more, we consider
340    // this track suspended for the time being.
341    return (ALooper::GetNowUs() - mLastOutputBufferQueuedTimeUs) > 60000ll;
342}
343#endif
344
345////////////////////////////////////////////////////////////////////////////////
346
347WifiDisplaySource::PlaybackSession::PlaybackSession(
348        const sp<ANetworkSession> &netSession,
349        const sp<AMessage> &notify,
350        const in_addr &interfaceAddr,
351        const sp<IHDCP> &hdcp,
352        const char *path)
353    : mNetSession(netSession),
354      mNotify(notify),
355      mInterfaceAddr(interfaceAddr),
356      mHDCP(hdcp),
357      mLocalRTPPort(-1),
358      mWeAreDead(false),
359      mPaused(false),
360      mLastLifesignUs(),
361      mVideoTrackIndex(-1),
362      mPrevTimeUs(-1ll),
363      mPullExtractorPending(false),
364      mPullExtractorGeneration(0),
365      mFirstSampleTimeRealUs(-1ll),
366      mFirstSampleTimeUs(-1ll) {
367    if (path != NULL) {
368        mMediaPath.setTo(path);
369    }
370}
371
372status_t WifiDisplaySource::PlaybackSession::init(
373        const char *clientIP,
374        int32_t clientRtp,
375        RTPSender::TransportMode rtpMode,
376        int32_t clientRtcp,
377        RTPSender::TransportMode rtcpMode,
378        bool enableAudio,
379        bool usePCMAudio,
380        bool enableVideo,
381        VideoFormats::ResolutionType videoResolutionType,
382        size_t videoResolutionIndex,
383        VideoFormats::ProfileType videoProfileType,
384        VideoFormats::LevelType videoLevelType) {
385    sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, id());
386    mMediaSender = new MediaSender(mNetSession, notify);
387    looper()->registerHandler(mMediaSender);
388
389    mMediaSender->setHDCP(mHDCP);
390
391    status_t err = setupPacketizer(
392            enableAudio,
393            usePCMAudio,
394            enableVideo,
395            videoResolutionType,
396            videoResolutionIndex,
397            videoProfileType,
398            videoLevelType);
399
400    if (err == OK) {
401        err = mMediaSender->initAsync(
402                -1 /* trackIndex */,
403                clientIP,
404                clientRtp,
405                rtpMode,
406                clientRtcp,
407                rtcpMode,
408                &mLocalRTPPort);
409    }
410
411    if (err != OK) {
412        mLocalRTPPort = -1;
413
414        looper()->unregisterHandler(mMediaSender->id());
415        mMediaSender.clear();
416
417        return err;
418    }
419
420    updateLiveness();
421
422    return OK;
423}
424
425WifiDisplaySource::PlaybackSession::~PlaybackSession() {
426}
427
428int32_t WifiDisplaySource::PlaybackSession::getRTPPort() const {
429    return mLocalRTPPort;
430}
431
432int64_t WifiDisplaySource::PlaybackSession::getLastLifesignUs() const {
433    return mLastLifesignUs;
434}
435
436void WifiDisplaySource::PlaybackSession::updateLiveness() {
437    mLastLifesignUs = ALooper::GetNowUs();
438}
439
440status_t WifiDisplaySource::PlaybackSession::play() {
441    updateLiveness();
442
443    (new AMessage(kWhatResume, id()))->post();
444
445    return OK;
446}
447
448status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() {
449    for (size_t i = 0; i < mTracks.size(); ++i) {
450        CHECK_EQ((status_t)OK, mTracks.editValueAt(i)->start());
451    }
452
453    sp<AMessage> notify = mNotify->dup();
454    notify->setInt32("what", kWhatSessionEstablished);
455    notify->post();
456
457    return OK;
458}
459
460status_t WifiDisplaySource::PlaybackSession::pause() {
461    updateLiveness();
462
463    (new AMessage(kWhatPause, id()))->post();
464
465    return OK;
466}
467
468void WifiDisplaySource::PlaybackSession::destroyAsync() {
469    ALOGI("destroyAsync");
470
471    for (size_t i = 0; i < mTracks.size(); ++i) {
472        mTracks.valueAt(i)->stopAsync();
473    }
474}
475
476void WifiDisplaySource::PlaybackSession::onMessageReceived(
477        const sp<AMessage> &msg) {
478    switch (msg->what()) {
479        case kWhatConverterNotify:
480        {
481            if (mWeAreDead) {
482                ALOGV("dropping msg '%s' because we're dead",
483                      msg->debugString().c_str());
484
485                break;
486            }
487
488            int32_t what;
489            CHECK(msg->findInt32("what", &what));
490
491            size_t trackIndex;
492            CHECK(msg->findSize("trackIndex", &trackIndex));
493
494            if (what == Converter::kWhatAccessUnit) {
495                sp<ABuffer> accessUnit;
496                CHECK(msg->findBuffer("accessUnit", &accessUnit));
497
498                const sp<Track> &track = mTracks.valueFor(trackIndex);
499
500                status_t err = mMediaSender->queueAccessUnit(
501                        track->mediaSenderTrackIndex(),
502                        accessUnit);
503
504                if (err != OK) {
505                    notifySessionDead();
506                }
507                break;
508            } else if (what == Converter::kWhatEOS) {
509                CHECK_EQ(what, Converter::kWhatEOS);
510
511                ALOGI("output EOS on track %d", trackIndex);
512
513                ssize_t index = mTracks.indexOfKey(trackIndex);
514                CHECK_GE(index, 0);
515
516                const sp<Converter> &converter =
517                    mTracks.valueAt(index)->converter();
518                looper()->unregisterHandler(converter->id());
519
520                mTracks.removeItemsAt(index);
521
522                if (mTracks.isEmpty()) {
523                    ALOGI("Reached EOS");
524                }
525            } else if (what != Converter::kWhatShutdownCompleted) {
526                CHECK_EQ(what, Converter::kWhatError);
527
528                status_t err;
529                CHECK(msg->findInt32("err", &err));
530
531                ALOGE("converter signaled error %d", err);
532
533                notifySessionDead();
534            }
535            break;
536        }
537
538        case kWhatMediaSenderNotify:
539        {
540            int32_t what;
541            CHECK(msg->findInt32("what", &what));
542
543            if (what == MediaSender::kWhatInitDone) {
544                status_t err;
545                CHECK(msg->findInt32("err", &err));
546
547                if (err == OK) {
548                    onMediaSenderInitialized();
549                } else {
550                    notifySessionDead();
551                }
552            } else if (what == MediaSender::kWhatError) {
553                notifySessionDead();
554            } else if (what == MediaSender::kWhatNetworkStall) {
555                size_t numBytesQueued;
556                CHECK(msg->findSize("numBytesQueued", &numBytesQueued));
557
558                if (mVideoTrackIndex >= 0) {
559                    const sp<Track> &videoTrack =
560                        mTracks.valueFor(mVideoTrackIndex);
561
562                    sp<Converter> converter = videoTrack->converter();
563                    if (converter != NULL) {
564                        converter->dropAFrame();
565                    }
566                }
567            } else if (what == MediaSender::kWhatInformSender) {
568                onSinkFeedback(msg);
569            } else {
570                TRESPASS();
571            }
572            break;
573        }
574
575        case kWhatTrackNotify:
576        {
577            int32_t what;
578            CHECK(msg->findInt32("what", &what));
579
580            size_t trackIndex;
581            CHECK(msg->findSize("trackIndex", &trackIndex));
582
583            if (what == Track::kWhatStopped) {
584                ALOGI("Track %d stopped", trackIndex);
585
586                sp<Track> track = mTracks.valueFor(trackIndex);
587                looper()->unregisterHandler(track->id());
588                mTracks.removeItem(trackIndex);
589                track.clear();
590
591                if (!mTracks.isEmpty()) {
592                    ALOGI("not all tracks are stopped yet");
593                    break;
594                }
595
596                looper()->unregisterHandler(mMediaSender->id());
597                mMediaSender.clear();
598
599                sp<AMessage> notify = mNotify->dup();
600                notify->setInt32("what", kWhatSessionDestroyed);
601                notify->post();
602            }
603            break;
604        }
605
606        case kWhatPause:
607        {
608            if (mExtractor != NULL) {
609                ++mPullExtractorGeneration;
610                mFirstSampleTimeRealUs = -1ll;
611                mFirstSampleTimeUs = -1ll;
612            }
613
614            if (mPaused) {
615                break;
616            }
617
618            for (size_t i = 0; i < mTracks.size(); ++i) {
619                mTracks.editValueAt(i)->pause();
620            }
621
622            mPaused = true;
623            break;
624        }
625
626        case kWhatResume:
627        {
628            if (mExtractor != NULL) {
629                schedulePullExtractor();
630            }
631
632            if (!mPaused) {
633                break;
634            }
635
636            for (size_t i = 0; i < mTracks.size(); ++i) {
637                mTracks.editValueAt(i)->resume();
638            }
639
640            mPaused = false;
641            break;
642        }
643
644        case kWhatPullExtractorSample:
645        {
646            int32_t generation;
647            CHECK(msg->findInt32("generation", &generation));
648
649            if (generation != mPullExtractorGeneration) {
650                break;
651            }
652
653            mPullExtractorPending = false;
654
655            onPullExtractor();
656            break;
657        }
658
659        default:
660            TRESPASS();
661    }
662}
663
664void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) {
665    int64_t avgLatencyUs;
666    CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs));
667
668    int64_t maxLatencyUs;
669    CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs));
670
671    ALOGI("sink reports avg. latency of %lld ms (max %lld ms)",
672          avgLatencyUs / 1000ll,
673          maxLatencyUs / 1000ll);
674
675    if (mVideoTrackIndex >= 0) {
676        const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex);
677        sp<Converter> converter = videoTrack->converter();
678
679        if (converter != NULL) {
680            int32_t videoBitrate =
681                Converter::GetInt32Property("media.wfd.video-bitrate", -1);
682
683            char val[PROPERTY_VALUE_MAX];
684            if (videoBitrate < 0
685                    && property_get("media.wfd.video-bitrate", val, NULL)
686                    && !strcasecmp("adaptive", val)) {
687                videoBitrate = converter->getVideoBitrate();
688
689                if (avgLatencyUs > 300000ll) {
690                    videoBitrate *= 0.6;
691                } else if (avgLatencyUs < 100000ll) {
692                    videoBitrate *= 1.1;
693                }
694            }
695
696            if (videoBitrate > 0) {
697                if (videoBitrate < 500000) {
698                    videoBitrate = 500000;
699                } else if (videoBitrate > 10000000) {
700                    videoBitrate = 10000000;
701                }
702
703                if (videoBitrate != converter->getVideoBitrate()) {
704                    ALOGI("setting video bitrate to %d bps", videoBitrate);
705
706                    converter->setVideoBitrate(videoBitrate);
707                }
708            }
709        }
710
711        sp<RepeaterSource> repeaterSource = videoTrack->repeaterSource();
712        if (repeaterSource != NULL) {
713            double rateHz =
714                Converter::GetInt32Property(
715                        "media.wfd.video-framerate", -1);
716
717            char val[PROPERTY_VALUE_MAX];
718            if (rateHz < 0.0
719                    && property_get("media.wfd.video-framerate", val, NULL)
720                    && !strcasecmp("adaptive", val)) {
721                 rateHz = repeaterSource->getFrameRate();
722
723                if (avgLatencyUs > 300000ll) {
724                    rateHz *= 0.9;
725                } else if (avgLatencyUs < 200000ll) {
726                    rateHz *= 1.1;
727                }
728            }
729
730            if (rateHz > 0) {
731                if (rateHz < 5.0) {
732                    rateHz = 5.0;
733                } else if (rateHz > 30.0) {
734                    rateHz = 30.0;
735                }
736
737                if (rateHz != repeaterSource->getFrameRate()) {
738                    ALOGI("setting frame rate to %.2f Hz", rateHz);
739
740                    repeaterSource->setFrameRate(rateHz);
741                }
742            }
743        }
744    }
745}
746
747status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer(
748        bool enableAudio, bool enableVideo) {
749    DataSource::RegisterDefaultSniffers();
750
751    mExtractor = new NuMediaExtractor;
752
753    status_t err = mExtractor->setDataSource(
754            NULL /* httpService */, mMediaPath.c_str());
755
756    if (err != OK) {
757        return err;
758    }
759
760    size_t n = mExtractor->countTracks();
761    bool haveAudio = false;
762    bool haveVideo = false;
763    for (size_t i = 0; i < n; ++i) {
764        sp<AMessage> format;
765        err = mExtractor->getTrackFormat(i, &format);
766
767        if (err != OK) {
768            continue;
769        }
770
771        AString mime;
772        CHECK(format->findString("mime", &mime));
773
774        bool isAudio = !strncasecmp(mime.c_str(), "audio/", 6);
775        bool isVideo = !strncasecmp(mime.c_str(), "video/", 6);
776
777        if (isAudio && enableAudio && !haveAudio) {
778            haveAudio = true;
779        } else if (isVideo && enableVideo && !haveVideo) {
780            haveVideo = true;
781        } else {
782            continue;
783        }
784
785        err = mExtractor->selectTrack(i);
786
787        size_t trackIndex = mTracks.size();
788
789        sp<AMessage> notify = new AMessage(kWhatTrackNotify, id());
790        notify->setSize("trackIndex", trackIndex);
791
792        sp<Track> track = new Track(notify, format);
793        looper()->registerHandler(track);
794
795        mTracks.add(trackIndex, track);
796
797        mExtractorTrackToInternalTrack.add(i, trackIndex);
798
799        if (isVideo) {
800            mVideoTrackIndex = trackIndex;
801        }
802
803        uint32_t flags = MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
804
805        ssize_t mediaSenderTrackIndex =
806            mMediaSender->addTrack(format, flags);
807        CHECK_GE(mediaSenderTrackIndex, 0);
808
809        track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
810
811        if ((haveAudio || !enableAudio) && (haveVideo || !enableVideo)) {
812            break;
813        }
814    }
815
816    return OK;
817}
818
819void WifiDisplaySource::PlaybackSession::schedulePullExtractor() {
820    if (mPullExtractorPending) {
821        return;
822    }
823
824    int64_t sampleTimeUs;
825    status_t err = mExtractor->getSampleTime(&sampleTimeUs);
826
827    int64_t nowUs = ALooper::GetNowUs();
828
829    if (mFirstSampleTimeRealUs < 0ll) {
830        mFirstSampleTimeRealUs = nowUs;
831        mFirstSampleTimeUs = sampleTimeUs;
832    }
833
834    int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs;
835
836    sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, id());
837    msg->setInt32("generation", mPullExtractorGeneration);
838    msg->post(whenUs - nowUs);
839
840    mPullExtractorPending = true;
841}
842
843void WifiDisplaySource::PlaybackSession::onPullExtractor() {
844    sp<ABuffer> accessUnit = new ABuffer(1024 * 1024);
845    status_t err = mExtractor->readSampleData(accessUnit);
846    if (err != OK) {
847        // EOS.
848        return;
849    }
850
851    int64_t timeUs;
852    CHECK_EQ((status_t)OK, mExtractor->getSampleTime(&timeUs));
853
854    accessUnit->meta()->setInt64(
855            "timeUs", mFirstSampleTimeRealUs + timeUs - mFirstSampleTimeUs);
856
857    size_t trackIndex;
858    CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex));
859
860    sp<AMessage> msg = new AMessage(kWhatConverterNotify, id());
861
862    msg->setSize(
863            "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex));
864
865    msg->setInt32("what", Converter::kWhatAccessUnit);
866    msg->setBuffer("accessUnit", accessUnit);
867    msg->post();
868
869    mExtractor->advance();
870
871    schedulePullExtractor();
872}
873
874status_t WifiDisplaySource::PlaybackSession::setupPacketizer(
875        bool enableAudio,
876        bool usePCMAudio,
877        bool enableVideo,
878        VideoFormats::ResolutionType videoResolutionType,
879        size_t videoResolutionIndex,
880        VideoFormats::ProfileType videoProfileType,
881        VideoFormats::LevelType videoLevelType) {
882    CHECK(enableAudio || enableVideo);
883
884    if (!mMediaPath.empty()) {
885        return setupMediaPacketizer(enableAudio, enableVideo);
886    }
887
888    if (enableVideo) {
889        status_t err = addVideoSource(
890                videoResolutionType, videoResolutionIndex, videoProfileType,
891                videoLevelType);
892
893        if (err != OK) {
894            return err;
895        }
896    }
897
898    if (!enableAudio) {
899        return OK;
900    }
901
902    return addAudioSource(usePCMAudio);
903}
904
905status_t WifiDisplaySource::PlaybackSession::addSource(
906        bool isVideo, const sp<MediaSource> &source, bool isRepeaterSource,
907        bool usePCMAudio, unsigned profileIdc, unsigned levelIdc,
908        unsigned constraintSet, size_t *numInputBuffers) {
909    CHECK(!usePCMAudio || !isVideo);
910    CHECK(!isRepeaterSource || isVideo);
911    CHECK(!profileIdc || isVideo);
912    CHECK(!levelIdc || isVideo);
913    CHECK(!constraintSet || isVideo);
914
915    sp<ALooper> pullLooper = new ALooper;
916    pullLooper->setName("pull_looper");
917
918    pullLooper->start(
919            false /* runOnCallingThread */,
920            false /* canCallJava */,
921            PRIORITY_AUDIO);
922
923    sp<ALooper> codecLooper = new ALooper;
924    codecLooper->setName("codec_looper");
925
926    codecLooper->start(
927            false /* runOnCallingThread */,
928            false /* canCallJava */,
929            PRIORITY_AUDIO);
930
931    size_t trackIndex;
932
933    sp<AMessage> notify;
934
935    trackIndex = mTracks.size();
936
937    sp<AMessage> format;
938    status_t err = convertMetaDataToMessage(source->getFormat(), &format);
939    CHECK_EQ(err, (status_t)OK);
940
941    if (isVideo) {
942        format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC);
943        format->setInt32("store-metadata-in-buffers", true);
944        format->setInt32("store-metadata-in-buffers-output", (mHDCP != NULL)
945                && (mHDCP->getCaps() & HDCPModule::HDCP_CAPS_ENCRYPT_NATIVE));
946        format->setInt32(
947                "color-format", OMX_COLOR_FormatAndroidOpaque);
948        format->setInt32("profile-idc", profileIdc);
949        format->setInt32("level-idc", levelIdc);
950        format->setInt32("constraint-set", constraintSet);
951    } else {
952        format->setString(
953                "mime",
954                usePCMAudio
955                    ? MEDIA_MIMETYPE_AUDIO_RAW : MEDIA_MIMETYPE_AUDIO_AAC);
956    }
957
958    notify = new AMessage(kWhatConverterNotify, id());
959    notify->setSize("trackIndex", trackIndex);
960
961    sp<Converter> converter = new Converter(notify, codecLooper, format);
962
963    looper()->registerHandler(converter);
964
965    err = converter->init();
966    if (err != OK) {
967        ALOGE("%s converter returned err %d", isVideo ? "video" : "audio", err);
968
969        looper()->unregisterHandler(converter->id());
970        return err;
971    }
972
973    notify = new AMessage(Converter::kWhatMediaPullerNotify, converter->id());
974    notify->setSize("trackIndex", trackIndex);
975
976    sp<MediaPuller> puller = new MediaPuller(source, notify);
977    pullLooper->registerHandler(puller);
978
979    if (numInputBuffers != NULL) {
980        *numInputBuffers = converter->getInputBufferCount();
981    }
982
983    notify = new AMessage(kWhatTrackNotify, id());
984    notify->setSize("trackIndex", trackIndex);
985
986    sp<Track> track = new Track(
987            notify, pullLooper, codecLooper, puller, converter);
988
989    if (isRepeaterSource) {
990        track->setRepeaterSource(static_cast<RepeaterSource *>(source.get()));
991    }
992
993    looper()->registerHandler(track);
994
995    mTracks.add(trackIndex, track);
996
997    if (isVideo) {
998        mVideoTrackIndex = trackIndex;
999    }
1000
1001    uint32_t flags = 0;
1002    if (converter->needToManuallyPrependSPSPPS()) {
1003        flags |= MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
1004    }
1005
1006    ssize_t mediaSenderTrackIndex =
1007        mMediaSender->addTrack(converter->getOutputFormat(), flags);
1008    CHECK_GE(mediaSenderTrackIndex, 0);
1009
1010    track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
1011
1012    return OK;
1013}
1014
1015status_t WifiDisplaySource::PlaybackSession::addVideoSource(
1016        VideoFormats::ResolutionType videoResolutionType,
1017        size_t videoResolutionIndex,
1018        VideoFormats::ProfileType videoProfileType,
1019        VideoFormats::LevelType videoLevelType) {
1020    size_t width, height, framesPerSecond;
1021    bool interlaced;
1022    CHECK(VideoFormats::GetConfiguration(
1023                videoResolutionType,
1024                videoResolutionIndex,
1025                &width,
1026                &height,
1027                &framesPerSecond,
1028                &interlaced));
1029
1030    unsigned profileIdc, levelIdc, constraintSet;
1031    CHECK(VideoFormats::GetProfileLevel(
1032                videoProfileType,
1033                videoLevelType,
1034                &profileIdc,
1035                &levelIdc,
1036                &constraintSet));
1037
1038    sp<SurfaceMediaSource> source = new SurfaceMediaSource(width, height);
1039
1040    source->setUseAbsoluteTimestamps();
1041
1042    sp<RepeaterSource> videoSource =
1043        new RepeaterSource(source, framesPerSecond);
1044
1045    size_t numInputBuffers;
1046    status_t err = addSource(
1047            true /* isVideo */, videoSource, true /* isRepeaterSource */,
1048            false /* usePCMAudio */, profileIdc, levelIdc, constraintSet,
1049            &numInputBuffers);
1050
1051    if (err != OK) {
1052        return err;
1053    }
1054
1055    err = source->setMaxAcquiredBufferCount(numInputBuffers);
1056    CHECK_EQ(err, (status_t)OK);
1057
1058    mProducer = source->getProducer();
1059
1060    return OK;
1061}
1062
1063status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) {
1064    sp<AudioSource> audioSource = new AudioSource(
1065            AUDIO_SOURCE_REMOTE_SUBMIX,
1066            48000 /* sampleRate */,
1067            2 /* channelCount */);
1068
1069    if (audioSource->initCheck() == OK) {
1070        return addSource(
1071                false /* isVideo */, audioSource, false /* isRepeaterSource */,
1072                usePCMAudio, 0 /* profileIdc */, 0 /* levelIdc */,
1073                0 /* constraintSet */, NULL /* numInputBuffers */);
1074    }
1075
1076    ALOGW("Unable to instantiate audio source");
1077
1078    return OK;
1079}
1080
1081sp<IGraphicBufferProducer> WifiDisplaySource::PlaybackSession::getSurfaceTexture() {
1082    return mProducer;
1083}
1084
1085void WifiDisplaySource::PlaybackSession::requestIDRFrame() {
1086    for (size_t i = 0; i < mTracks.size(); ++i) {
1087        const sp<Track> &track = mTracks.valueAt(i);
1088
1089        track->requestIDRFrame();
1090    }
1091}
1092
1093void WifiDisplaySource::PlaybackSession::notifySessionDead() {
1094    // Inform WifiDisplaySource of our premature death (wish).
1095    sp<AMessage> notify = mNotify->dup();
1096    notify->setInt32("what", kWhatSessionDead);
1097    notify->post();
1098
1099    mWeAreDead = true;
1100}
1101
1102}  // namespace android
1103
1104