1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "PlaybackSession"
19#include <utils/Log.h>
20
21#include "PlaybackSession.h"
22
23#include "Converter.h"
24#include "MediaPuller.h"
25#include "RepeaterSource.h"
26#include "include/avc_utils.h"
27#include "WifiDisplaySource.h"
28
29#include <binder/IServiceManager.h>
30#include <cutils/properties.h>
31#include <media/IHDCP.h>
32#include <media/IMediaHTTPService.h>
33#include <media/stagefright/foundation/ABitReader.h>
34#include <media/stagefright/foundation/ABuffer.h>
35#include <media/stagefright/foundation/ADebug.h>
36#include <media/stagefright/foundation/AMessage.h>
37#include <media/stagefright/foundation/hexdump.h>
38#include <media/stagefright/AudioSource.h>
39#include <media/stagefright/MediaDefs.h>
40#include <media/stagefright/MediaErrors.h>
41#include <media/stagefright/MediaSource.h>
42#include <media/stagefright/MetaData.h>
43#include <media/stagefright/NuMediaExtractor.h>
44#include <media/stagefright/SurfaceMediaSource.h>
45#include <media/stagefright/Utils.h>
46
47#include <OMX_IVCommon.h>
48
49namespace android {
50
51struct WifiDisplaySource::PlaybackSession::Track : public AHandler {
52    enum {
53        kWhatStopped,
54    };
55
56    Track(const sp<AMessage> &notify,
57          const sp<ALooper> &pullLooper,
58          const sp<ALooper> &codecLooper,
59          const sp<MediaPuller> &mediaPuller,
60          const sp<Converter> &converter);
61
62    Track(const sp<AMessage> &notify, const sp<AMessage> &format);
63
64    void setRepeaterSource(const sp<RepeaterSource> &source);
65
66    sp<AMessage> getFormat();
67    bool isAudio() const;
68
69    const sp<Converter> &converter() const;
70    const sp<RepeaterSource> &repeaterSource() const;
71
72    ssize_t mediaSenderTrackIndex() const;
73    void setMediaSenderTrackIndex(size_t index);
74
75    status_t start();
76    void stopAsync();
77
78    void pause();
79    void resume();
80
81    void queueAccessUnit(const sp<ABuffer> &accessUnit);
82    sp<ABuffer> dequeueAccessUnit();
83
84    bool hasOutputBuffer(int64_t *timeUs) const;
85    void queueOutputBuffer(const sp<ABuffer> &accessUnit);
86    sp<ABuffer> dequeueOutputBuffer();
87
88#if SUSPEND_VIDEO_IF_IDLE
89    bool isSuspended() const;
90#endif
91
92    size_t countQueuedOutputBuffers() const {
93        return mQueuedOutputBuffers.size();
94    }
95
96    void requestIDRFrame();
97
98protected:
99    virtual void onMessageReceived(const sp<AMessage> &msg);
100    virtual ~Track();
101
102private:
103    enum {
104        kWhatMediaPullerStopped,
105    };
106
107    sp<AMessage> mNotify;
108    sp<ALooper> mPullLooper;
109    sp<ALooper> mCodecLooper;
110    sp<MediaPuller> mMediaPuller;
111    sp<Converter> mConverter;
112    sp<AMessage> mFormat;
113    bool mStarted;
114    ssize_t mMediaSenderTrackIndex;
115    bool mIsAudio;
116    List<sp<ABuffer> > mQueuedAccessUnits;
117    sp<RepeaterSource> mRepeaterSource;
118    List<sp<ABuffer> > mQueuedOutputBuffers;
119    int64_t mLastOutputBufferQueuedTimeUs;
120
121    static bool IsAudioFormat(const sp<AMessage> &format);
122
123    DISALLOW_EVIL_CONSTRUCTORS(Track);
124};
125
126WifiDisplaySource::PlaybackSession::Track::Track(
127        const sp<AMessage> &notify,
128        const sp<ALooper> &pullLooper,
129        const sp<ALooper> &codecLooper,
130        const sp<MediaPuller> &mediaPuller,
131        const sp<Converter> &converter)
132    : mNotify(notify),
133      mPullLooper(pullLooper),
134      mCodecLooper(codecLooper),
135      mMediaPuller(mediaPuller),
136      mConverter(converter),
137      mStarted(false),
138      mIsAudio(IsAudioFormat(mConverter->getOutputFormat())),
139      mLastOutputBufferQueuedTimeUs(-1ll) {
140}
141
142WifiDisplaySource::PlaybackSession::Track::Track(
143        const sp<AMessage> &notify, const sp<AMessage> &format)
144    : mNotify(notify),
145      mFormat(format),
146      mStarted(false),
147      mIsAudio(IsAudioFormat(format)),
148      mLastOutputBufferQueuedTimeUs(-1ll) {
149}
150
151WifiDisplaySource::PlaybackSession::Track::~Track() {
152    CHECK(!mStarted);
153}
154
155// static
156bool WifiDisplaySource::PlaybackSession::Track::IsAudioFormat(
157        const sp<AMessage> &format) {
158    AString mime;
159    CHECK(format->findString("mime", &mime));
160
161    return !strncasecmp(mime.c_str(), "audio/", 6);
162}
163
164sp<AMessage> WifiDisplaySource::PlaybackSession::Track::getFormat() {
165    return mFormat != NULL ? mFormat : mConverter->getOutputFormat();
166}
167
168bool WifiDisplaySource::PlaybackSession::Track::isAudio() const {
169    return mIsAudio;
170}
171
172const sp<Converter> &WifiDisplaySource::PlaybackSession::Track::converter() const {
173    return mConverter;
174}
175
176const sp<RepeaterSource> &
177WifiDisplaySource::PlaybackSession::Track::repeaterSource() const {
178    return mRepeaterSource;
179}
180
181ssize_t WifiDisplaySource::PlaybackSession::Track::mediaSenderTrackIndex() const {
182    CHECK_GE(mMediaSenderTrackIndex, 0);
183    return mMediaSenderTrackIndex;
184}
185
186void WifiDisplaySource::PlaybackSession::Track::setMediaSenderTrackIndex(
187        size_t index) {
188    mMediaSenderTrackIndex = index;
189}
190
191status_t WifiDisplaySource::PlaybackSession::Track::start() {
192    ALOGV("Track::start isAudio=%d", mIsAudio);
193
194    CHECK(!mStarted);
195
196    status_t err = OK;
197
198    if (mMediaPuller != NULL) {
199        err = mMediaPuller->start();
200    }
201
202    if (err == OK) {
203        mStarted = true;
204    }
205
206    return err;
207}
208
209void WifiDisplaySource::PlaybackSession::Track::stopAsync() {
210    ALOGV("Track::stopAsync isAudio=%d", mIsAudio);
211
212    if (mConverter != NULL) {
213        mConverter->shutdownAsync();
214    }
215
216    sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, this);
217
218    if (mStarted && mMediaPuller != NULL) {
219        if (mRepeaterSource != NULL) {
220            // Let's unblock MediaPuller's MediaSource::read().
221            mRepeaterSource->wakeUp();
222        }
223
224        mMediaPuller->stopAsync(msg);
225    } else {
226        mStarted = false;
227        msg->post();
228    }
229}
230
231void WifiDisplaySource::PlaybackSession::Track::pause() {
232    mMediaPuller->pause();
233}
234
235void WifiDisplaySource::PlaybackSession::Track::resume() {
236    mMediaPuller->resume();
237}
238
239void WifiDisplaySource::PlaybackSession::Track::onMessageReceived(
240        const sp<AMessage> &msg) {
241    switch (msg->what()) {
242        case kWhatMediaPullerStopped:
243        {
244            mConverter.clear();
245
246            mStarted = false;
247
248            sp<AMessage> notify = mNotify->dup();
249            notify->setInt32("what", kWhatStopped);
250            notify->post();
251
252            ALOGI("kWhatStopped %s posted", mIsAudio ? "audio" : "video");
253            break;
254        }
255
256        default:
257            TRESPASS();
258    }
259}
260
261void WifiDisplaySource::PlaybackSession::Track::queueAccessUnit(
262        const sp<ABuffer> &accessUnit) {
263    mQueuedAccessUnits.push_back(accessUnit);
264}
265
266sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueAccessUnit() {
267    if (mQueuedAccessUnits.empty()) {
268        return NULL;
269    }
270
271    sp<ABuffer> accessUnit = *mQueuedAccessUnits.begin();
272    CHECK(accessUnit != NULL);
273
274    mQueuedAccessUnits.erase(mQueuedAccessUnits.begin());
275
276    return accessUnit;
277}
278
279void WifiDisplaySource::PlaybackSession::Track::setRepeaterSource(
280        const sp<RepeaterSource> &source) {
281    mRepeaterSource = source;
282}
283
284void WifiDisplaySource::PlaybackSession::Track::requestIDRFrame() {
285    if (mIsAudio) {
286        return;
287    }
288
289    if (mRepeaterSource != NULL) {
290        mRepeaterSource->wakeUp();
291    }
292
293    mConverter->requestIDRFrame();
294}
295
296bool WifiDisplaySource::PlaybackSession::Track::hasOutputBuffer(
297        int64_t *timeUs) const {
298    *timeUs = 0ll;
299
300    if (mQueuedOutputBuffers.empty()) {
301        return false;
302    }
303
304    const sp<ABuffer> &outputBuffer = *mQueuedOutputBuffers.begin();
305
306    CHECK(outputBuffer->meta()->findInt64("timeUs", timeUs));
307
308    return true;
309}
310
311void WifiDisplaySource::PlaybackSession::Track::queueOutputBuffer(
312        const sp<ABuffer> &accessUnit) {
313    mQueuedOutputBuffers.push_back(accessUnit);
314    mLastOutputBufferQueuedTimeUs = ALooper::GetNowUs();
315}
316
317sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueOutputBuffer() {
318    CHECK(!mQueuedOutputBuffers.empty());
319
320    sp<ABuffer> outputBuffer = *mQueuedOutputBuffers.begin();
321    mQueuedOutputBuffers.erase(mQueuedOutputBuffers.begin());
322
323    return outputBuffer;
324}
325
326#if SUSPEND_VIDEO_IF_IDLE
327bool WifiDisplaySource::PlaybackSession::Track::isSuspended() const {
328    if (!mQueuedOutputBuffers.empty()) {
329        return false;
330    }
331
332    if (mLastOutputBufferQueuedTimeUs < 0ll) {
333        // We've never seen an output buffer queued, but tracks start
334        // out live, not suspended.
335        return false;
336    }
337
338    // If we've not seen new output data for 60ms or more, we consider
339    // this track suspended for the time being.
340    return (ALooper::GetNowUs() - mLastOutputBufferQueuedTimeUs) > 60000ll;
341}
342#endif
343
344////////////////////////////////////////////////////////////////////////////////
345
346WifiDisplaySource::PlaybackSession::PlaybackSession(
347        const String16 &opPackageName,
348        const sp<ANetworkSession> &netSession,
349        const sp<AMessage> &notify,
350        const in_addr &interfaceAddr,
351        const sp<IHDCP> &hdcp,
352        const char *path)
353    : mOpPackageName(opPackageName),
354      mNetSession(netSession),
355      mNotify(notify),
356      mInterfaceAddr(interfaceAddr),
357      mHDCP(hdcp),
358      mLocalRTPPort(-1),
359      mWeAreDead(false),
360      mPaused(false),
361      mLastLifesignUs(),
362      mVideoTrackIndex(-1),
363      mPrevTimeUs(-1ll),
364      mPullExtractorPending(false),
365      mPullExtractorGeneration(0),
366      mFirstSampleTimeRealUs(-1ll),
367      mFirstSampleTimeUs(-1ll) {
368    if (path != NULL) {
369        mMediaPath.setTo(path);
370    }
371}
372
373status_t WifiDisplaySource::PlaybackSession::init(
374        const char *clientIP,
375        int32_t clientRtp,
376        RTPSender::TransportMode rtpMode,
377        int32_t clientRtcp,
378        RTPSender::TransportMode rtcpMode,
379        bool enableAudio,
380        bool usePCMAudio,
381        bool enableVideo,
382        VideoFormats::ResolutionType videoResolutionType,
383        size_t videoResolutionIndex,
384        VideoFormats::ProfileType videoProfileType,
385        VideoFormats::LevelType videoLevelType) {
386    sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, this);
387    mMediaSender = new MediaSender(mNetSession, notify);
388    looper()->registerHandler(mMediaSender);
389
390    mMediaSender->setHDCP(mHDCP);
391
392    status_t err = setupPacketizer(
393            enableAudio,
394            usePCMAudio,
395            enableVideo,
396            videoResolutionType,
397            videoResolutionIndex,
398            videoProfileType,
399            videoLevelType);
400
401    if (err == OK) {
402        err = mMediaSender->initAsync(
403                -1 /* trackIndex */,
404                clientIP,
405                clientRtp,
406                rtpMode,
407                clientRtcp,
408                rtcpMode,
409                &mLocalRTPPort);
410    }
411
412    if (err != OK) {
413        mLocalRTPPort = -1;
414
415        looper()->unregisterHandler(mMediaSender->id());
416        mMediaSender.clear();
417
418        return err;
419    }
420
421    updateLiveness();
422
423    return OK;
424}
425
426WifiDisplaySource::PlaybackSession::~PlaybackSession() {
427}
428
429int32_t WifiDisplaySource::PlaybackSession::getRTPPort() const {
430    return mLocalRTPPort;
431}
432
433int64_t WifiDisplaySource::PlaybackSession::getLastLifesignUs() const {
434    return mLastLifesignUs;
435}
436
437void WifiDisplaySource::PlaybackSession::updateLiveness() {
438    mLastLifesignUs = ALooper::GetNowUs();
439}
440
441status_t WifiDisplaySource::PlaybackSession::play() {
442    updateLiveness();
443
444    (new AMessage(kWhatResume, this))->post();
445
446    return OK;
447}
448
449status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() {
450    for (size_t i = 0; i < mTracks.size(); ++i) {
451        CHECK_EQ((status_t)OK, mTracks.editValueAt(i)->start());
452    }
453
454    sp<AMessage> notify = mNotify->dup();
455    notify->setInt32("what", kWhatSessionEstablished);
456    notify->post();
457
458    return OK;
459}
460
461status_t WifiDisplaySource::PlaybackSession::pause() {
462    updateLiveness();
463
464    (new AMessage(kWhatPause, this))->post();
465
466    return OK;
467}
468
469void WifiDisplaySource::PlaybackSession::destroyAsync() {
470    ALOGI("destroyAsync");
471
472    for (size_t i = 0; i < mTracks.size(); ++i) {
473        mTracks.valueAt(i)->stopAsync();
474    }
475}
476
477void WifiDisplaySource::PlaybackSession::onMessageReceived(
478        const sp<AMessage> &msg) {
479    switch (msg->what()) {
480        case kWhatConverterNotify:
481        {
482            if (mWeAreDead) {
483                ALOGV("dropping msg '%s' because we're dead",
484                      msg->debugString().c_str());
485
486                break;
487            }
488
489            int32_t what;
490            CHECK(msg->findInt32("what", &what));
491
492            size_t trackIndex;
493            CHECK(msg->findSize("trackIndex", &trackIndex));
494
495            if (what == Converter::kWhatAccessUnit) {
496                sp<ABuffer> accessUnit;
497                CHECK(msg->findBuffer("accessUnit", &accessUnit));
498
499                const sp<Track> &track = mTracks.valueFor(trackIndex);
500
501                status_t err = mMediaSender->queueAccessUnit(
502                        track->mediaSenderTrackIndex(),
503                        accessUnit);
504
505                if (err != OK) {
506                    notifySessionDead();
507                }
508                break;
509            } else if (what == Converter::kWhatEOS) {
510                CHECK_EQ(what, Converter::kWhatEOS);
511
512                ALOGI("output EOS on track %zu", trackIndex);
513
514                ssize_t index = mTracks.indexOfKey(trackIndex);
515                CHECK_GE(index, 0);
516
517                const sp<Converter> &converter =
518                    mTracks.valueAt(index)->converter();
519                looper()->unregisterHandler(converter->id());
520
521                mTracks.removeItemsAt(index);
522
523                if (mTracks.isEmpty()) {
524                    ALOGI("Reached EOS");
525                }
526            } else if (what != Converter::kWhatShutdownCompleted) {
527                CHECK_EQ(what, Converter::kWhatError);
528
529                status_t err;
530                CHECK(msg->findInt32("err", &err));
531
532                ALOGE("converter signaled error %d", err);
533
534                notifySessionDead();
535            }
536            break;
537        }
538
539        case kWhatMediaSenderNotify:
540        {
541            int32_t what;
542            CHECK(msg->findInt32("what", &what));
543
544            if (what == MediaSender::kWhatInitDone) {
545                status_t err;
546                CHECK(msg->findInt32("err", &err));
547
548                if (err == OK) {
549                    onMediaSenderInitialized();
550                } else {
551                    notifySessionDead();
552                }
553            } else if (what == MediaSender::kWhatError) {
554                notifySessionDead();
555            } else if (what == MediaSender::kWhatNetworkStall) {
556                size_t numBytesQueued;
557                CHECK(msg->findSize("numBytesQueued", &numBytesQueued));
558
559                if (mVideoTrackIndex >= 0) {
560                    const sp<Track> &videoTrack =
561                        mTracks.valueFor(mVideoTrackIndex);
562
563                    sp<Converter> converter = videoTrack->converter();
564                    if (converter != NULL) {
565                        converter->dropAFrame();
566                    }
567                }
568            } else if (what == MediaSender::kWhatInformSender) {
569                onSinkFeedback(msg);
570            } else {
571                TRESPASS();
572            }
573            break;
574        }
575
576        case kWhatTrackNotify:
577        {
578            int32_t what;
579            CHECK(msg->findInt32("what", &what));
580
581            size_t trackIndex;
582            CHECK(msg->findSize("trackIndex", &trackIndex));
583
584            if (what == Track::kWhatStopped) {
585                ALOGI("Track %zu stopped", trackIndex);
586
587                sp<Track> track = mTracks.valueFor(trackIndex);
588                looper()->unregisterHandler(track->id());
589                mTracks.removeItem(trackIndex);
590                track.clear();
591
592                if (!mTracks.isEmpty()) {
593                    ALOGI("not all tracks are stopped yet");
594                    break;
595                }
596
597                looper()->unregisterHandler(mMediaSender->id());
598                mMediaSender.clear();
599
600                sp<AMessage> notify = mNotify->dup();
601                notify->setInt32("what", kWhatSessionDestroyed);
602                notify->post();
603            }
604            break;
605        }
606
607        case kWhatPause:
608        {
609            if (mExtractor != NULL) {
610                ++mPullExtractorGeneration;
611                mFirstSampleTimeRealUs = -1ll;
612                mFirstSampleTimeUs = -1ll;
613            }
614
615            if (mPaused) {
616                break;
617            }
618
619            for (size_t i = 0; i < mTracks.size(); ++i) {
620                mTracks.editValueAt(i)->pause();
621            }
622
623            mPaused = true;
624            break;
625        }
626
627        case kWhatResume:
628        {
629            if (mExtractor != NULL) {
630                schedulePullExtractor();
631            }
632
633            if (!mPaused) {
634                break;
635            }
636
637            for (size_t i = 0; i < mTracks.size(); ++i) {
638                mTracks.editValueAt(i)->resume();
639            }
640
641            mPaused = false;
642            break;
643        }
644
645        case kWhatPullExtractorSample:
646        {
647            int32_t generation;
648            CHECK(msg->findInt32("generation", &generation));
649
650            if (generation != mPullExtractorGeneration) {
651                break;
652            }
653
654            mPullExtractorPending = false;
655
656            onPullExtractor();
657            break;
658        }
659
660        default:
661            TRESPASS();
662    }
663}
664
665void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) {
666    int64_t avgLatencyUs;
667    CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs));
668
669    int64_t maxLatencyUs;
670    CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs));
671
672    ALOGI("sink reports avg. latency of %lld ms (max %lld ms)",
673          avgLatencyUs / 1000ll,
674          maxLatencyUs / 1000ll);
675
676    if (mVideoTrackIndex >= 0) {
677        const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex);
678        sp<Converter> converter = videoTrack->converter();
679
680        if (converter != NULL) {
681            int32_t videoBitrate =
682                Converter::GetInt32Property("media.wfd.video-bitrate", -1);
683
684            char val[PROPERTY_VALUE_MAX];
685            if (videoBitrate < 0
686                    && property_get("media.wfd.video-bitrate", val, NULL)
687                    && !strcasecmp("adaptive", val)) {
688                videoBitrate = converter->getVideoBitrate();
689
690                if (avgLatencyUs > 300000ll) {
691                    videoBitrate *= 0.6;
692                } else if (avgLatencyUs < 100000ll) {
693                    videoBitrate *= 1.1;
694                }
695            }
696
697            if (videoBitrate > 0) {
698                if (videoBitrate < 500000) {
699                    videoBitrate = 500000;
700                } else if (videoBitrate > 10000000) {
701                    videoBitrate = 10000000;
702                }
703
704                if (videoBitrate != converter->getVideoBitrate()) {
705                    ALOGI("setting video bitrate to %d bps", videoBitrate);
706
707                    converter->setVideoBitrate(videoBitrate);
708                }
709            }
710        }
711
712        sp<RepeaterSource> repeaterSource = videoTrack->repeaterSource();
713        if (repeaterSource != NULL) {
714            double rateHz =
715                Converter::GetInt32Property(
716                        "media.wfd.video-framerate", -1);
717
718            char val[PROPERTY_VALUE_MAX];
719            if (rateHz < 0.0
720                    && property_get("media.wfd.video-framerate", val, NULL)
721                    && !strcasecmp("adaptive", val)) {
722                 rateHz = repeaterSource->getFrameRate();
723
724                if (avgLatencyUs > 300000ll) {
725                    rateHz *= 0.9;
726                } else if (avgLatencyUs < 200000ll) {
727                    rateHz *= 1.1;
728                }
729            }
730
731            if (rateHz > 0) {
732                if (rateHz < 5.0) {
733                    rateHz = 5.0;
734                } else if (rateHz > 30.0) {
735                    rateHz = 30.0;
736                }
737
738                if (rateHz != repeaterSource->getFrameRate()) {
739                    ALOGI("setting frame rate to %.2f Hz", rateHz);
740
741                    repeaterSource->setFrameRate(rateHz);
742                }
743            }
744        }
745    }
746}
747
748status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer(
749        bool enableAudio, bool enableVideo) {
750    mExtractor = new NuMediaExtractor;
751
752    status_t err = mExtractor->setDataSource(
753            NULL /* httpService */, mMediaPath.c_str());
754
755    if (err != OK) {
756        return err;
757    }
758
759    size_t n = mExtractor->countTracks();
760    bool haveAudio = false;
761    bool haveVideo = false;
762    for (size_t i = 0; i < n; ++i) {
763        sp<AMessage> format;
764        err = mExtractor->getTrackFormat(i, &format);
765
766        if (err != OK) {
767            continue;
768        }
769
770        AString mime;
771        CHECK(format->findString("mime", &mime));
772
773        bool isAudio = !strncasecmp(mime.c_str(), "audio/", 6);
774        bool isVideo = !strncasecmp(mime.c_str(), "video/", 6);
775
776        if (isAudio && enableAudio && !haveAudio) {
777            haveAudio = true;
778        } else if (isVideo && enableVideo && !haveVideo) {
779            haveVideo = true;
780        } else {
781            continue;
782        }
783
784        err = mExtractor->selectTrack(i);
785
786        size_t trackIndex = mTracks.size();
787
788        sp<AMessage> notify = new AMessage(kWhatTrackNotify, this);
789        notify->setSize("trackIndex", trackIndex);
790
791        sp<Track> track = new Track(notify, format);
792        looper()->registerHandler(track);
793
794        mTracks.add(trackIndex, track);
795
796        mExtractorTrackToInternalTrack.add(i, trackIndex);
797
798        if (isVideo) {
799            mVideoTrackIndex = trackIndex;
800        }
801
802        uint32_t flags = MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
803
804        ssize_t mediaSenderTrackIndex =
805            mMediaSender->addTrack(format, flags);
806        CHECK_GE(mediaSenderTrackIndex, 0);
807
808        track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
809
810        if ((haveAudio || !enableAudio) && (haveVideo || !enableVideo)) {
811            break;
812        }
813    }
814
815    return OK;
816}
817
818void WifiDisplaySource::PlaybackSession::schedulePullExtractor() {
819    if (mPullExtractorPending) {
820        return;
821    }
822
823    int64_t delayUs = 1000000; // default delay is 1 sec
824    int64_t sampleTimeUs;
825    status_t err = mExtractor->getSampleTime(&sampleTimeUs);
826
827    if (err == OK) {
828        int64_t nowUs = ALooper::GetNowUs();
829
830        if (mFirstSampleTimeRealUs < 0ll) {
831            mFirstSampleTimeRealUs = nowUs;
832            mFirstSampleTimeUs = sampleTimeUs;
833        }
834
835        int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs;
836        delayUs = whenUs - nowUs;
837    } else {
838        ALOGW("could not get sample time (%d)", err);
839    }
840
841    sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, this);
842    msg->setInt32("generation", mPullExtractorGeneration);
843    msg->post(delayUs);
844
845    mPullExtractorPending = true;
846}
847
848void WifiDisplaySource::PlaybackSession::onPullExtractor() {
849    sp<ABuffer> accessUnit = new ABuffer(1024 * 1024);
850    status_t err = mExtractor->readSampleData(accessUnit);
851    if (err != OK) {
852        // EOS.
853        return;
854    }
855
856    int64_t timeUs;
857    CHECK_EQ((status_t)OK, mExtractor->getSampleTime(&timeUs));
858
859    accessUnit->meta()->setInt64(
860            "timeUs", mFirstSampleTimeRealUs + timeUs - mFirstSampleTimeUs);
861
862    size_t trackIndex;
863    CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex));
864
865    sp<AMessage> msg = new AMessage(kWhatConverterNotify, this);
866
867    msg->setSize(
868            "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex));
869
870    msg->setInt32("what", Converter::kWhatAccessUnit);
871    msg->setBuffer("accessUnit", accessUnit);
872    msg->post();
873
874    mExtractor->advance();
875
876    schedulePullExtractor();
877}
878
879status_t WifiDisplaySource::PlaybackSession::setupPacketizer(
880        bool enableAudio,
881        bool usePCMAudio,
882        bool enableVideo,
883        VideoFormats::ResolutionType videoResolutionType,
884        size_t videoResolutionIndex,
885        VideoFormats::ProfileType videoProfileType,
886        VideoFormats::LevelType videoLevelType) {
887    CHECK(enableAudio || enableVideo);
888
889    if (!mMediaPath.empty()) {
890        return setupMediaPacketizer(enableAudio, enableVideo);
891    }
892
893    if (enableVideo) {
894        status_t err = addVideoSource(
895                videoResolutionType, videoResolutionIndex, videoProfileType,
896                videoLevelType);
897
898        if (err != OK) {
899            return err;
900        }
901    }
902
903    if (!enableAudio) {
904        return OK;
905    }
906
907    return addAudioSource(usePCMAudio);
908}
909
910status_t WifiDisplaySource::PlaybackSession::addSource(
911        bool isVideo, const sp<MediaSource> &source, bool isRepeaterSource,
912        bool usePCMAudio, unsigned profileIdc, unsigned levelIdc,
913        unsigned constraintSet, size_t *numInputBuffers) {
914    CHECK(!usePCMAudio || !isVideo);
915    CHECK(!isRepeaterSource || isVideo);
916    CHECK(!profileIdc || isVideo);
917    CHECK(!levelIdc || isVideo);
918    CHECK(!constraintSet || isVideo);
919
920    sp<ALooper> pullLooper = new ALooper;
921    pullLooper->setName("pull_looper");
922
923    pullLooper->start(
924            false /* runOnCallingThread */,
925            false /* canCallJava */,
926            PRIORITY_AUDIO);
927
928    sp<ALooper> codecLooper = new ALooper;
929    codecLooper->setName("codec_looper");
930
931    codecLooper->start(
932            false /* runOnCallingThread */,
933            false /* canCallJava */,
934            PRIORITY_AUDIO);
935
936    size_t trackIndex;
937
938    sp<AMessage> notify;
939
940    trackIndex = mTracks.size();
941
942    sp<AMessage> format;
943    status_t err = convertMetaDataToMessage(source->getFormat(), &format);
944    CHECK_EQ(err, (status_t)OK);
945
946    if (isVideo) {
947        format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC);
948        format->setInt32(
949                "android._input-metadata-buffer-type", kMetadataBufferTypeANWBuffer);
950        format->setInt32("android._store-metadata-in-buffers-output", (mHDCP != NULL)
951                && (mHDCP->getCaps() & HDCPModule::HDCP_CAPS_ENCRYPT_NATIVE));
952        format->setInt32(
953                "color-format", OMX_COLOR_FormatAndroidOpaque);
954        format->setInt32("profile-idc", profileIdc);
955        format->setInt32("level-idc", levelIdc);
956        format->setInt32("constraint-set", constraintSet);
957    } else {
958        if (usePCMAudio) {
959            format->setInt32("pcm-encoding", kAudioEncodingPcm16bit);
960            format->setString("mime", MEDIA_MIMETYPE_AUDIO_RAW);
961        } else {
962            format->setString("mime", MEDIA_MIMETYPE_AUDIO_AAC);
963        }
964    }
965
966    notify = new AMessage(kWhatConverterNotify, this);
967    notify->setSize("trackIndex", trackIndex);
968
969    sp<Converter> converter = new Converter(notify, codecLooper, format);
970
971    looper()->registerHandler(converter);
972
973    err = converter->init();
974    if (err != OK) {
975        ALOGE("%s converter returned err %d", isVideo ? "video" : "audio", err);
976
977        looper()->unregisterHandler(converter->id());
978        return err;
979    }
980
981    notify = new AMessage(Converter::kWhatMediaPullerNotify, converter);
982    notify->setSize("trackIndex", trackIndex);
983
984    sp<MediaPuller> puller = new MediaPuller(source, notify);
985    pullLooper->registerHandler(puller);
986
987    if (numInputBuffers != NULL) {
988        *numInputBuffers = converter->getInputBufferCount();
989    }
990
991    notify = new AMessage(kWhatTrackNotify, this);
992    notify->setSize("trackIndex", trackIndex);
993
994    sp<Track> track = new Track(
995            notify, pullLooper, codecLooper, puller, converter);
996
997    if (isRepeaterSource) {
998        track->setRepeaterSource(static_cast<RepeaterSource *>(source.get()));
999    }
1000
1001    looper()->registerHandler(track);
1002
1003    mTracks.add(trackIndex, track);
1004
1005    if (isVideo) {
1006        mVideoTrackIndex = trackIndex;
1007    }
1008
1009    uint32_t flags = 0;
1010    if (converter->needToManuallyPrependSPSPPS()) {
1011        flags |= MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
1012    }
1013
1014    ssize_t mediaSenderTrackIndex =
1015        mMediaSender->addTrack(converter->getOutputFormat(), flags);
1016    CHECK_GE(mediaSenderTrackIndex, 0);
1017
1018    track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
1019
1020    return OK;
1021}
1022
1023status_t WifiDisplaySource::PlaybackSession::addVideoSource(
1024        VideoFormats::ResolutionType videoResolutionType,
1025        size_t videoResolutionIndex,
1026        VideoFormats::ProfileType videoProfileType,
1027        VideoFormats::LevelType videoLevelType) {
1028    size_t width, height, framesPerSecond;
1029    bool interlaced;
1030    CHECK(VideoFormats::GetConfiguration(
1031                videoResolutionType,
1032                videoResolutionIndex,
1033                &width,
1034                &height,
1035                &framesPerSecond,
1036                &interlaced));
1037
1038    unsigned profileIdc, levelIdc, constraintSet;
1039    CHECK(VideoFormats::GetProfileLevel(
1040                videoProfileType,
1041                videoLevelType,
1042                &profileIdc,
1043                &levelIdc,
1044                &constraintSet));
1045
1046    sp<SurfaceMediaSource> source = new SurfaceMediaSource(width, height);
1047
1048    source->setUseAbsoluteTimestamps();
1049
1050    sp<RepeaterSource> videoSource =
1051        new RepeaterSource(source, framesPerSecond);
1052
1053    size_t numInputBuffers;
1054    status_t err = addSource(
1055            true /* isVideo */, videoSource, true /* isRepeaterSource */,
1056            false /* usePCMAudio */, profileIdc, levelIdc, constraintSet,
1057            &numInputBuffers);
1058
1059    if (err != OK) {
1060        return err;
1061    }
1062
1063    err = source->setMaxAcquiredBufferCount(numInputBuffers);
1064    CHECK_EQ(err, (status_t)OK);
1065
1066    mProducer = source->getProducer();
1067
1068    return OK;
1069}
1070
1071status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) {
1072    sp<AudioSource> audioSource = new AudioSource(
1073            AUDIO_SOURCE_REMOTE_SUBMIX,
1074            mOpPackageName,
1075            48000 /* sampleRate */,
1076            2 /* channelCount */);
1077
1078    if (audioSource->initCheck() == OK) {
1079        return addSource(
1080                false /* isVideo */, audioSource, false /* isRepeaterSource */,
1081                usePCMAudio, 0 /* profileIdc */, 0 /* levelIdc */,
1082                0 /* constraintSet */, NULL /* numInputBuffers */);
1083    }
1084
1085    ALOGW("Unable to instantiate audio source");
1086
1087    return OK;
1088}
1089
1090sp<IGraphicBufferProducer> WifiDisplaySource::PlaybackSession::getSurfaceTexture() {
1091    return mProducer;
1092}
1093
1094void WifiDisplaySource::PlaybackSession::requestIDRFrame() {
1095    for (size_t i = 0; i < mTracks.size(); ++i) {
1096        const sp<Track> &track = mTracks.valueAt(i);
1097
1098        track->requestIDRFrame();
1099    }
1100}
1101
1102void WifiDisplaySource::PlaybackSession::notifySessionDead() {
1103    // Inform WifiDisplaySource of our premature death (wish).
1104    sp<AMessage> notify = mNotify->dup();
1105    notify->setInt32("what", kWhatSessionDead);
1106    notify->post();
1107
1108    mWeAreDead = true;
1109}
1110
1111}  // namespace android
1112
1113