NuPlayer.cpp revision f4c0a94ab32541611f5ed6d407fc25b394c1988b 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerDecoder.h"
25#include "NuPlayerDecoderPassThrough.h"
26#include "NuPlayerDriver.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29#include "RTSPSource.h"
30#include "StreamingSource.h"
31#include "GenericSource.h"
32#include "TextDescriptions.h"
33
34#include "ATSParser.h"
35
36#include <media/stagefright/foundation/hexdump.h>
37#include <media/stagefright/foundation/ABuffer.h>
38#include <media/stagefright/foundation/ADebug.h>
39#include <media/stagefright/foundation/AMessage.h>
40#include <media/stagefright/MediaBuffer.h>
41#include <media/stagefright/MediaDefs.h>
42#include <media/stagefright/MediaErrors.h>
43#include <media/stagefright/MetaData.h>
44#include <gui/IGraphicBufferProducer.h>
45
46#include "avc_utils.h"
47
48#include "ESDS.h"
49#include <media/stagefright/Utils.h>
50
51namespace android {
52
53struct NuPlayer::Action : public RefBase {
54    Action() {}
55
56    virtual void execute(NuPlayer *player) = 0;
57
58private:
59    DISALLOW_EVIL_CONSTRUCTORS(Action);
60};
61
62struct NuPlayer::SeekAction : public Action {
63    SeekAction(int64_t seekTimeUs)
64        : mSeekTimeUs(seekTimeUs) {
65    }
66
67    virtual void execute(NuPlayer *player) {
68        player->performSeek(mSeekTimeUs);
69    }
70
71private:
72    int64_t mSeekTimeUs;
73
74    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
75};
76
77struct NuPlayer::SetSurfaceAction : public Action {
78    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
79        : mWrapper(wrapper) {
80    }
81
82    virtual void execute(NuPlayer *player) {
83        player->performSetSurface(mWrapper);
84    }
85
86private:
87    sp<NativeWindowWrapper> mWrapper;
88
89    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
90};
91
92struct NuPlayer::ShutdownDecoderAction : public Action {
93    ShutdownDecoderAction(bool audio, bool video)
94        : mAudio(audio),
95          mVideo(video) {
96    }
97
98    virtual void execute(NuPlayer *player) {
99        player->performDecoderShutdown(mAudio, mVideo);
100    }
101
102private:
103    bool mAudio;
104    bool mVideo;
105
106    DISALLOW_EVIL_CONSTRUCTORS(ShutdownDecoderAction);
107};
108
109struct NuPlayer::PostMessageAction : public Action {
110    PostMessageAction(const sp<AMessage> &msg)
111        : mMessage(msg) {
112    }
113
114    virtual void execute(NuPlayer *) {
115        mMessage->post();
116    }
117
118private:
119    sp<AMessage> mMessage;
120
121    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
122};
123
124// Use this if there's no state necessary to save in order to execute
125// the action.
126struct NuPlayer::SimpleAction : public Action {
127    typedef void (NuPlayer::*ActionFunc)();
128
129    SimpleAction(ActionFunc func)
130        : mFunc(func) {
131    }
132
133    virtual void execute(NuPlayer *player) {
134        (player->*mFunc)();
135    }
136
137private:
138    ActionFunc mFunc;
139
140    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
141};
142
143////////////////////////////////////////////////////////////////////////////////
144
145NuPlayer::NuPlayer()
146    : mUIDValid(false),
147      mSourceFlags(0),
148      mVideoIsAVC(false),
149      mOffloadAudio(false),
150      mAudioEOS(false),
151      mVideoEOS(false),
152      mScanSourcesPending(false),
153      mScanSourcesGeneration(0),
154      mPollDurationGeneration(0),
155      mTimedTextGeneration(0),
156      mTimeDiscontinuityPending(false),
157      mFlushingAudio(NONE),
158      mFlushingVideo(NONE),
159      mSkipRenderingAudioUntilMediaTimeUs(-1ll),
160      mSkipRenderingVideoUntilMediaTimeUs(-1ll),
161      mVideoLateByUs(0ll),
162      mNumFramesTotal(0ll),
163      mNumFramesDropped(0ll),
164      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
165      mStarted(false) {
166}
167
168NuPlayer::~NuPlayer() {
169}
170
171void NuPlayer::setUID(uid_t uid) {
172    mUIDValid = true;
173    mUID = uid;
174}
175
176void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
177    mDriver = driver;
178}
179
180void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
181    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
182
183    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
184
185    msg->setObject("source", new StreamingSource(notify, source));
186    msg->post();
187}
188
189static bool IsHTTPLiveURL(const char *url) {
190    if (!strncasecmp("http://", url, 7)
191            || !strncasecmp("https://", url, 8)
192            || !strncasecmp("file://", url, 7)) {
193        size_t len = strlen(url);
194        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
195            return true;
196        }
197
198        if (strstr(url,"m3u8")) {
199            return true;
200        }
201    }
202
203    return false;
204}
205
206void NuPlayer::setDataSourceAsync(
207        const sp<IMediaHTTPService> &httpService,
208        const char *url,
209        const KeyedVector<String8, String8> *headers) {
210
211    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
212    size_t len = strlen(url);
213
214    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
215
216    sp<Source> source;
217    if (IsHTTPLiveURL(url)) {
218        source = new HTTPLiveSource(notify, httpService, url, headers);
219    } else if (!strncasecmp(url, "rtsp://", 7)) {
220        source = new RTSPSource(
221                notify, httpService, url, headers, mUIDValid, mUID);
222    } else if ((!strncasecmp(url, "http://", 7)
223                || !strncasecmp(url, "https://", 8))
224                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
225                    || strstr(url, ".sdp?"))) {
226        source = new RTSPSource(
227                notify, httpService, url, headers, mUIDValid, mUID, true);
228    } else {
229        sp<GenericSource> genericSource =
230                new GenericSource(notify, mUIDValid, mUID);
231        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
232        // The correct flags will be updated in Source::kWhatFlagsChanged
233        // handler when  GenericSource is prepared.
234
235        status_t err = genericSource->setDataSource(httpService, url, headers);
236
237        if (err == OK) {
238            source = genericSource;
239        } else {
240            ALOGE("Failed to set data source!");
241        }
242    }
243    msg->setObject("source", source);
244    msg->post();
245}
246
247void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
248    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
249
250    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
251
252    sp<GenericSource> source =
253            new GenericSource(notify, mUIDValid, mUID);
254
255    status_t err = source->setDataSource(fd, offset, length);
256
257    if (err != OK) {
258        ALOGE("Failed to set data source!");
259        source = NULL;
260    }
261
262    msg->setObject("source", source);
263    msg->post();
264}
265
266void NuPlayer::prepareAsync() {
267    (new AMessage(kWhatPrepare, id()))->post();
268}
269
270void NuPlayer::setVideoSurfaceTextureAsync(
271        const sp<IGraphicBufferProducer> &bufferProducer) {
272    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id());
273
274    if (bufferProducer == NULL) {
275        msg->setObject("native-window", NULL);
276    } else {
277        msg->setObject(
278                "native-window",
279                new NativeWindowWrapper(
280                    new Surface(bufferProducer)));
281    }
282
283    msg->post();
284}
285
286void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
287    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id());
288    msg->setObject("sink", sink);
289    msg->post();
290}
291
292void NuPlayer::start() {
293    (new AMessage(kWhatStart, id()))->post();
294}
295
296void NuPlayer::pause() {
297    (new AMessage(kWhatPause, id()))->post();
298}
299
300void NuPlayer::resume() {
301    (new AMessage(kWhatResume, id()))->post();
302}
303
304void NuPlayer::resetAsync() {
305    (new AMessage(kWhatReset, id()))->post();
306}
307
308void NuPlayer::seekToAsync(int64_t seekTimeUs) {
309    sp<AMessage> msg = new AMessage(kWhatSeek, id());
310    msg->setInt64("seekTimeUs", seekTimeUs);
311    msg->post();
312}
313
314// static
315bool NuPlayer::IsFlushingState(FlushStatus state, bool *needShutdown) {
316    switch (state) {
317        case FLUSHING_DECODER:
318            if (needShutdown != NULL) {
319                *needShutdown = false;
320            }
321            return true;
322
323        case FLUSHING_DECODER_SHUTDOWN:
324            if (needShutdown != NULL) {
325                *needShutdown = true;
326            }
327            return true;
328
329        default:
330            return false;
331    }
332}
333
334void NuPlayer::writeTrackInfo(
335        Parcel* reply, const sp<AMessage> format) const {
336    int32_t trackType;
337    CHECK(format->findInt32("type", &trackType));
338
339    AString lang;
340    CHECK(format->findString("language", &lang));
341
342    reply->writeInt32(2); // write something non-zero
343    reply->writeInt32(trackType);
344    reply->writeString16(String16(lang.c_str()));
345
346    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
347        AString mime;
348        CHECK(format->findString("mime", &mime));
349
350        int32_t isAuto, isDefault, isForced;
351        CHECK(format->findInt32("auto", &isAuto));
352        CHECK(format->findInt32("default", &isDefault));
353        CHECK(format->findInt32("forced", &isForced));
354
355        reply->writeString16(String16(mime.c_str()));
356        reply->writeInt32(isAuto);
357        reply->writeInt32(isDefault);
358        reply->writeInt32(isForced);
359    }
360}
361
362void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
363    switch (msg->what()) {
364        case kWhatSetDataSource:
365        {
366            ALOGV("kWhatSetDataSource");
367
368            CHECK(mSource == NULL);
369
370            status_t err = OK;
371            sp<RefBase> obj;
372            CHECK(msg->findObject("source", &obj));
373            if (obj != NULL) {
374                mSource = static_cast<Source *>(obj.get());
375                looper()->registerHandler(mSource);
376            } else {
377                err = UNKNOWN_ERROR;
378            }
379
380            CHECK(mDriver != NULL);
381            sp<NuPlayerDriver> driver = mDriver.promote();
382            if (driver != NULL) {
383                driver->notifySetDataSourceCompleted(err);
384            }
385            break;
386        }
387
388        case kWhatPrepare:
389        {
390            mSource->prepareAsync();
391            break;
392        }
393
394        case kWhatGetTrackInfo:
395        {
396            uint32_t replyID;
397            CHECK(msg->senderAwaitsResponse(&replyID));
398
399            Parcel* reply;
400            CHECK(msg->findPointer("reply", (void**)&reply));
401
402            size_t inbandTracks = 0;
403            if (mSource != NULL) {
404                inbandTracks = mSource->getTrackCount();
405            }
406
407            size_t ccTracks = 0;
408            if (mCCDecoder != NULL) {
409                ccTracks = mCCDecoder->getTrackCount();
410            }
411
412            // total track count
413            reply->writeInt32(inbandTracks + ccTracks);
414
415            // write inband tracks
416            for (size_t i = 0; i < inbandTracks; ++i) {
417                writeTrackInfo(reply, mSource->getTrackInfo(i));
418            }
419
420            // write CC track
421            for (size_t i = 0; i < ccTracks; ++i) {
422                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
423            }
424
425            sp<AMessage> response = new AMessage;
426            response->postReply(replyID);
427            break;
428        }
429
430        case kWhatSelectTrack:
431        {
432            uint32_t replyID;
433            CHECK(msg->senderAwaitsResponse(&replyID));
434
435            size_t trackIndex;
436            int32_t select;
437            CHECK(msg->findSize("trackIndex", &trackIndex));
438            CHECK(msg->findInt32("select", &select));
439
440            status_t err = INVALID_OPERATION;
441
442            size_t inbandTracks = 0;
443            if (mSource != NULL) {
444                inbandTracks = mSource->getTrackCount();
445            }
446            size_t ccTracks = 0;
447            if (mCCDecoder != NULL) {
448                ccTracks = mCCDecoder->getTrackCount();
449            }
450
451            if (trackIndex < inbandTracks) {
452                err = mSource->selectTrack(trackIndex, select);
453
454                if (!select && err == OK) {
455                    int32_t type;
456                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
457                    if (info != NULL
458                            && info->findInt32("type", &type)
459                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
460                        ++mTimedTextGeneration;
461                    }
462                }
463            } else {
464                trackIndex -= inbandTracks;
465
466                if (trackIndex < ccTracks) {
467                    err = mCCDecoder->selectTrack(trackIndex, select);
468                }
469            }
470
471            sp<AMessage> response = new AMessage;
472            response->setInt32("err", err);
473
474            response->postReply(replyID);
475            break;
476        }
477
478        case kWhatPollDuration:
479        {
480            int32_t generation;
481            CHECK(msg->findInt32("generation", &generation));
482
483            if (generation != mPollDurationGeneration) {
484                // stale
485                break;
486            }
487
488            int64_t durationUs;
489            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
490                sp<NuPlayerDriver> driver = mDriver.promote();
491                if (driver != NULL) {
492                    driver->notifyDuration(durationUs);
493                }
494            }
495
496            msg->post(1000000ll);  // poll again in a second.
497            break;
498        }
499
500        case kWhatSetVideoNativeWindow:
501        {
502            ALOGV("kWhatSetVideoNativeWindow");
503
504            mDeferredActions.push_back(
505                    new ShutdownDecoderAction(
506                        false /* audio */, true /* video */));
507
508            sp<RefBase> obj;
509            CHECK(msg->findObject("native-window", &obj));
510
511            mDeferredActions.push_back(
512                    new SetSurfaceAction(
513                        static_cast<NativeWindowWrapper *>(obj.get())));
514
515            if (obj != NULL) {
516                // If there is a new surface texture, instantiate decoders
517                // again if possible.
518                mDeferredActions.push_back(
519                        new SimpleAction(&NuPlayer::performScanSources));
520            }
521
522            processDeferredActions();
523            break;
524        }
525
526        case kWhatSetAudioSink:
527        {
528            ALOGV("kWhatSetAudioSink");
529
530            sp<RefBase> obj;
531            CHECK(msg->findObject("sink", &obj));
532
533            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
534            break;
535        }
536
537        case kWhatStart:
538        {
539            ALOGV("kWhatStart");
540
541            mVideoIsAVC = false;
542            mOffloadAudio = false;
543            mAudioEOS = false;
544            mVideoEOS = false;
545            mSkipRenderingAudioUntilMediaTimeUs = -1;
546            mSkipRenderingVideoUntilMediaTimeUs = -1;
547            mVideoLateByUs = 0;
548            mNumFramesTotal = 0;
549            mNumFramesDropped = 0;
550            mStarted = true;
551
552            /* instantiate decoders now for secure playback */
553            if (mSourceFlags & Source::FLAG_SECURE) {
554                if (mNativeWindow != NULL) {
555                    instantiateDecoder(false, &mVideoDecoder);
556                }
557
558                if (mAudioSink != NULL) {
559                    instantiateDecoder(true, &mAudioDecoder);
560                }
561            }
562
563            mSource->start();
564
565            uint32_t flags = 0;
566
567            if (mSource->isRealTime()) {
568                flags |= Renderer::FLAG_REAL_TIME;
569            }
570
571            sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
572            audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
573            if (mAudioSink != NULL) {
574                streamType = mAudioSink->getAudioStreamType();
575            }
576
577            sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
578
579            mOffloadAudio =
580                canOffloadStream(audioMeta, (videoFormat != NULL),
581                                 true /* is_streaming */, streamType);
582            if (mOffloadAudio) {
583                flags |= Renderer::FLAG_OFFLOAD_AUDIO;
584            }
585
586            mRenderer = new Renderer(
587                    mAudioSink,
588                    new AMessage(kWhatRendererNotify, id()),
589                    flags);
590
591            mRendererLooper = new ALooper;
592            mRendererLooper->setName("NuPlayerRenderer");
593            mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
594            mRendererLooper->registerHandler(mRenderer);
595
596            postScanSources();
597            break;
598        }
599
600        case kWhatScanSources:
601        {
602            int32_t generation;
603            CHECK(msg->findInt32("generation", &generation));
604            if (generation != mScanSourcesGeneration) {
605                // Drop obsolete msg.
606                break;
607            }
608
609            mScanSourcesPending = false;
610
611            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
612                 mAudioDecoder != NULL, mVideoDecoder != NULL);
613
614            bool mHadAnySourcesBefore =
615                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
616
617            if (mNativeWindow != NULL) {
618                instantiateDecoder(false, &mVideoDecoder);
619            }
620
621            if (mAudioSink != NULL) {
622                instantiateDecoder(true, &mAudioDecoder);
623            }
624
625            if (!mHadAnySourcesBefore
626                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
627                // This is the first time we've found anything playable.
628
629                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
630                    schedulePollDuration();
631                }
632            }
633
634            status_t err;
635            if ((err = mSource->feedMoreTSData()) != OK) {
636                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
637                    // We're not currently decoding anything (no audio or
638                    // video tracks found) and we just ran out of input data.
639
640                    if (err == ERROR_END_OF_STREAM) {
641                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
642                    } else {
643                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
644                    }
645                }
646                break;
647            }
648
649            if ((mAudioDecoder == NULL && mAudioSink != NULL)
650                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
651                msg->post(100000ll);
652                mScanSourcesPending = true;
653            }
654            break;
655        }
656
657        case kWhatVideoNotify:
658        case kWhatAudioNotify:
659        {
660            bool audio = msg->what() == kWhatAudioNotify;
661
662            int32_t what;
663            CHECK(msg->findInt32("what", &what));
664
665            if (what == Decoder::kWhatFillThisBuffer) {
666                status_t err = feedDecoderInputData(
667                        audio, msg);
668
669                if (err == -EWOULDBLOCK) {
670                    if (mSource->feedMoreTSData() == OK) {
671                        msg->post(10000ll);
672                    }
673                }
674            } else if (what == Decoder::kWhatEOS) {
675                int32_t err;
676                CHECK(msg->findInt32("err", &err));
677
678                if (err == ERROR_END_OF_STREAM) {
679                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
680                } else {
681                    ALOGV("got %s decoder EOS w/ error %d",
682                         audio ? "audio" : "video",
683                         err);
684                }
685
686                mRenderer->queueEOS(audio, err);
687            } else if (what == Decoder::kWhatFlushCompleted) {
688                bool needShutdown;
689
690                if (audio) {
691                    CHECK(IsFlushingState(mFlushingAudio, &needShutdown));
692                    mFlushingAudio = FLUSHED;
693                } else {
694                    CHECK(IsFlushingState(mFlushingVideo, &needShutdown));
695                    mFlushingVideo = FLUSHED;
696
697                    mVideoLateByUs = 0;
698                }
699
700                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
701
702                if (needShutdown) {
703                    ALOGV("initiating %s decoder shutdown",
704                         audio ? "audio" : "video");
705
706                    (audio ? mAudioDecoder : mVideoDecoder)->initiateShutdown();
707
708                    if (audio) {
709                        mFlushingAudio = SHUTTING_DOWN_DECODER;
710                    } else {
711                        mFlushingVideo = SHUTTING_DOWN_DECODER;
712                    }
713                }
714
715                finishFlushIfPossible();
716            } else if (what == Decoder::kWhatOutputFormatChanged) {
717                sp<AMessage> format;
718                CHECK(msg->findMessage("format", &format));
719
720                if (audio) {
721                    int32_t numChannels;
722                    CHECK(format->findInt32(
723                                "channel-count", &numChannels));
724
725                    int32_t sampleRate;
726                    CHECK(format->findInt32("sample-rate", &sampleRate));
727
728                    ALOGV("Audio output format changed to %d Hz, %d channels",
729                         sampleRate, numChannels);
730
731                    mAudioSink->close();
732
733                    uint32_t flags;
734                    int64_t durationUs;
735                    // FIXME: we should handle the case where the video decoder
736                    // is created after we receive the format change indication.
737                    // Current code will just make that we select deep buffer
738                    // with video which should not be a problem as it should
739                    // not prevent from keeping A/V sync.
740                    if (mVideoDecoder == NULL &&
741                            mSource->getDuration(&durationUs) == OK &&
742                            durationUs
743                                > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
744                        flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
745                    } else {
746                        flags = AUDIO_OUTPUT_FLAG_NONE;
747                    }
748
749                    int32_t channelMask;
750                    if (!format->findInt32("channel-mask", &channelMask)) {
751                        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
752                    }
753
754                    if (mOffloadAudio) {
755                        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
756                        audio_offload_info_t offloadInfo =
757                                AUDIO_INFO_INITIALIZER;
758
759                        AString mime;
760                        CHECK(format->findString("mime", &mime));
761
762                        status_t err =
763                            mapMimeToAudioFormat(audioFormat, mime.c_str());
764                        if (err != OK) {
765                            ALOGE("Couldn't map mime \"%s\" to a valid "
766                                    "audio_format", mime.c_str());
767                            mOffloadAudio = false;
768                        } else {
769                            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
770                                    mime.c_str(), audioFormat);
771
772                            int32_t aacProfile = -1;
773                            if (audioFormat == AUDIO_FORMAT_AAC
774                                    && format->findInt32("aac-profile", &aacProfile)) {
775                                // Redefine AAC format as per aac profile
776                                mapAACProfileToAudioFormat(
777                                        audioFormat,
778                                        aacProfile);
779                            }
780
781                            flags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
782
783                            offloadInfo.duration_us = -1;
784                            format->findInt64(
785                                    "durationUs", &offloadInfo.duration_us);
786
787                            int avgBitRate = -1;
788                            format->findInt32("bit-rate", &avgBitRate);
789
790                            offloadInfo.sample_rate = sampleRate;
791                            offloadInfo.channel_mask = channelMask;
792                            offloadInfo.format = audioFormat;
793                            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
794                            offloadInfo.bit_rate = avgBitRate;
795                            offloadInfo.has_video = (mVideoDecoder != NULL);
796                            offloadInfo.is_streaming = true;
797
798                            ALOGV("try to open AudioSink in offload mode");
799                            err = mAudioSink->open(
800                                    sampleRate,
801                                    numChannels,
802                                    (audio_channel_mask_t)channelMask,
803                                    audioFormat,
804                                    8 /* bufferCount */,
805                                    &NuPlayer::Renderer::AudioSinkCallback,
806                                    mRenderer.get(),
807                                    (audio_output_flags_t)flags,
808                                    &offloadInfo);
809
810                            if (err == OK) {
811                                // If the playback is offloaded to h/w, we pass
812                                // the HAL some metadata information.
813                                // We don't want to do this for PCM because it
814                                // will be going through the AudioFlinger mixer
815                                // before reaching the hardware.
816                                sp<MetaData> audioMeta =
817                                    mSource->getFormatMeta(true /* audio */);
818                                sendMetaDataToHal(mAudioSink, audioMeta);
819
820                                err = mAudioSink->start();
821                            }
822                        }
823
824                        if (err != OK) {
825                            // Clean up, fall back to non offload mode.
826                            mAudioSink->close();
827                            mAudioDecoder.clear();
828                            mRenderer->signalDisableOffloadAudio();
829                            mOffloadAudio = false;
830
831                            instantiateDecoder(
832                                    true /* audio */, &mAudioDecoder);
833                        }
834                    }
835
836                    if (!mOffloadAudio) {
837                        flags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
838                        ALOGV("open AudioSink in NON-offload mode");
839                        CHECK_EQ(mAudioSink->open(
840                                    sampleRate,
841                                    numChannels,
842                                    (audio_channel_mask_t)channelMask,
843                                    AUDIO_FORMAT_PCM_16_BIT,
844                                    8 /* bufferCount */,
845                                    NULL,
846                                    NULL,
847                                    (audio_output_flags_t)flags),
848                                 (status_t)OK);
849                        mAudioSink->start();
850                    }
851
852                    mRenderer->signalAudioSinkChanged();
853                } else {
854                    // video
855                    sp<AMessage> inputFormat =
856                            mSource->getFormat(false /* audio */);
857
858                    updateVideoSize(inputFormat, format);
859                }
860            } else if (what == Decoder::kWhatShutdownCompleted) {
861                ALOGV("%s shutdown completed", audio ? "audio" : "video");
862                if (audio) {
863                    mAudioDecoder.clear();
864
865                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
866                    mFlushingAudio = SHUT_DOWN;
867                } else {
868                    mVideoDecoder.clear();
869
870                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
871                    mFlushingVideo = SHUT_DOWN;
872                }
873
874                finishFlushIfPossible();
875            } else if (what == Decoder::kWhatError) {
876                ALOGE("Received error from %s decoder, aborting playback.",
877                     audio ? "audio" : "video");
878
879                status_t err;
880                if (!msg->findInt32("err", &err)) {
881                    err = UNKNOWN_ERROR;
882                }
883                mRenderer->queueEOS(audio, err);
884            } else if (what == Decoder::kWhatDrainThisBuffer) {
885                renderBuffer(audio, msg);
886            } else {
887                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
888                      what,
889                      what >> 24,
890                      (what >> 16) & 0xff,
891                      (what >> 8) & 0xff,
892                      what & 0xff);
893            }
894
895            break;
896        }
897
898        case kWhatRendererNotify:
899        {
900            int32_t what;
901            CHECK(msg->findInt32("what", &what));
902
903            if (what == Renderer::kWhatEOS) {
904                int32_t audio;
905                CHECK(msg->findInt32("audio", &audio));
906
907                int32_t finalResult;
908                CHECK(msg->findInt32("finalResult", &finalResult));
909
910                if (audio) {
911                    mAudioEOS = true;
912                } else {
913                    mVideoEOS = true;
914                }
915
916                if (finalResult == ERROR_END_OF_STREAM) {
917                    ALOGV("reached %s EOS", audio ? "audio" : "video");
918                } else {
919                    ALOGE("%s track encountered an error (%d)",
920                         audio ? "audio" : "video", finalResult);
921
922                    notifyListener(
923                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
924                }
925
926                if ((mAudioEOS || mAudioDecoder == NULL)
927                        && (mVideoEOS || mVideoDecoder == NULL)) {
928                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
929                }
930            } else if (what == Renderer::kWhatPosition) {
931                int64_t positionUs;
932                CHECK(msg->findInt64("positionUs", &positionUs));
933
934                CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs));
935
936                if (mDriver != NULL) {
937                    sp<NuPlayerDriver> driver = mDriver.promote();
938                    if (driver != NULL) {
939                        driver->notifyPosition(positionUs);
940
941                        driver->notifyFrameStats(
942                                mNumFramesTotal, mNumFramesDropped);
943                    }
944                }
945            } else if (what == Renderer::kWhatFlushComplete) {
946                int32_t audio;
947                CHECK(msg->findInt32("audio", &audio));
948
949                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
950            } else if (what == Renderer::kWhatVideoRenderingStart) {
951                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
952            } else if (what == Renderer::kWhatMediaRenderingStart) {
953                ALOGV("media rendering started");
954                notifyListener(MEDIA_STARTED, 0, 0);
955            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
956                ALOGV("Tear down audio offload, fall back to s/w path");
957                int64_t positionUs;
958                CHECK(msg->findInt64("positionUs", &positionUs));
959                mAudioSink->close();
960                mAudioDecoder.clear();
961                mRenderer->flush(true /* audio */);
962                if (mVideoDecoder != NULL) {
963                    mRenderer->flush(false /* audio */);
964                }
965                mRenderer->signalDisableOffloadAudio();
966                mOffloadAudio = false;
967
968                performSeek(positionUs);
969                instantiateDecoder(true /* audio */, &mAudioDecoder);
970            }
971            break;
972        }
973
974        case kWhatMoreDataQueued:
975        {
976            break;
977        }
978
979        case kWhatReset:
980        {
981            ALOGV("kWhatReset");
982
983            mDeferredActions.push_back(
984                    new ShutdownDecoderAction(
985                        true /* audio */, true /* video */));
986
987            mDeferredActions.push_back(
988                    new SimpleAction(&NuPlayer::performReset));
989
990            processDeferredActions();
991            break;
992        }
993
994        case kWhatSeek:
995        {
996            int64_t seekTimeUs;
997            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
998
999            ALOGV("kWhatSeek seekTimeUs=%lld us", seekTimeUs);
1000
1001            mDeferredActions.push_back(
1002                    new SimpleAction(&NuPlayer::performDecoderFlush));
1003
1004            mDeferredActions.push_back(new SeekAction(seekTimeUs));
1005
1006            processDeferredActions();
1007            break;
1008        }
1009
1010        case kWhatPause:
1011        {
1012            CHECK(mRenderer != NULL);
1013            mSource->pause();
1014            mRenderer->pause();
1015            break;
1016        }
1017
1018        case kWhatResume:
1019        {
1020            CHECK(mRenderer != NULL);
1021            mSource->resume();
1022            mRenderer->resume();
1023            break;
1024        }
1025
1026        case kWhatSourceNotify:
1027        {
1028            onSourceNotify(msg);
1029            break;
1030        }
1031
1032        case kWhatClosedCaptionNotify:
1033        {
1034            onClosedCaptionNotify(msg);
1035            break;
1036        }
1037
1038        default:
1039            TRESPASS();
1040            break;
1041    }
1042}
1043
1044void NuPlayer::finishFlushIfPossible() {
1045    if (mFlushingAudio != FLUSHED && mFlushingAudio != SHUT_DOWN) {
1046        return;
1047    }
1048
1049    if (mFlushingVideo != FLUSHED && mFlushingVideo != SHUT_DOWN) {
1050        return;
1051    }
1052
1053    ALOGV("both audio and video are flushed now.");
1054
1055    if (mTimeDiscontinuityPending) {
1056        mRenderer->signalTimeDiscontinuity();
1057        mTimeDiscontinuityPending = false;
1058    }
1059
1060    if (mAudioDecoder != NULL) {
1061        mAudioDecoder->signalResume();
1062    }
1063
1064    if (mVideoDecoder != NULL) {
1065        mVideoDecoder->signalResume();
1066    }
1067
1068    mFlushingAudio = NONE;
1069    mFlushingVideo = NONE;
1070
1071    processDeferredActions();
1072}
1073
1074void NuPlayer::postScanSources() {
1075    if (mScanSourcesPending) {
1076        return;
1077    }
1078
1079    sp<AMessage> msg = new AMessage(kWhatScanSources, id());
1080    msg->setInt32("generation", mScanSourcesGeneration);
1081    msg->post();
1082
1083    mScanSourcesPending = true;
1084}
1085
1086status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
1087    if (*decoder != NULL) {
1088        return OK;
1089    }
1090
1091    sp<AMessage> format = mSource->getFormat(audio);
1092
1093    if (format == NULL) {
1094        return -EWOULDBLOCK;
1095    }
1096
1097    if (!audio) {
1098        AString mime;
1099        CHECK(format->findString("mime", &mime));
1100        mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
1101
1102        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
1103        mCCDecoder = new CCDecoder(ccNotify);
1104
1105        if (mSourceFlags & Source::FLAG_SECURE) {
1106            format->setInt32("secure", true);
1107        }
1108    }
1109
1110    sp<AMessage> notify =
1111        new AMessage(audio ? kWhatAudioNotify : kWhatVideoNotify,
1112                     id());
1113
1114    if (audio) {
1115        if (mOffloadAudio) {
1116            *decoder = new DecoderPassThrough(notify);
1117        } else {
1118            *decoder = new Decoder(notify);
1119        }
1120    } else {
1121        *decoder = new Decoder(notify, mNativeWindow);
1122    }
1123    (*decoder)->init();
1124    (*decoder)->configure(format);
1125
1126    // allocate buffers to decrypt widevine source buffers
1127    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1128        Vector<sp<ABuffer> > inputBufs;
1129        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1130
1131        Vector<MediaBuffer *> mediaBufs;
1132        for (size_t i = 0; i < inputBufs.size(); i++) {
1133            const sp<ABuffer> &buffer = inputBufs[i];
1134            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1135            mediaBufs.push(mbuf);
1136        }
1137
1138        status_t err = mSource->setBuffers(audio, mediaBufs);
1139        if (err != OK) {
1140            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1141                mediaBufs[i]->release();
1142            }
1143            mediaBufs.clear();
1144            ALOGE("Secure source didn't support secure mediaBufs.");
1145            return err;
1146        }
1147    }
1148    return OK;
1149}
1150
1151status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
1152    sp<AMessage> reply;
1153    CHECK(msg->findMessage("reply", &reply));
1154
1155    if ((audio && IsFlushingState(mFlushingAudio))
1156            || (!audio && IsFlushingState(mFlushingVideo))) {
1157        reply->setInt32("err", INFO_DISCONTINUITY);
1158        reply->post();
1159        return OK;
1160    }
1161
1162    sp<ABuffer> accessUnit;
1163
1164    bool dropAccessUnit;
1165    do {
1166        status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
1167
1168        if (err == -EWOULDBLOCK) {
1169            return err;
1170        } else if (err != OK) {
1171            if (err == INFO_DISCONTINUITY) {
1172                int32_t type;
1173                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
1174
1175                bool formatChange =
1176                    (audio &&
1177                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
1178                    || (!audio &&
1179                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
1180
1181                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
1182
1183                ALOGI("%s discontinuity (formatChange=%d, time=%d)",
1184                     audio ? "audio" : "video", formatChange, timeChange);
1185
1186                if (audio) {
1187                    mSkipRenderingAudioUntilMediaTimeUs = -1;
1188                } else {
1189                    mSkipRenderingVideoUntilMediaTimeUs = -1;
1190                }
1191
1192                if (timeChange) {
1193                    sp<AMessage> extra;
1194                    if (accessUnit->meta()->findMessage("extra", &extra)
1195                            && extra != NULL) {
1196                        int64_t resumeAtMediaTimeUs;
1197                        if (extra->findInt64(
1198                                    "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
1199                            ALOGI("suppressing rendering of %s until %lld us",
1200                                    audio ? "audio" : "video", resumeAtMediaTimeUs);
1201
1202                            if (audio) {
1203                                mSkipRenderingAudioUntilMediaTimeUs =
1204                                    resumeAtMediaTimeUs;
1205                            } else {
1206                                mSkipRenderingVideoUntilMediaTimeUs =
1207                                    resumeAtMediaTimeUs;
1208                            }
1209                        }
1210                    }
1211                }
1212
1213                mTimeDiscontinuityPending =
1214                    mTimeDiscontinuityPending || timeChange;
1215
1216                if (mFlushingAudio == NONE && mFlushingVideo == NONE) {
1217                    // And we'll resume scanning sources once we're done
1218                    // flushing.
1219                    mDeferredActions.push_front(
1220                            new SimpleAction(
1221                                &NuPlayer::performScanSources));
1222                }
1223
1224                if (formatChange || timeChange) {
1225
1226                    sp<AMessage> newFormat = mSource->getFormat(audio);
1227                    sp<Decoder> &decoder = audio ? mAudioDecoder : mVideoDecoder;
1228                    if (formatChange && !decoder->supportsSeamlessFormatChange(newFormat)) {
1229                        flushDecoder(audio, /* needShutdown = */ true);
1230                    } else {
1231                        flushDecoder(audio, /* needShutdown = */ false);
1232                        err = OK;
1233                    }
1234                } else {
1235                    // This stream is unaffected by the discontinuity
1236
1237                    if (audio) {
1238                        mFlushingAudio = FLUSHED;
1239                    } else {
1240                        mFlushingVideo = FLUSHED;
1241                    }
1242
1243                    finishFlushIfPossible();
1244
1245                    return -EWOULDBLOCK;
1246                }
1247            }
1248
1249            reply->setInt32("err", err);
1250            reply->post();
1251            return OK;
1252        }
1253
1254        if (!audio) {
1255            ++mNumFramesTotal;
1256        }
1257
1258        dropAccessUnit = false;
1259        if (!audio
1260                && !(mSourceFlags & Source::FLAG_SECURE)
1261                && mVideoLateByUs > 100000ll
1262                && mVideoIsAVC
1263                && !IsAVCReferenceFrame(accessUnit)) {
1264            dropAccessUnit = true;
1265            ++mNumFramesDropped;
1266        }
1267    } while (dropAccessUnit);
1268
1269    // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
1270
1271#if 0
1272    int64_t mediaTimeUs;
1273    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
1274    ALOGV("feeding %s input buffer at media time %.2f secs",
1275         audio ? "audio" : "video",
1276         mediaTimeUs / 1E6);
1277#endif
1278
1279    if (!audio) {
1280        mCCDecoder->decode(accessUnit);
1281    }
1282
1283    reply->setBuffer("buffer", accessUnit);
1284    reply->post();
1285
1286    return OK;
1287}
1288
1289void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
1290    // ALOGV("renderBuffer %s", audio ? "audio" : "video");
1291
1292    sp<AMessage> reply;
1293    CHECK(msg->findMessage("reply", &reply));
1294
1295    if (IsFlushingState(audio ? mFlushingAudio : mFlushingVideo)) {
1296        // We're currently attempting to flush the decoder, in order
1297        // to complete this, the decoder wants all its buffers back,
1298        // so we don't want any output buffers it sent us (from before
1299        // we initiated the flush) to be stuck in the renderer's queue.
1300
1301        ALOGV("we're still flushing the %s decoder, sending its output buffer"
1302             " right back.", audio ? "audio" : "video");
1303
1304        reply->post();
1305        return;
1306    }
1307
1308    sp<ABuffer> buffer;
1309    CHECK(msg->findBuffer("buffer", &buffer));
1310
1311    int64_t mediaTimeUs;
1312    CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));
1313
1314    int64_t &skipUntilMediaTimeUs =
1315        audio
1316            ? mSkipRenderingAudioUntilMediaTimeUs
1317            : mSkipRenderingVideoUntilMediaTimeUs;
1318
1319    if (skipUntilMediaTimeUs >= 0) {
1320
1321        if (mediaTimeUs < skipUntilMediaTimeUs) {
1322            ALOGV("dropping %s buffer at time %lld as requested.",
1323                 audio ? "audio" : "video",
1324                 mediaTimeUs);
1325
1326            reply->post();
1327            return;
1328        }
1329
1330        skipUntilMediaTimeUs = -1;
1331    }
1332
1333    if (!audio && mCCDecoder->isSelected()) {
1334        mCCDecoder->display(mediaTimeUs);
1335    }
1336
1337    mRenderer->queueBuffer(audio, buffer, reply);
1338}
1339
1340void NuPlayer::updateVideoSize(
1341        const sp<AMessage> &inputFormat,
1342        const sp<AMessage> &outputFormat) {
1343    if (inputFormat == NULL) {
1344        ALOGW("Unknown video size, reporting 0x0!");
1345        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1346        return;
1347    }
1348
1349    int32_t displayWidth, displayHeight;
1350    int32_t cropLeft, cropTop, cropRight, cropBottom;
1351
1352    if (outputFormat != NULL) {
1353        int32_t width, height;
1354        CHECK(outputFormat->findInt32("width", &width));
1355        CHECK(outputFormat->findInt32("height", &height));
1356
1357        int32_t cropLeft, cropTop, cropRight, cropBottom;
1358        CHECK(outputFormat->findRect(
1359                    "crop",
1360                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1361
1362        displayWidth = cropRight - cropLeft + 1;
1363        displayHeight = cropBottom - cropTop + 1;
1364
1365        ALOGV("Video output format changed to %d x %d "
1366             "(crop: %d x %d @ (%d, %d))",
1367             width, height,
1368             displayWidth,
1369             displayHeight,
1370             cropLeft, cropTop);
1371    } else {
1372        CHECK(inputFormat->findInt32("width", &displayWidth));
1373        CHECK(inputFormat->findInt32("height", &displayHeight));
1374
1375        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1376    }
1377
1378    // Take into account sample aspect ratio if necessary:
1379    int32_t sarWidth, sarHeight;
1380    if (inputFormat->findInt32("sar-width", &sarWidth)
1381            && inputFormat->findInt32("sar-height", &sarHeight)) {
1382        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1383
1384        displayWidth = (displayWidth * sarWidth) / sarHeight;
1385
1386        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1387    }
1388
1389    int32_t rotationDegrees;
1390    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1391        rotationDegrees = 0;
1392    }
1393
1394    if (rotationDegrees == 90 || rotationDegrees == 270) {
1395        int32_t tmp = displayWidth;
1396        displayWidth = displayHeight;
1397        displayHeight = tmp;
1398    }
1399
1400    notifyListener(
1401            MEDIA_SET_VIDEO_SIZE,
1402            displayWidth,
1403            displayHeight);
1404}
1405
1406void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1407    if (mDriver == NULL) {
1408        return;
1409    }
1410
1411    sp<NuPlayerDriver> driver = mDriver.promote();
1412
1413    if (driver == NULL) {
1414        return;
1415    }
1416
1417    driver->notifyListener(msg, ext1, ext2, in);
1418}
1419
1420void NuPlayer::flushDecoder(bool audio, bool needShutdown) {
1421    ALOGV("[%s] flushDecoder needShutdown=%d",
1422          audio ? "audio" : "video", needShutdown);
1423
1424    if ((audio && mAudioDecoder == NULL) || (!audio && mVideoDecoder == NULL)) {
1425        ALOGI("flushDecoder %s without decoder present",
1426             audio ? "audio" : "video");
1427    }
1428
1429    // Make sure we don't continue to scan sources until we finish flushing.
1430    ++mScanSourcesGeneration;
1431    mScanSourcesPending = false;
1432
1433    (audio ? mAudioDecoder : mVideoDecoder)->signalFlush();
1434    mRenderer->flush(audio);
1435
1436    FlushStatus newStatus =
1437        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1438
1439    if (audio) {
1440        CHECK(mFlushingAudio == NONE
1441                || mFlushingAudio == AWAITING_DISCONTINUITY);
1442
1443        mFlushingAudio = newStatus;
1444
1445        if (mFlushingVideo == NONE) {
1446            mFlushingVideo = (mVideoDecoder != NULL)
1447                ? AWAITING_DISCONTINUITY
1448                : FLUSHED;
1449        }
1450    } else {
1451        CHECK(mFlushingVideo == NONE
1452                || mFlushingVideo == AWAITING_DISCONTINUITY);
1453
1454        mFlushingVideo = newStatus;
1455
1456        if (mFlushingAudio == NONE) {
1457            mFlushingAudio = (mAudioDecoder != NULL)
1458                ? AWAITING_DISCONTINUITY
1459                : FLUSHED;
1460        }
1461    }
1462}
1463
1464void NuPlayer::queueDecoderShutdown(
1465        bool audio, bool video, const sp<AMessage> &reply) {
1466    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1467
1468    mDeferredActions.push_back(
1469            new ShutdownDecoderAction(audio, video));
1470
1471    mDeferredActions.push_back(
1472            new SimpleAction(&NuPlayer::performScanSources));
1473
1474    mDeferredActions.push_back(new PostMessageAction(reply));
1475
1476    processDeferredActions();
1477}
1478
1479status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1480    mVideoScalingMode = mode;
1481    if (mNativeWindow != NULL) {
1482        status_t ret = native_window_set_scaling_mode(
1483                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1484        if (ret != OK) {
1485            ALOGE("Failed to set scaling mode (%d): %s",
1486                -ret, strerror(-ret));
1487            return ret;
1488        }
1489    }
1490    return OK;
1491}
1492
1493status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1494    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id());
1495    msg->setPointer("reply", reply);
1496
1497    sp<AMessage> response;
1498    status_t err = msg->postAndAwaitResponse(&response);
1499    return err;
1500}
1501
1502status_t NuPlayer::selectTrack(size_t trackIndex, bool select) {
1503    sp<AMessage> msg = new AMessage(kWhatSelectTrack, id());
1504    msg->setSize("trackIndex", trackIndex);
1505    msg->setInt32("select", select);
1506
1507    sp<AMessage> response;
1508    status_t err = msg->postAndAwaitResponse(&response);
1509
1510    if (err != OK) {
1511        return err;
1512    }
1513
1514    if (!response->findInt32("err", &err)) {
1515        err = OK;
1516    }
1517
1518    return err;
1519}
1520
1521void NuPlayer::schedulePollDuration() {
1522    sp<AMessage> msg = new AMessage(kWhatPollDuration, id());
1523    msg->setInt32("generation", mPollDurationGeneration);
1524    msg->post();
1525}
1526
1527void NuPlayer::cancelPollDuration() {
1528    ++mPollDurationGeneration;
1529}
1530
1531void NuPlayer::processDeferredActions() {
1532    while (!mDeferredActions.empty()) {
1533        // We won't execute any deferred actions until we're no longer in
1534        // an intermediate state, i.e. one more more decoders are currently
1535        // flushing or shutting down.
1536
1537        if (mRenderer != NULL) {
1538            // There's an edge case where the renderer owns all output
1539            // buffers and is paused, therefore the decoder will not read
1540            // more input data and will never encounter the matching
1541            // discontinuity. To avoid this, we resume the renderer.
1542
1543            if (mFlushingAudio == AWAITING_DISCONTINUITY
1544                    || mFlushingVideo == AWAITING_DISCONTINUITY) {
1545                mRenderer->resume();
1546            }
1547        }
1548
1549        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1550            // We're currently flushing, postpone the reset until that's
1551            // completed.
1552
1553            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1554                  mFlushingAudio, mFlushingVideo);
1555
1556            break;
1557        }
1558
1559        sp<Action> action = *mDeferredActions.begin();
1560        mDeferredActions.erase(mDeferredActions.begin());
1561
1562        action->execute(this);
1563    }
1564}
1565
1566void NuPlayer::performSeek(int64_t seekTimeUs) {
1567    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs)",
1568          seekTimeUs,
1569          seekTimeUs / 1E6);
1570
1571    mSource->seekTo(seekTimeUs);
1572    ++mTimedTextGeneration;
1573
1574    if (mDriver != NULL) {
1575        sp<NuPlayerDriver> driver = mDriver.promote();
1576        if (driver != NULL) {
1577            driver->notifyPosition(seekTimeUs);
1578            driver->notifySeekComplete();
1579        }
1580    }
1581
1582    // everything's flushed, continue playback.
1583}
1584
1585void NuPlayer::performDecoderFlush() {
1586    ALOGV("performDecoderFlush");
1587
1588    if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
1589        return;
1590    }
1591
1592    mTimeDiscontinuityPending = true;
1593
1594    if (mAudioDecoder != NULL) {
1595        flushDecoder(true /* audio */, false /* needShutdown */);
1596    }
1597
1598    if (mVideoDecoder != NULL) {
1599        flushDecoder(false /* audio */, false /* needShutdown */);
1600    }
1601}
1602
1603void NuPlayer::performDecoderShutdown(bool audio, bool video) {
1604    ALOGV("performDecoderShutdown audio=%d, video=%d", audio, video);
1605
1606    if ((!audio || mAudioDecoder == NULL)
1607            && (!video || mVideoDecoder == NULL)) {
1608        return;
1609    }
1610
1611    mTimeDiscontinuityPending = true;
1612
1613    if (mFlushingAudio == NONE && (!audio || mAudioDecoder == NULL)) {
1614        mFlushingAudio = FLUSHED;
1615    }
1616
1617    if (mFlushingVideo == NONE && (!video || mVideoDecoder == NULL)) {
1618        mFlushingVideo = FLUSHED;
1619    }
1620
1621    if (audio && mAudioDecoder != NULL) {
1622        flushDecoder(true /* audio */, true /* needShutdown */);
1623    }
1624
1625    if (video && mVideoDecoder != NULL) {
1626        flushDecoder(false /* audio */, true /* needShutdown */);
1627    }
1628}
1629
1630void NuPlayer::performReset() {
1631    ALOGV("performReset");
1632
1633    CHECK(mAudioDecoder == NULL);
1634    CHECK(mVideoDecoder == NULL);
1635
1636    cancelPollDuration();
1637
1638    ++mScanSourcesGeneration;
1639    mScanSourcesPending = false;
1640
1641    if (mRendererLooper != NULL) {
1642        if (mRenderer != NULL) {
1643            mRendererLooper->unregisterHandler(mRenderer->id());
1644        }
1645        mRendererLooper->stop();
1646        mRendererLooper.clear();
1647    }
1648    mRenderer.clear();
1649
1650    if (mSource != NULL) {
1651        mSource->stop();
1652
1653        looper()->unregisterHandler(mSource->id());
1654
1655        mSource.clear();
1656    }
1657
1658    if (mDriver != NULL) {
1659        sp<NuPlayerDriver> driver = mDriver.promote();
1660        if (driver != NULL) {
1661            driver->notifyResetComplete();
1662        }
1663    }
1664
1665    mStarted = false;
1666}
1667
1668void NuPlayer::performScanSources() {
1669    ALOGV("performScanSources");
1670
1671    if (!mStarted) {
1672        return;
1673    }
1674
1675    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1676        postScanSources();
1677    }
1678}
1679
1680void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1681    ALOGV("performSetSurface");
1682
1683    mNativeWindow = wrapper;
1684
1685    // XXX - ignore error from setVideoScalingMode for now
1686    setVideoScalingMode(mVideoScalingMode);
1687
1688    if (mDriver != NULL) {
1689        sp<NuPlayerDriver> driver = mDriver.promote();
1690        if (driver != NULL) {
1691            driver->notifySetSurfaceComplete();
1692        }
1693    }
1694}
1695
1696void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1697    int32_t what;
1698    CHECK(msg->findInt32("what", &what));
1699
1700    switch (what) {
1701        case Source::kWhatPrepared:
1702        {
1703            if (mSource == NULL) {
1704                // This is a stale notification from a source that was
1705                // asynchronously preparing when the client called reset().
1706                // We handled the reset, the source is gone.
1707                break;
1708            }
1709
1710            int32_t err;
1711            CHECK(msg->findInt32("err", &err));
1712
1713            sp<NuPlayerDriver> driver = mDriver.promote();
1714            if (driver != NULL) {
1715                // notify duration first, so that it's definitely set when
1716                // the app received the "prepare complete" callback.
1717                int64_t durationUs;
1718                if (mSource->getDuration(&durationUs) == OK) {
1719                    driver->notifyDuration(durationUs);
1720                }
1721                driver->notifyPrepareCompleted(err);
1722            }
1723
1724            break;
1725        }
1726
1727        case Source::kWhatFlagsChanged:
1728        {
1729            uint32_t flags;
1730            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1731
1732            sp<NuPlayerDriver> driver = mDriver.promote();
1733            if (driver != NULL) {
1734                driver->notifyFlagsChanged(flags);
1735            }
1736
1737            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1738                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
1739                cancelPollDuration();
1740            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1741                    && (flags & Source::FLAG_DYNAMIC_DURATION)
1742                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
1743                schedulePollDuration();
1744            }
1745
1746            mSourceFlags = flags;
1747            break;
1748        }
1749
1750        case Source::kWhatVideoSizeChanged:
1751        {
1752            sp<AMessage> format;
1753            CHECK(msg->findMessage("format", &format));
1754
1755            updateVideoSize(format);
1756            break;
1757        }
1758
1759        case Source::kWhatBufferingStart:
1760        {
1761            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
1762            break;
1763        }
1764
1765        case Source::kWhatBufferingEnd:
1766        {
1767            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
1768            break;
1769        }
1770
1771        case Source::kWhatSubtitleData:
1772        {
1773            sp<ABuffer> buffer;
1774            CHECK(msg->findBuffer("buffer", &buffer));
1775
1776            sendSubtitleData(buffer, 0 /* baseIndex */);
1777            break;
1778        }
1779
1780        case Source::kWhatTimedTextData:
1781        {
1782            int32_t generation;
1783            if (msg->findInt32("generation", &generation)
1784                    && generation != mTimedTextGeneration) {
1785                break;
1786            }
1787
1788            sp<ABuffer> buffer;
1789            CHECK(msg->findBuffer("buffer", &buffer));
1790
1791            sp<NuPlayerDriver> driver = mDriver.promote();
1792            if (driver == NULL) {
1793                break;
1794            }
1795
1796            int posMs;
1797            int64_t timeUs, posUs;
1798            driver->getCurrentPosition(&posMs);
1799            posUs = posMs * 1000;
1800            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1801
1802            if (posUs < timeUs) {
1803                if (!msg->findInt32("generation", &generation)) {
1804                    msg->setInt32("generation", mTimedTextGeneration);
1805                }
1806                msg->post(timeUs - posUs);
1807            } else {
1808                sendTimedTextData(buffer);
1809            }
1810            break;
1811        }
1812
1813        case Source::kWhatQueueDecoderShutdown:
1814        {
1815            int32_t audio, video;
1816            CHECK(msg->findInt32("audio", &audio));
1817            CHECK(msg->findInt32("video", &video));
1818
1819            sp<AMessage> reply;
1820            CHECK(msg->findMessage("reply", &reply));
1821
1822            queueDecoderShutdown(audio, video, reply);
1823            break;
1824        }
1825
1826        default:
1827            TRESPASS();
1828    }
1829}
1830
1831void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
1832    int32_t what;
1833    CHECK(msg->findInt32("what", &what));
1834
1835    switch (what) {
1836        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
1837        {
1838            sp<ABuffer> buffer;
1839            CHECK(msg->findBuffer("buffer", &buffer));
1840
1841            size_t inbandTracks = 0;
1842            if (mSource != NULL) {
1843                inbandTracks = mSource->getTrackCount();
1844            }
1845
1846            sendSubtitleData(buffer, inbandTracks);
1847            break;
1848        }
1849
1850        case NuPlayer::CCDecoder::kWhatTrackAdded:
1851        {
1852            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
1853
1854            break;
1855        }
1856
1857        default:
1858            TRESPASS();
1859    }
1860
1861
1862}
1863
1864void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
1865    int32_t trackIndex;
1866    int64_t timeUs, durationUs;
1867    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
1868    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1869    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
1870
1871    Parcel in;
1872    in.writeInt32(trackIndex + baseIndex);
1873    in.writeInt64(timeUs);
1874    in.writeInt64(durationUs);
1875    in.writeInt32(buffer->size());
1876    in.writeInt32(buffer->size());
1877    in.write(buffer->data(), buffer->size());
1878
1879    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
1880}
1881
1882void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
1883    const void *data;
1884    size_t size = 0;
1885    int64_t timeUs;
1886    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
1887
1888    AString mime;
1889    CHECK(buffer->meta()->findString("mime", &mime));
1890    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
1891
1892    data = buffer->data();
1893    size = buffer->size();
1894
1895    Parcel parcel;
1896    if (size > 0) {
1897        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1898        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
1899        TextDescriptions::getParcelOfDescriptions(
1900                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
1901    }
1902
1903    if ((parcel.dataSize() > 0)) {
1904        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
1905    } else {  // send an empty timed text
1906        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
1907    }
1908}
1909////////////////////////////////////////////////////////////////////////////////
1910
1911sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
1912    sp<MetaData> meta = getFormatMeta(audio);
1913
1914    if (meta == NULL) {
1915        return NULL;
1916    }
1917
1918    sp<AMessage> msg = new AMessage;
1919
1920    if(convertMetaDataToMessage(meta, &msg) == OK) {
1921        return msg;
1922    }
1923    return NULL;
1924}
1925
1926void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
1927    sp<AMessage> notify = dupNotify();
1928    notify->setInt32("what", kWhatFlagsChanged);
1929    notify->setInt32("flags", flags);
1930    notify->post();
1931}
1932
1933void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
1934    sp<AMessage> notify = dupNotify();
1935    notify->setInt32("what", kWhatVideoSizeChanged);
1936    notify->setMessage("format", format);
1937    notify->post();
1938}
1939
1940void NuPlayer::Source::notifyPrepared(status_t err) {
1941    sp<AMessage> notify = dupNotify();
1942    notify->setInt32("what", kWhatPrepared);
1943    notify->setInt32("err", err);
1944    notify->post();
1945}
1946
1947void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
1948    TRESPASS();
1949}
1950
1951}  // namespace android
1952