NuPlayer.cpp revision 8b63533e6aaff121378878998925c57dbe9a9e16 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerCCDecoder.h"
25#include "NuPlayerDecoder.h"
26#include "NuPlayerDecoderBase.h"
27#include "NuPlayerDecoderPassThrough.h"
28#include "NuPlayerDriver.h"
29#include "NuPlayerRenderer.h"
30#include "NuPlayerSource.h"
31#include "RTSPSource.h"
32#include "StreamingSource.h"
33#include "GenericSource.h"
34#include "TextDescriptions.h"
35
36#include "ATSParser.h"
37
38#include <cutils/properties.h>
39
40#include <media/stagefright/foundation/hexdump.h>
41#include <media/stagefright/foundation/ABuffer.h>
42#include <media/stagefright/foundation/ADebug.h>
43#include <media/stagefright/foundation/AMessage.h>
44#include <media/stagefright/MediaBuffer.h>
45#include <media/stagefright/MediaDefs.h>
46#include <media/stagefright/MediaErrors.h>
47#include <media/stagefright/MetaData.h>
48#include <gui/IGraphicBufferProducer.h>
49
50#include "avc_utils.h"
51
52#include "ESDS.h"
53#include <media/stagefright/Utils.h>
54
55namespace android {
56
57struct NuPlayer::Action : public RefBase {
58    Action() {}
59
60    virtual void execute(NuPlayer *player) = 0;
61
62private:
63    DISALLOW_EVIL_CONSTRUCTORS(Action);
64};
65
66struct NuPlayer::SeekAction : public Action {
67    SeekAction(int64_t seekTimeUs, bool needNotify)
68        : mSeekTimeUs(seekTimeUs),
69          mNeedNotify(needNotify) {
70    }
71
72    virtual void execute(NuPlayer *player) {
73        player->performSeek(mSeekTimeUs, mNeedNotify);
74    }
75
76private:
77    int64_t mSeekTimeUs;
78    bool mNeedNotify;
79
80    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
81};
82
83struct NuPlayer::ResumeDecoderAction : public Action {
84    ResumeDecoderAction(bool needNotify)
85        : mNeedNotify(needNotify) {
86    }
87
88    virtual void execute(NuPlayer *player) {
89        player->performResumeDecoders(mNeedNotify);
90    }
91
92private:
93    bool mNeedNotify;
94
95    DISALLOW_EVIL_CONSTRUCTORS(ResumeDecoderAction);
96};
97
98struct NuPlayer::SetSurfaceAction : public Action {
99    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
100        : mWrapper(wrapper) {
101    }
102
103    virtual void execute(NuPlayer *player) {
104        player->performSetSurface(mWrapper);
105    }
106
107private:
108    sp<NativeWindowWrapper> mWrapper;
109
110    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
111};
112
113struct NuPlayer::FlushDecoderAction : public Action {
114    FlushDecoderAction(FlushCommand audio, FlushCommand video)
115        : mAudio(audio),
116          mVideo(video) {
117    }
118
119    virtual void execute(NuPlayer *player) {
120        player->performDecoderFlush(mAudio, mVideo);
121    }
122
123private:
124    FlushCommand mAudio;
125    FlushCommand mVideo;
126
127    DISALLOW_EVIL_CONSTRUCTORS(FlushDecoderAction);
128};
129
130struct NuPlayer::PostMessageAction : public Action {
131    PostMessageAction(const sp<AMessage> &msg)
132        : mMessage(msg) {
133    }
134
135    virtual void execute(NuPlayer *) {
136        mMessage->post();
137    }
138
139private:
140    sp<AMessage> mMessage;
141
142    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
143};
144
145// Use this if there's no state necessary to save in order to execute
146// the action.
147struct NuPlayer::SimpleAction : public Action {
148    typedef void (NuPlayer::*ActionFunc)();
149
150    SimpleAction(ActionFunc func)
151        : mFunc(func) {
152    }
153
154    virtual void execute(NuPlayer *player) {
155        (player->*mFunc)();
156    }
157
158private:
159    ActionFunc mFunc;
160
161    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
162};
163
164////////////////////////////////////////////////////////////////////////////////
165
166NuPlayer::NuPlayer()
167    : mUIDValid(false),
168      mSourceFlags(0),
169      mOffloadAudio(false),
170      mAudioDecoderGeneration(0),
171      mVideoDecoderGeneration(0),
172      mRendererGeneration(0),
173      mAudioEOS(false),
174      mVideoEOS(false),
175      mScanSourcesPending(false),
176      mScanSourcesGeneration(0),
177      mPollDurationGeneration(0),
178      mTimedTextGeneration(0),
179      mFlushingAudio(NONE),
180      mFlushingVideo(NONE),
181      mResumePending(false),
182      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
183      mPlaybackRate(1.0),
184      mStarted(false),
185      mPaused(false),
186      mPausedByClient(false) {
187    clearFlushComplete();
188}
189
190NuPlayer::~NuPlayer() {
191}
192
193void NuPlayer::setUID(uid_t uid) {
194    mUIDValid = true;
195    mUID = uid;
196}
197
198void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
199    mDriver = driver;
200}
201
202void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
203    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
204
205    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
206
207    msg->setObject("source", new StreamingSource(notify, source));
208    msg->post();
209}
210
211static bool IsHTTPLiveURL(const char *url) {
212    if (!strncasecmp("http://", url, 7)
213            || !strncasecmp("https://", url, 8)
214            || !strncasecmp("file://", url, 7)) {
215        size_t len = strlen(url);
216        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
217            return true;
218        }
219
220        if (strstr(url,"m3u8")) {
221            return true;
222        }
223    }
224
225    return false;
226}
227
228void NuPlayer::setDataSourceAsync(
229        const sp<IMediaHTTPService> &httpService,
230        const char *url,
231        const KeyedVector<String8, String8> *headers) {
232
233    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
234    size_t len = strlen(url);
235
236    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
237
238    sp<Source> source;
239    if (IsHTTPLiveURL(url)) {
240        source = new HTTPLiveSource(notify, httpService, url, headers);
241    } else if (!strncasecmp(url, "rtsp://", 7)) {
242        source = new RTSPSource(
243                notify, httpService, url, headers, mUIDValid, mUID);
244    } else if ((!strncasecmp(url, "http://", 7)
245                || !strncasecmp(url, "https://", 8))
246                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
247                    || strstr(url, ".sdp?"))) {
248        source = new RTSPSource(
249                notify, httpService, url, headers, mUIDValid, mUID, true);
250    } else {
251        sp<GenericSource> genericSource =
252                new GenericSource(notify, mUIDValid, mUID);
253        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
254        // The correct flags will be updated in Source::kWhatFlagsChanged
255        // handler when  GenericSource is prepared.
256
257        status_t err = genericSource->setDataSource(httpService, url, headers);
258
259        if (err == OK) {
260            source = genericSource;
261        } else {
262            ALOGE("Failed to set data source!");
263        }
264    }
265    msg->setObject("source", source);
266    msg->post();
267}
268
269void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
270    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
271
272    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
273
274    sp<GenericSource> source =
275            new GenericSource(notify, mUIDValid, mUID);
276
277    status_t err = source->setDataSource(fd, offset, length);
278
279    if (err != OK) {
280        ALOGE("Failed to set data source!");
281        source = NULL;
282    }
283
284    msg->setObject("source", source);
285    msg->post();
286}
287
288void NuPlayer::prepareAsync() {
289    (new AMessage(kWhatPrepare, this))->post();
290}
291
292void NuPlayer::setVideoSurfaceTextureAsync(
293        const sp<IGraphicBufferProducer> &bufferProducer) {
294    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, this);
295
296    if (bufferProducer == NULL) {
297        msg->setObject("native-window", NULL);
298    } else {
299        msg->setObject(
300                "native-window",
301                new NativeWindowWrapper(
302                    new Surface(bufferProducer, true /* controlledByApp */)));
303    }
304
305    msg->post();
306}
307
308void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
309    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, this);
310    msg->setObject("sink", sink);
311    msg->post();
312}
313
314void NuPlayer::start() {
315    (new AMessage(kWhatStart, this))->post();
316}
317
318void NuPlayer::setPlaybackRate(float rate) {
319    sp<AMessage> msg = new AMessage(kWhatSetRate, this);
320    msg->setFloat("rate", rate);
321    msg->post();
322}
323
324void NuPlayer::pause() {
325    (new AMessage(kWhatPause, this))->post();
326}
327
328void NuPlayer::resetAsync() {
329    if (mSource != NULL) {
330        // During a reset, the data source might be unresponsive already, we need to
331        // disconnect explicitly so that reads exit promptly.
332        // We can't queue the disconnect request to the looper, as it might be
333        // queued behind a stuck read and never gets processed.
334        // Doing a disconnect outside the looper to allows the pending reads to exit
335        // (either successfully or with error).
336        mSource->disconnect();
337    }
338
339    (new AMessage(kWhatReset, this))->post();
340}
341
342void NuPlayer::seekToAsync(int64_t seekTimeUs, bool needNotify) {
343    sp<AMessage> msg = new AMessage(kWhatSeek, this);
344    msg->setInt64("seekTimeUs", seekTimeUs);
345    msg->setInt32("needNotify", needNotify);
346    msg->post();
347}
348
349
350void NuPlayer::writeTrackInfo(
351        Parcel* reply, const sp<AMessage> format) const {
352    int32_t trackType;
353    CHECK(format->findInt32("type", &trackType));
354
355    AString lang;
356    CHECK(format->findString("language", &lang));
357
358    reply->writeInt32(2); // write something non-zero
359    reply->writeInt32(trackType);
360    reply->writeString16(String16(lang.c_str()));
361
362    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
363        AString mime;
364        CHECK(format->findString("mime", &mime));
365
366        int32_t isAuto, isDefault, isForced;
367        CHECK(format->findInt32("auto", &isAuto));
368        CHECK(format->findInt32("default", &isDefault));
369        CHECK(format->findInt32("forced", &isForced));
370
371        reply->writeString16(String16(mime.c_str()));
372        reply->writeInt32(isAuto);
373        reply->writeInt32(isDefault);
374        reply->writeInt32(isForced);
375    }
376}
377
378void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
379    switch (msg->what()) {
380        case kWhatSetDataSource:
381        {
382            ALOGV("kWhatSetDataSource");
383
384            CHECK(mSource == NULL);
385
386            status_t err = OK;
387            sp<RefBase> obj;
388            CHECK(msg->findObject("source", &obj));
389            if (obj != NULL) {
390                mSource = static_cast<Source *>(obj.get());
391            } else {
392                err = UNKNOWN_ERROR;
393            }
394
395            CHECK(mDriver != NULL);
396            sp<NuPlayerDriver> driver = mDriver.promote();
397            if (driver != NULL) {
398                driver->notifySetDataSourceCompleted(err);
399            }
400            break;
401        }
402
403        case kWhatPrepare:
404        {
405            mSource->prepareAsync();
406            break;
407        }
408
409        case kWhatGetTrackInfo:
410        {
411            sp<AReplyToken> replyID;
412            CHECK(msg->senderAwaitsResponse(&replyID));
413
414            Parcel* reply;
415            CHECK(msg->findPointer("reply", (void**)&reply));
416
417            size_t inbandTracks = 0;
418            if (mSource != NULL) {
419                inbandTracks = mSource->getTrackCount();
420            }
421
422            size_t ccTracks = 0;
423            if (mCCDecoder != NULL) {
424                ccTracks = mCCDecoder->getTrackCount();
425            }
426
427            // total track count
428            reply->writeInt32(inbandTracks + ccTracks);
429
430            // write inband tracks
431            for (size_t i = 0; i < inbandTracks; ++i) {
432                writeTrackInfo(reply, mSource->getTrackInfo(i));
433            }
434
435            // write CC track
436            for (size_t i = 0; i < ccTracks; ++i) {
437                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
438            }
439
440            sp<AMessage> response = new AMessage;
441            response->postReply(replyID);
442            break;
443        }
444
445        case kWhatGetSelectedTrack:
446        {
447            status_t err = INVALID_OPERATION;
448            if (mSource != NULL) {
449                err = OK;
450
451                int32_t type32;
452                CHECK(msg->findInt32("type", (int32_t*)&type32));
453                media_track_type type = (media_track_type)type32;
454                ssize_t selectedTrack = mSource->getSelectedTrack(type);
455
456                Parcel* reply;
457                CHECK(msg->findPointer("reply", (void**)&reply));
458                reply->writeInt32(selectedTrack);
459            }
460
461            sp<AMessage> response = new AMessage;
462            response->setInt32("err", err);
463
464            sp<AReplyToken> replyID;
465            CHECK(msg->senderAwaitsResponse(&replyID));
466            response->postReply(replyID);
467            break;
468        }
469
470        case kWhatSelectTrack:
471        {
472            sp<AReplyToken> replyID;
473            CHECK(msg->senderAwaitsResponse(&replyID));
474
475            size_t trackIndex;
476            int32_t select;
477            int64_t timeUs;
478            CHECK(msg->findSize("trackIndex", &trackIndex));
479            CHECK(msg->findInt32("select", &select));
480            CHECK(msg->findInt64("timeUs", &timeUs));
481
482            status_t err = INVALID_OPERATION;
483
484            size_t inbandTracks = 0;
485            if (mSource != NULL) {
486                inbandTracks = mSource->getTrackCount();
487            }
488            size_t ccTracks = 0;
489            if (mCCDecoder != NULL) {
490                ccTracks = mCCDecoder->getTrackCount();
491            }
492
493            if (trackIndex < inbandTracks) {
494                err = mSource->selectTrack(trackIndex, select, timeUs);
495
496                if (!select && err == OK) {
497                    int32_t type;
498                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
499                    if (info != NULL
500                            && info->findInt32("type", &type)
501                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
502                        ++mTimedTextGeneration;
503                    }
504                }
505            } else {
506                trackIndex -= inbandTracks;
507
508                if (trackIndex < ccTracks) {
509                    err = mCCDecoder->selectTrack(trackIndex, select);
510                }
511            }
512
513            sp<AMessage> response = new AMessage;
514            response->setInt32("err", err);
515
516            response->postReply(replyID);
517            break;
518        }
519
520        case kWhatPollDuration:
521        {
522            int32_t generation;
523            CHECK(msg->findInt32("generation", &generation));
524
525            if (generation != mPollDurationGeneration) {
526                // stale
527                break;
528            }
529
530            int64_t durationUs;
531            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
532                sp<NuPlayerDriver> driver = mDriver.promote();
533                if (driver != NULL) {
534                    driver->notifyDuration(durationUs);
535                }
536            }
537
538            msg->post(1000000ll);  // poll again in a second.
539            break;
540        }
541
542        case kWhatSetVideoNativeWindow:
543        {
544            ALOGV("kWhatSetVideoNativeWindow");
545
546            sp<RefBase> obj;
547            CHECK(msg->findObject("native-window", &obj));
548
549            if (mSource == NULL || mSource->getFormat(false /* audio */) == NULL) {
550                performSetSurface(static_cast<NativeWindowWrapper *>(obj.get()));
551                break;
552            }
553
554            mDeferredActions.push_back(
555                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
556                                           FLUSH_CMD_SHUTDOWN /* video */));
557
558            mDeferredActions.push_back(
559                    new SetSurfaceAction(
560                        static_cast<NativeWindowWrapper *>(obj.get())));
561
562            if (obj != NULL) {
563                if (mStarted) {
564                    // Issue a seek to refresh the video screen only if started otherwise
565                    // the extractor may not yet be started and will assert.
566                    // If the video decoder is not set (perhaps audio only in this case)
567                    // do not perform a seek as it is not needed.
568                    int64_t currentPositionUs = 0;
569                    if (getCurrentPosition(&currentPositionUs) == OK) {
570                        mDeferredActions.push_back(
571                                new SeekAction(currentPositionUs, false /* needNotify */));
572                    }
573                }
574
575                // If there is a new surface texture, instantiate decoders
576                // again if possible.
577                mDeferredActions.push_back(
578                        new SimpleAction(&NuPlayer::performScanSources));
579            }
580
581            // After a flush without shutdown, decoder is paused.
582            // Don't resume it until source seek is done, otherwise it could
583            // start pulling stale data too soon.
584            mDeferredActions.push_back(
585                    new ResumeDecoderAction(false /* needNotify */));
586
587            processDeferredActions();
588            break;
589        }
590
591        case kWhatSetAudioSink:
592        {
593            ALOGV("kWhatSetAudioSink");
594
595            sp<RefBase> obj;
596            CHECK(msg->findObject("sink", &obj));
597
598            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
599            break;
600        }
601
602        case kWhatStart:
603        {
604            ALOGV("kWhatStart");
605            if (mStarted) {
606                onResume();
607            } else {
608                onStart();
609            }
610            mPausedByClient = false;
611            break;
612        }
613
614        case kWhatSetRate:
615        {
616            ALOGV("kWhatSetRate");
617            CHECK(msg->findFloat("rate", &mPlaybackRate));
618            if (mRenderer != NULL) {
619                mRenderer->setPlaybackRate(mPlaybackRate);
620            }
621            break;
622        }
623
624        case kWhatScanSources:
625        {
626            int32_t generation;
627            CHECK(msg->findInt32("generation", &generation));
628            if (generation != mScanSourcesGeneration) {
629                // Drop obsolete msg.
630                break;
631            }
632
633            mScanSourcesPending = false;
634
635            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
636                 mAudioDecoder != NULL, mVideoDecoder != NULL);
637
638            bool mHadAnySourcesBefore =
639                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
640
641            // initialize video before audio because successful initialization of
642            // video may change deep buffer mode of audio.
643            if (mNativeWindow != NULL) {
644                instantiateDecoder(false, &mVideoDecoder);
645            }
646
647            // Don't try to re-open audio sink if there's an existing decoder.
648            if (mAudioSink != NULL && mAudioDecoder == NULL) {
649                sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
650                sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
651                audio_stream_type_t streamType = mAudioSink->getAudioStreamType();
652                const bool hasVideo = (videoFormat != NULL);
653                const bool canOffload = canOffloadStream(
654                        audioMeta, hasVideo, true /* is_streaming */, streamType);
655                if (canOffload) {
656                    if (!mOffloadAudio) {
657                        mRenderer->signalEnableOffloadAudio();
658                    }
659                    // open audio sink early under offload mode.
660                    sp<AMessage> format = mSource->getFormat(true /*audio*/);
661                    tryOpenAudioSinkForOffload(format, hasVideo);
662                }
663                instantiateDecoder(true, &mAudioDecoder);
664            }
665
666            if (!mHadAnySourcesBefore
667                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
668                // This is the first time we've found anything playable.
669
670                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
671                    schedulePollDuration();
672                }
673            }
674
675            status_t err;
676            if ((err = mSource->feedMoreTSData()) != OK) {
677                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
678                    // We're not currently decoding anything (no audio or
679                    // video tracks found) and we just ran out of input data.
680
681                    if (err == ERROR_END_OF_STREAM) {
682                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
683                    } else {
684                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
685                    }
686                }
687                break;
688            }
689
690            if ((mAudioDecoder == NULL && mAudioSink != NULL)
691                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
692                msg->post(100000ll);
693                mScanSourcesPending = true;
694            }
695            break;
696        }
697
698        case kWhatVideoNotify:
699        case kWhatAudioNotify:
700        {
701            bool audio = msg->what() == kWhatAudioNotify;
702
703            int32_t currentDecoderGeneration =
704                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
705            int32_t requesterGeneration = currentDecoderGeneration - 1;
706            CHECK(msg->findInt32("generation", &requesterGeneration));
707
708            if (requesterGeneration != currentDecoderGeneration) {
709                ALOGV("got message from old %s decoder, generation(%d:%d)",
710                        audio ? "audio" : "video", requesterGeneration,
711                        currentDecoderGeneration);
712                sp<AMessage> reply;
713                if (!(msg->findMessage("reply", &reply))) {
714                    return;
715                }
716
717                reply->setInt32("err", INFO_DISCONTINUITY);
718                reply->post();
719                return;
720            }
721
722            int32_t what;
723            CHECK(msg->findInt32("what", &what));
724
725            if (what == DecoderBase::kWhatInputDiscontinuity) {
726                int32_t formatChange;
727                CHECK(msg->findInt32("formatChange", &formatChange));
728
729                ALOGV("%s discontinuity: formatChange %d",
730                        audio ? "audio" : "video", formatChange);
731
732                if (formatChange) {
733                    mDeferredActions.push_back(
734                            new FlushDecoderAction(
735                                audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
736                                audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN));
737                }
738
739                mDeferredActions.push_back(
740                        new SimpleAction(
741                                &NuPlayer::performScanSources));
742
743                processDeferredActions();
744            } else if (what == DecoderBase::kWhatEOS) {
745                int32_t err;
746                CHECK(msg->findInt32("err", &err));
747
748                if (err == ERROR_END_OF_STREAM) {
749                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
750                } else {
751                    ALOGV("got %s decoder EOS w/ error %d",
752                         audio ? "audio" : "video",
753                         err);
754                }
755
756                mRenderer->queueEOS(audio, err);
757            } else if (what == DecoderBase::kWhatFlushCompleted) {
758                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
759
760                handleFlushComplete(audio, true /* isDecoder */);
761                finishFlushIfPossible();
762            } else if (what == DecoderBase::kWhatVideoSizeChanged) {
763                sp<AMessage> format;
764                CHECK(msg->findMessage("format", &format));
765
766                sp<AMessage> inputFormat =
767                        mSource->getFormat(false /* audio */);
768
769                updateVideoSize(inputFormat, format);
770            } else if (what == DecoderBase::kWhatShutdownCompleted) {
771                ALOGV("%s shutdown completed", audio ? "audio" : "video");
772                if (audio) {
773                    mAudioDecoder.clear();
774                    ++mAudioDecoderGeneration;
775
776                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
777                    mFlushingAudio = SHUT_DOWN;
778                } else {
779                    mVideoDecoder.clear();
780                    ++mVideoDecoderGeneration;
781
782                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
783                    mFlushingVideo = SHUT_DOWN;
784                }
785
786                finishFlushIfPossible();
787            } else if (what == DecoderBase::kWhatResumeCompleted) {
788                finishResume();
789            } else if (what == DecoderBase::kWhatError) {
790                status_t err;
791                if (!msg->findInt32("err", &err) || err == OK) {
792                    err = UNKNOWN_ERROR;
793                }
794
795                // Decoder errors can be due to Source (e.g. from streaming),
796                // or from decoding corrupted bitstreams, or from other decoder
797                // MediaCodec operations (e.g. from an ongoing reset or seek).
798                // They may also be due to openAudioSink failure at
799                // decoder start or after a format change.
800                //
801                // We try to gracefully shut down the affected decoder if possible,
802                // rather than trying to force the shutdown with something
803                // similar to performReset(). This method can lead to a hang
804                // if MediaCodec functions block after an error, but they should
805                // typically return INVALID_OPERATION instead of blocking.
806
807                FlushStatus *flushing = audio ? &mFlushingAudio : &mFlushingVideo;
808                ALOGE("received error(%#x) from %s decoder, flushing(%d), now shutting down",
809                        err, audio ? "audio" : "video", *flushing);
810
811                switch (*flushing) {
812                    case NONE:
813                        mDeferredActions.push_back(
814                                new FlushDecoderAction(
815                                    audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
816                                    audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN));
817                        processDeferredActions();
818                        break;
819                    case FLUSHING_DECODER:
820                        *flushing = FLUSHING_DECODER_SHUTDOWN; // initiate shutdown after flush.
821                        break; // Wait for flush to complete.
822                    case FLUSHING_DECODER_SHUTDOWN:
823                        break; // Wait for flush to complete.
824                    case SHUTTING_DOWN_DECODER:
825                        break; // Wait for shutdown to complete.
826                    case FLUSHED:
827                        // Widevine source reads must stop before releasing the video decoder.
828                        if (!audio && mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
829                            mSource->stop();
830                        }
831                        getDecoder(audio)->initiateShutdown(); // In the middle of a seek.
832                        *flushing = SHUTTING_DOWN_DECODER;     // Shut down.
833                        break;
834                    case SHUT_DOWN:
835                        finishFlushIfPossible();  // Should not occur.
836                        break;                    // Finish anyways.
837                }
838                notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
839            } else {
840                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
841                      what,
842                      what >> 24,
843                      (what >> 16) & 0xff,
844                      (what >> 8) & 0xff,
845                      what & 0xff);
846            }
847
848            break;
849        }
850
851        case kWhatRendererNotify:
852        {
853            int32_t requesterGeneration = mRendererGeneration - 1;
854            CHECK(msg->findInt32("generation", &requesterGeneration));
855            if (requesterGeneration != mRendererGeneration) {
856                ALOGV("got message from old renderer, generation(%d:%d)",
857                        requesterGeneration, mRendererGeneration);
858                return;
859            }
860
861            int32_t what;
862            CHECK(msg->findInt32("what", &what));
863
864            if (what == Renderer::kWhatEOS) {
865                int32_t audio;
866                CHECK(msg->findInt32("audio", &audio));
867
868                int32_t finalResult;
869                CHECK(msg->findInt32("finalResult", &finalResult));
870
871                if (audio) {
872                    mAudioEOS = true;
873                } else {
874                    mVideoEOS = true;
875                }
876
877                if (finalResult == ERROR_END_OF_STREAM) {
878                    ALOGV("reached %s EOS", audio ? "audio" : "video");
879                } else {
880                    ALOGE("%s track encountered an error (%d)",
881                         audio ? "audio" : "video", finalResult);
882
883                    notifyListener(
884                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
885                }
886
887                if ((mAudioEOS || mAudioDecoder == NULL)
888                        && (mVideoEOS || mVideoDecoder == NULL)) {
889                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
890                }
891            } else if (what == Renderer::kWhatFlushComplete) {
892                int32_t audio;
893                CHECK(msg->findInt32("audio", &audio));
894
895                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
896                handleFlushComplete(audio, false /* isDecoder */);
897                finishFlushIfPossible();
898            } else if (what == Renderer::kWhatVideoRenderingStart) {
899                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
900            } else if (what == Renderer::kWhatMediaRenderingStart) {
901                ALOGV("media rendering started");
902                notifyListener(MEDIA_STARTED, 0, 0);
903            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
904                ALOGV("Tear down audio offload, fall back to s/w path if due to error.");
905                int64_t positionUs;
906                CHECK(msg->findInt64("positionUs", &positionUs));
907                int32_t reason;
908                CHECK(msg->findInt32("reason", &reason));
909                closeAudioSink();
910                mAudioDecoder.clear();
911                ++mAudioDecoderGeneration;
912                mRenderer->flush(
913                        true /* audio */, false /* notifyComplete */);
914                if (mVideoDecoder != NULL) {
915                    mRenderer->flush(
916                            false /* audio */, false /* notifyComplete */);
917                }
918
919                performSeek(positionUs, false /* needNotify */);
920                if (reason == Renderer::kDueToError) {
921                    mRenderer->signalDisableOffloadAudio();
922                    mOffloadAudio = false;
923                    instantiateDecoder(true /* audio */, &mAudioDecoder);
924                }
925            }
926            break;
927        }
928
929        case kWhatMoreDataQueued:
930        {
931            break;
932        }
933
934        case kWhatReset:
935        {
936            ALOGV("kWhatReset");
937
938            mDeferredActions.push_back(
939                    new FlushDecoderAction(
940                        FLUSH_CMD_SHUTDOWN /* audio */,
941                        FLUSH_CMD_SHUTDOWN /* video */));
942
943            mDeferredActions.push_back(
944                    new SimpleAction(&NuPlayer::performReset));
945
946            processDeferredActions();
947            break;
948        }
949
950        case kWhatSeek:
951        {
952            int64_t seekTimeUs;
953            int32_t needNotify;
954            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
955            CHECK(msg->findInt32("needNotify", &needNotify));
956
957            ALOGV("kWhatSeek seekTimeUs=%lld us, needNotify=%d",
958                    seekTimeUs, needNotify);
959
960            mDeferredActions.push_back(
961                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
962                                           FLUSH_CMD_FLUSH /* video */));
963
964            mDeferredActions.push_back(
965                    new SeekAction(seekTimeUs, needNotify));
966
967            // After a flush without shutdown, decoder is paused.
968            // Don't resume it until source seek is done, otherwise it could
969            // start pulling stale data too soon.
970            mDeferredActions.push_back(
971                    new ResumeDecoderAction(needNotify));
972
973            processDeferredActions();
974            break;
975        }
976
977        case kWhatPause:
978        {
979            onPause();
980            mPausedByClient = true;
981            break;
982        }
983
984        case kWhatSourceNotify:
985        {
986            onSourceNotify(msg);
987            break;
988        }
989
990        case kWhatClosedCaptionNotify:
991        {
992            onClosedCaptionNotify(msg);
993            break;
994        }
995
996        default:
997            TRESPASS();
998            break;
999    }
1000}
1001
1002void NuPlayer::onResume() {
1003    if (!mPaused) {
1004        return;
1005    }
1006    mPaused = false;
1007    if (mSource != NULL) {
1008        mSource->resume();
1009    } else {
1010        ALOGW("resume called when source is gone or not set");
1011    }
1012    // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if
1013    // needed.
1014    if (audioDecoderStillNeeded() && mAudioDecoder == NULL) {
1015        instantiateDecoder(true /* audio */, &mAudioDecoder);
1016    }
1017    if (mRenderer != NULL) {
1018        mRenderer->resume();
1019    } else {
1020        ALOGW("resume called when renderer is gone or not set");
1021    }
1022}
1023
1024void NuPlayer::onStart() {
1025    mOffloadAudio = false;
1026    mAudioEOS = false;
1027    mVideoEOS = false;
1028    mStarted = true;
1029
1030    /* instantiate decoders now for secure playback */
1031    if (mSourceFlags & Source::FLAG_SECURE) {
1032        if (mNativeWindow != NULL) {
1033            instantiateDecoder(false, &mVideoDecoder);
1034        }
1035
1036        if (mAudioSink != NULL) {
1037            instantiateDecoder(true, &mAudioDecoder);
1038        }
1039    }
1040
1041    mSource->start();
1042
1043    uint32_t flags = 0;
1044
1045    if (mSource->isRealTime()) {
1046        flags |= Renderer::FLAG_REAL_TIME;
1047    }
1048
1049    sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
1050    audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
1051    if (mAudioSink != NULL) {
1052        streamType = mAudioSink->getAudioStreamType();
1053    }
1054
1055    sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
1056
1057    mOffloadAudio =
1058        canOffloadStream(audioMeta, (videoFormat != NULL),
1059                         true /* is_streaming */, streamType);
1060    if (mOffloadAudio) {
1061        flags |= Renderer::FLAG_OFFLOAD_AUDIO;
1062    }
1063
1064    sp<AMessage> notify = new AMessage(kWhatRendererNotify, this);
1065    ++mRendererGeneration;
1066    notify->setInt32("generation", mRendererGeneration);
1067    mRenderer = new Renderer(mAudioSink, notify, flags);
1068    mRendererLooper = new ALooper;
1069    mRendererLooper->setName("NuPlayerRenderer");
1070    mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
1071    mRendererLooper->registerHandler(mRenderer);
1072    if (mPlaybackRate != 1.0) {
1073        mRenderer->setPlaybackRate(mPlaybackRate);
1074    }
1075
1076    sp<MetaData> meta = getFileMeta();
1077    int32_t rate;
1078    if (meta != NULL
1079            && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) {
1080        mRenderer->setVideoFrameRate(rate);
1081    }
1082
1083    if (mVideoDecoder != NULL) {
1084        mVideoDecoder->setRenderer(mRenderer);
1085    }
1086    if (mAudioDecoder != NULL) {
1087        mAudioDecoder->setRenderer(mRenderer);
1088    }
1089
1090    postScanSources();
1091}
1092
1093void NuPlayer::onPause() {
1094    if (mPaused) {
1095        return;
1096    }
1097    mPaused = true;
1098    if (mSource != NULL) {
1099        mSource->pause();
1100    } else {
1101        ALOGW("pause called when source is gone or not set");
1102    }
1103    if (mRenderer != NULL) {
1104        mRenderer->pause();
1105    } else {
1106        ALOGW("pause called when renderer is gone or not set");
1107    }
1108}
1109
1110bool NuPlayer::audioDecoderStillNeeded() {
1111    // Audio decoder is no longer needed if it's in shut/shutting down status.
1112    return ((mFlushingAudio != SHUT_DOWN) && (mFlushingAudio != SHUTTING_DOWN_DECODER));
1113}
1114
1115void NuPlayer::handleFlushComplete(bool audio, bool isDecoder) {
1116    // We wait for both the decoder flush and the renderer flush to complete
1117    // before entering either the FLUSHED or the SHUTTING_DOWN_DECODER state.
1118
1119    mFlushComplete[audio][isDecoder] = true;
1120    if (!mFlushComplete[audio][!isDecoder]) {
1121        return;
1122    }
1123
1124    FlushStatus *state = audio ? &mFlushingAudio : &mFlushingVideo;
1125    switch (*state) {
1126        case FLUSHING_DECODER:
1127        {
1128            *state = FLUSHED;
1129            break;
1130        }
1131
1132        case FLUSHING_DECODER_SHUTDOWN:
1133        {
1134            *state = SHUTTING_DOWN_DECODER;
1135
1136            ALOGV("initiating %s decoder shutdown", audio ? "audio" : "video");
1137            if (!audio) {
1138                // Widevine source reads must stop before releasing the video decoder.
1139                if (mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
1140                    mSource->stop();
1141                }
1142            }
1143            getDecoder(audio)->initiateShutdown();
1144            break;
1145        }
1146
1147        default:
1148            // decoder flush completes only occur in a flushing state.
1149            LOG_ALWAYS_FATAL_IF(isDecoder, "decoder flush in invalid state %d", *state);
1150            break;
1151    }
1152}
1153
1154void NuPlayer::finishFlushIfPossible() {
1155    if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED
1156            && mFlushingAudio != SHUT_DOWN) {
1157        return;
1158    }
1159
1160    if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED
1161            && mFlushingVideo != SHUT_DOWN) {
1162        return;
1163    }
1164
1165    ALOGV("both audio and video are flushed now.");
1166
1167    mFlushingAudio = NONE;
1168    mFlushingVideo = NONE;
1169
1170    clearFlushComplete();
1171
1172    processDeferredActions();
1173}
1174
1175void NuPlayer::postScanSources() {
1176    if (mScanSourcesPending) {
1177        return;
1178    }
1179
1180    sp<AMessage> msg = new AMessage(kWhatScanSources, this);
1181    msg->setInt32("generation", mScanSourcesGeneration);
1182    msg->post();
1183
1184    mScanSourcesPending = true;
1185}
1186
1187void NuPlayer::tryOpenAudioSinkForOffload(const sp<AMessage> &format, bool hasVideo) {
1188    // Note: This is called early in NuPlayer to determine whether offloading
1189    // is possible; otherwise the decoders call the renderer openAudioSink directly.
1190
1191    status_t err = mRenderer->openAudioSink(
1192            format, true /* offloadOnly */, hasVideo, AUDIO_OUTPUT_FLAG_NONE, &mOffloadAudio);
1193    if (err != OK) {
1194        // Any failure we turn off mOffloadAudio.
1195        mOffloadAudio = false;
1196    } else if (mOffloadAudio) {
1197        sp<MetaData> audioMeta =
1198                mSource->getFormatMeta(true /* audio */);
1199        sendMetaDataToHal(mAudioSink, audioMeta);
1200    }
1201}
1202
1203void NuPlayer::closeAudioSink() {
1204    mRenderer->closeAudioSink();
1205}
1206
1207status_t NuPlayer::instantiateDecoder(bool audio, sp<DecoderBase> *decoder) {
1208    if (*decoder != NULL) {
1209        return OK;
1210    }
1211
1212    sp<AMessage> format = mSource->getFormat(audio);
1213
1214    if (format == NULL) {
1215        return -EWOULDBLOCK;
1216    }
1217
1218    if (!audio) {
1219        AString mime;
1220        CHECK(format->findString("mime", &mime));
1221
1222        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, this);
1223        if (mCCDecoder == NULL) {
1224            mCCDecoder = new CCDecoder(ccNotify);
1225        }
1226
1227        if (mSourceFlags & Source::FLAG_SECURE) {
1228            format->setInt32("secure", true);
1229        }
1230
1231        if (mSourceFlags & Source::FLAG_PROTECTED) {
1232            format->setInt32("protected", true);
1233        }
1234    }
1235
1236    if (audio) {
1237        sp<AMessage> notify = new AMessage(kWhatAudioNotify, this);
1238        ++mAudioDecoderGeneration;
1239        notify->setInt32("generation", mAudioDecoderGeneration);
1240
1241        if (mOffloadAudio) {
1242            const bool hasVideo = (mSource->getFormat(false /*audio */) != NULL);
1243            format->setInt32("has-video", hasVideo);
1244            *decoder = new DecoderPassThrough(notify, mSource, mRenderer);
1245        } else {
1246            *decoder = new Decoder(notify, mSource, mRenderer);
1247        }
1248    } else {
1249        sp<AMessage> notify = new AMessage(kWhatVideoNotify, this);
1250        ++mVideoDecoderGeneration;
1251        notify->setInt32("generation", mVideoDecoderGeneration);
1252
1253        *decoder = new Decoder(
1254                notify, mSource, mRenderer, mNativeWindow, mCCDecoder);
1255
1256        // enable FRC if high-quality AV sync is requested, even if not
1257        // queuing to native window, as this will even improve textureview
1258        // playback.
1259        {
1260            char value[PROPERTY_VALUE_MAX];
1261            if (property_get("persist.sys.media.avsync", value, NULL) &&
1262                    (!strcmp("1", value) || !strcasecmp("true", value))) {
1263                format->setInt32("auto-frc", 1);
1264            }
1265        }
1266    }
1267    (*decoder)->init();
1268    (*decoder)->configure(format);
1269
1270    // allocate buffers to decrypt widevine source buffers
1271    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1272        Vector<sp<ABuffer> > inputBufs;
1273        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1274
1275        Vector<MediaBuffer *> mediaBufs;
1276        for (size_t i = 0; i < inputBufs.size(); i++) {
1277            const sp<ABuffer> &buffer = inputBufs[i];
1278            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1279            mediaBufs.push(mbuf);
1280        }
1281
1282        status_t err = mSource->setBuffers(audio, mediaBufs);
1283        if (err != OK) {
1284            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1285                mediaBufs[i]->release();
1286            }
1287            mediaBufs.clear();
1288            ALOGE("Secure source didn't support secure mediaBufs.");
1289            return err;
1290        }
1291    }
1292    return OK;
1293}
1294
1295void NuPlayer::updateVideoSize(
1296        const sp<AMessage> &inputFormat,
1297        const sp<AMessage> &outputFormat) {
1298    if (inputFormat == NULL) {
1299        ALOGW("Unknown video size, reporting 0x0!");
1300        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1301        return;
1302    }
1303
1304    int32_t displayWidth, displayHeight;
1305    int32_t cropLeft, cropTop, cropRight, cropBottom;
1306
1307    if (outputFormat != NULL) {
1308        int32_t width, height;
1309        CHECK(outputFormat->findInt32("width", &width));
1310        CHECK(outputFormat->findInt32("height", &height));
1311
1312        int32_t cropLeft, cropTop, cropRight, cropBottom;
1313        CHECK(outputFormat->findRect(
1314                    "crop",
1315                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1316
1317        displayWidth = cropRight - cropLeft + 1;
1318        displayHeight = cropBottom - cropTop + 1;
1319
1320        ALOGV("Video output format changed to %d x %d "
1321             "(crop: %d x %d @ (%d, %d))",
1322             width, height,
1323             displayWidth,
1324             displayHeight,
1325             cropLeft, cropTop);
1326    } else {
1327        CHECK(inputFormat->findInt32("width", &displayWidth));
1328        CHECK(inputFormat->findInt32("height", &displayHeight));
1329
1330        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1331    }
1332
1333    // Take into account sample aspect ratio if necessary:
1334    int32_t sarWidth, sarHeight;
1335    if (inputFormat->findInt32("sar-width", &sarWidth)
1336            && inputFormat->findInt32("sar-height", &sarHeight)) {
1337        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1338
1339        displayWidth = (displayWidth * sarWidth) / sarHeight;
1340
1341        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1342    }
1343
1344    int32_t rotationDegrees;
1345    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1346        rotationDegrees = 0;
1347    }
1348
1349    if (rotationDegrees == 90 || rotationDegrees == 270) {
1350        int32_t tmp = displayWidth;
1351        displayWidth = displayHeight;
1352        displayHeight = tmp;
1353    }
1354
1355    notifyListener(
1356            MEDIA_SET_VIDEO_SIZE,
1357            displayWidth,
1358            displayHeight);
1359}
1360
1361void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1362    if (mDriver == NULL) {
1363        return;
1364    }
1365
1366    sp<NuPlayerDriver> driver = mDriver.promote();
1367
1368    if (driver == NULL) {
1369        return;
1370    }
1371
1372    driver->notifyListener(msg, ext1, ext2, in);
1373}
1374
1375void NuPlayer::flushDecoder(bool audio, bool needShutdown) {
1376    ALOGV("[%s] flushDecoder needShutdown=%d",
1377          audio ? "audio" : "video", needShutdown);
1378
1379    const sp<DecoderBase> &decoder = getDecoder(audio);
1380    if (decoder == NULL) {
1381        ALOGI("flushDecoder %s without decoder present",
1382             audio ? "audio" : "video");
1383        return;
1384    }
1385
1386    // Make sure we don't continue to scan sources until we finish flushing.
1387    ++mScanSourcesGeneration;
1388    mScanSourcesPending = false;
1389
1390    decoder->signalFlush();
1391
1392    FlushStatus newStatus =
1393        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1394
1395    mFlushComplete[audio][false /* isDecoder */] = false;
1396    mFlushComplete[audio][true /* isDecoder */] = false;
1397    if (audio) {
1398        ALOGE_IF(mFlushingAudio != NONE,
1399                "audio flushDecoder() is called in state %d", mFlushingAudio);
1400        mFlushingAudio = newStatus;
1401    } else {
1402        ALOGE_IF(mFlushingVideo != NONE,
1403                "video flushDecoder() is called in state %d", mFlushingVideo);
1404        mFlushingVideo = newStatus;
1405    }
1406}
1407
1408void NuPlayer::queueDecoderShutdown(
1409        bool audio, bool video, const sp<AMessage> &reply) {
1410    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1411
1412    mDeferredActions.push_back(
1413            new FlushDecoderAction(
1414                audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
1415                video ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE));
1416
1417    mDeferredActions.push_back(
1418            new SimpleAction(&NuPlayer::performScanSources));
1419
1420    mDeferredActions.push_back(new PostMessageAction(reply));
1421
1422    processDeferredActions();
1423}
1424
1425status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1426    mVideoScalingMode = mode;
1427    if (mNativeWindow != NULL) {
1428        status_t ret = native_window_set_scaling_mode(
1429                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1430        if (ret != OK) {
1431            ALOGE("Failed to set scaling mode (%d): %s",
1432                -ret, strerror(-ret));
1433            return ret;
1434        }
1435    }
1436    return OK;
1437}
1438
1439status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1440    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, this);
1441    msg->setPointer("reply", reply);
1442
1443    sp<AMessage> response;
1444    status_t err = msg->postAndAwaitResponse(&response);
1445    return err;
1446}
1447
1448status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1449    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, this);
1450    msg->setPointer("reply", reply);
1451    msg->setInt32("type", type);
1452
1453    sp<AMessage> response;
1454    status_t err = msg->postAndAwaitResponse(&response);
1455    if (err == OK && response != NULL) {
1456        CHECK(response->findInt32("err", &err));
1457    }
1458    return err;
1459}
1460
1461status_t NuPlayer::selectTrack(size_t trackIndex, bool select, int64_t timeUs) {
1462    sp<AMessage> msg = new AMessage(kWhatSelectTrack, this);
1463    msg->setSize("trackIndex", trackIndex);
1464    msg->setInt32("select", select);
1465    msg->setInt64("timeUs", timeUs);
1466
1467    sp<AMessage> response;
1468    status_t err = msg->postAndAwaitResponse(&response);
1469
1470    if (err != OK) {
1471        return err;
1472    }
1473
1474    if (!response->findInt32("err", &err)) {
1475        err = OK;
1476    }
1477
1478    return err;
1479}
1480
1481status_t NuPlayer::getCurrentPosition(int64_t *mediaUs) {
1482    sp<Renderer> renderer = mRenderer;
1483    if (renderer == NULL) {
1484        return NO_INIT;
1485    }
1486
1487    return renderer->getCurrentPosition(mediaUs);
1488}
1489
1490void NuPlayer::getStats(int64_t *numFramesTotal, int64_t *numFramesDropped) {
1491    sp<DecoderBase> decoder = getDecoder(false /* audio */);
1492    if (decoder != NULL) {
1493        decoder->getStats(numFramesTotal, numFramesDropped);
1494    } else {
1495        *numFramesTotal = 0;
1496        *numFramesDropped = 0;
1497    }
1498}
1499
1500sp<MetaData> NuPlayer::getFileMeta() {
1501    return mSource->getFileFormatMeta();
1502}
1503
1504void NuPlayer::schedulePollDuration() {
1505    sp<AMessage> msg = new AMessage(kWhatPollDuration, this);
1506    msg->setInt32("generation", mPollDurationGeneration);
1507    msg->post();
1508}
1509
1510void NuPlayer::cancelPollDuration() {
1511    ++mPollDurationGeneration;
1512}
1513
1514void NuPlayer::processDeferredActions() {
1515    while (!mDeferredActions.empty()) {
1516        // We won't execute any deferred actions until we're no longer in
1517        // an intermediate state, i.e. one more more decoders are currently
1518        // flushing or shutting down.
1519
1520        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1521            // We're currently flushing, postpone the reset until that's
1522            // completed.
1523
1524            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1525                  mFlushingAudio, mFlushingVideo);
1526
1527            break;
1528        }
1529
1530        sp<Action> action = *mDeferredActions.begin();
1531        mDeferredActions.erase(mDeferredActions.begin());
1532
1533        action->execute(this);
1534    }
1535}
1536
1537void NuPlayer::performSeek(int64_t seekTimeUs, bool needNotify) {
1538    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs), needNotify(%d)",
1539          seekTimeUs,
1540          seekTimeUs / 1E6,
1541          needNotify);
1542
1543    if (mSource == NULL) {
1544        // This happens when reset occurs right before the loop mode
1545        // asynchronously seeks to the start of the stream.
1546        LOG_ALWAYS_FATAL_IF(mAudioDecoder != NULL || mVideoDecoder != NULL,
1547                "mSource is NULL and decoders not NULL audio(%p) video(%p)",
1548                mAudioDecoder.get(), mVideoDecoder.get());
1549        return;
1550    }
1551    mSource->seekTo(seekTimeUs);
1552    ++mTimedTextGeneration;
1553
1554    // everything's flushed, continue playback.
1555}
1556
1557void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) {
1558    ALOGV("performDecoderFlush audio=%d, video=%d", audio, video);
1559
1560    if ((audio == FLUSH_CMD_NONE || mAudioDecoder == NULL)
1561            && (video == FLUSH_CMD_NONE || mVideoDecoder == NULL)) {
1562        return;
1563    }
1564
1565    if (audio != FLUSH_CMD_NONE && mAudioDecoder != NULL) {
1566        flushDecoder(true /* audio */, (audio == FLUSH_CMD_SHUTDOWN));
1567    }
1568
1569    if (video != FLUSH_CMD_NONE && mVideoDecoder != NULL) {
1570        flushDecoder(false /* audio */, (video == FLUSH_CMD_SHUTDOWN));
1571    }
1572}
1573
1574void NuPlayer::performReset() {
1575    ALOGV("performReset");
1576
1577    CHECK(mAudioDecoder == NULL);
1578    CHECK(mVideoDecoder == NULL);
1579
1580    cancelPollDuration();
1581
1582    ++mScanSourcesGeneration;
1583    mScanSourcesPending = false;
1584
1585    if (mRendererLooper != NULL) {
1586        if (mRenderer != NULL) {
1587            mRendererLooper->unregisterHandler(mRenderer->id());
1588        }
1589        mRendererLooper->stop();
1590        mRendererLooper.clear();
1591    }
1592    mRenderer.clear();
1593    ++mRendererGeneration;
1594
1595    if (mSource != NULL) {
1596        mSource->stop();
1597
1598        mSource.clear();
1599    }
1600
1601    if (mDriver != NULL) {
1602        sp<NuPlayerDriver> driver = mDriver.promote();
1603        if (driver != NULL) {
1604            driver->notifyResetComplete();
1605        }
1606    }
1607
1608    mStarted = false;
1609}
1610
1611void NuPlayer::performScanSources() {
1612    ALOGV("performScanSources");
1613
1614    if (!mStarted) {
1615        return;
1616    }
1617
1618    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1619        postScanSources();
1620    }
1621}
1622
1623void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1624    ALOGV("performSetSurface");
1625
1626    mNativeWindow = wrapper;
1627
1628    // XXX - ignore error from setVideoScalingMode for now
1629    setVideoScalingMode(mVideoScalingMode);
1630
1631    if (mDriver != NULL) {
1632        sp<NuPlayerDriver> driver = mDriver.promote();
1633        if (driver != NULL) {
1634            driver->notifySetSurfaceComplete();
1635        }
1636    }
1637}
1638
1639void NuPlayer::performResumeDecoders(bool needNotify) {
1640    if (needNotify) {
1641        mResumePending = true;
1642        if (mVideoDecoder == NULL) {
1643            // if audio-only, we can notify seek complete now,
1644            // as the resume operation will be relatively fast.
1645            finishResume();
1646        }
1647    }
1648
1649    if (mVideoDecoder != NULL) {
1650        // When there is continuous seek, MediaPlayer will cache the seek
1651        // position, and send down new seek request when previous seek is
1652        // complete. Let's wait for at least one video output frame before
1653        // notifying seek complete, so that the video thumbnail gets updated
1654        // when seekbar is dragged.
1655        mVideoDecoder->signalResume(needNotify);
1656    }
1657
1658    if (mAudioDecoder != NULL) {
1659        mAudioDecoder->signalResume(false /* needNotify */);
1660    }
1661}
1662
1663void NuPlayer::finishResume() {
1664    if (mResumePending) {
1665        mResumePending = false;
1666        if (mDriver != NULL) {
1667            sp<NuPlayerDriver> driver = mDriver.promote();
1668            if (driver != NULL) {
1669                driver->notifySeekComplete();
1670            }
1671        }
1672    }
1673}
1674
1675void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1676    int32_t what;
1677    CHECK(msg->findInt32("what", &what));
1678
1679    switch (what) {
1680        case Source::kWhatPrepared:
1681        {
1682            if (mSource == NULL) {
1683                // This is a stale notification from a source that was
1684                // asynchronously preparing when the client called reset().
1685                // We handled the reset, the source is gone.
1686                break;
1687            }
1688
1689            int32_t err;
1690            CHECK(msg->findInt32("err", &err));
1691
1692            sp<NuPlayerDriver> driver = mDriver.promote();
1693            if (driver != NULL) {
1694                // notify duration first, so that it's definitely set when
1695                // the app received the "prepare complete" callback.
1696                int64_t durationUs;
1697                if (mSource->getDuration(&durationUs) == OK) {
1698                    driver->notifyDuration(durationUs);
1699                }
1700                driver->notifyPrepareCompleted(err);
1701            }
1702
1703            break;
1704        }
1705
1706        case Source::kWhatFlagsChanged:
1707        {
1708            uint32_t flags;
1709            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1710
1711            sp<NuPlayerDriver> driver = mDriver.promote();
1712            if (driver != NULL) {
1713                if ((flags & NuPlayer::Source::FLAG_CAN_SEEK) == 0) {
1714                    driver->notifyListener(
1715                            MEDIA_INFO, MEDIA_INFO_NOT_SEEKABLE, 0);
1716                }
1717                driver->notifyFlagsChanged(flags);
1718            }
1719
1720            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1721                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
1722                cancelPollDuration();
1723            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1724                    && (flags & Source::FLAG_DYNAMIC_DURATION)
1725                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
1726                schedulePollDuration();
1727            }
1728
1729            mSourceFlags = flags;
1730            break;
1731        }
1732
1733        case Source::kWhatVideoSizeChanged:
1734        {
1735            sp<AMessage> format;
1736            CHECK(msg->findMessage("format", &format));
1737
1738            updateVideoSize(format);
1739            break;
1740        }
1741
1742        case Source::kWhatBufferingUpdate:
1743        {
1744            int32_t percentage;
1745            CHECK(msg->findInt32("percentage", &percentage));
1746
1747            notifyListener(MEDIA_BUFFERING_UPDATE, percentage, 0);
1748            break;
1749        }
1750
1751        case Source::kWhatPauseOnBufferingStart:
1752        {
1753            // ignore if not playing
1754            if (mStarted && !mPausedByClient) {
1755                ALOGI("buffer low, pausing...");
1756
1757                onPause();
1758            }
1759            // fall-thru
1760        }
1761
1762        case Source::kWhatBufferingStart:
1763        {
1764            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
1765            break;
1766        }
1767
1768        case Source::kWhatResumeOnBufferingEnd:
1769        {
1770            // ignore if not playing
1771            if (mStarted && !mPausedByClient) {
1772                ALOGI("buffer ready, resuming...");
1773
1774                onResume();
1775            }
1776            // fall-thru
1777        }
1778
1779        case Source::kWhatBufferingEnd:
1780        {
1781            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
1782            break;
1783        }
1784
1785        case Source::kWhatCacheStats:
1786        {
1787            int32_t kbps;
1788            CHECK(msg->findInt32("bandwidth", &kbps));
1789
1790            notifyListener(MEDIA_INFO, MEDIA_INFO_NETWORK_BANDWIDTH, kbps);
1791            break;
1792        }
1793
1794        case Source::kWhatSubtitleData:
1795        {
1796            sp<ABuffer> buffer;
1797            CHECK(msg->findBuffer("buffer", &buffer));
1798
1799            sendSubtitleData(buffer, 0 /* baseIndex */);
1800            break;
1801        }
1802
1803        case Source::kWhatTimedTextData:
1804        {
1805            int32_t generation;
1806            if (msg->findInt32("generation", &generation)
1807                    && generation != mTimedTextGeneration) {
1808                break;
1809            }
1810
1811            sp<ABuffer> buffer;
1812            CHECK(msg->findBuffer("buffer", &buffer));
1813
1814            sp<NuPlayerDriver> driver = mDriver.promote();
1815            if (driver == NULL) {
1816                break;
1817            }
1818
1819            int posMs;
1820            int64_t timeUs, posUs;
1821            driver->getCurrentPosition(&posMs);
1822            posUs = posMs * 1000;
1823            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1824
1825            if (posUs < timeUs) {
1826                if (!msg->findInt32("generation", &generation)) {
1827                    msg->setInt32("generation", mTimedTextGeneration);
1828                }
1829                msg->post(timeUs - posUs);
1830            } else {
1831                sendTimedTextData(buffer);
1832            }
1833            break;
1834        }
1835
1836        case Source::kWhatQueueDecoderShutdown:
1837        {
1838            int32_t audio, video;
1839            CHECK(msg->findInt32("audio", &audio));
1840            CHECK(msg->findInt32("video", &video));
1841
1842            sp<AMessage> reply;
1843            CHECK(msg->findMessage("reply", &reply));
1844
1845            queueDecoderShutdown(audio, video, reply);
1846            break;
1847        }
1848
1849        case Source::kWhatDrmNoLicense:
1850        {
1851            notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, ERROR_DRM_NO_LICENSE);
1852            break;
1853        }
1854
1855        default:
1856            TRESPASS();
1857    }
1858}
1859
1860void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
1861    int32_t what;
1862    CHECK(msg->findInt32("what", &what));
1863
1864    switch (what) {
1865        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
1866        {
1867            sp<ABuffer> buffer;
1868            CHECK(msg->findBuffer("buffer", &buffer));
1869
1870            size_t inbandTracks = 0;
1871            if (mSource != NULL) {
1872                inbandTracks = mSource->getTrackCount();
1873            }
1874
1875            sendSubtitleData(buffer, inbandTracks);
1876            break;
1877        }
1878
1879        case NuPlayer::CCDecoder::kWhatTrackAdded:
1880        {
1881            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
1882
1883            break;
1884        }
1885
1886        default:
1887            TRESPASS();
1888    }
1889
1890
1891}
1892
1893void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
1894    int32_t trackIndex;
1895    int64_t timeUs, durationUs;
1896    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
1897    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1898    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
1899
1900    Parcel in;
1901    in.writeInt32(trackIndex + baseIndex);
1902    in.writeInt64(timeUs);
1903    in.writeInt64(durationUs);
1904    in.writeInt32(buffer->size());
1905    in.writeInt32(buffer->size());
1906    in.write(buffer->data(), buffer->size());
1907
1908    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
1909}
1910
1911void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
1912    const void *data;
1913    size_t size = 0;
1914    int64_t timeUs;
1915    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
1916
1917    AString mime;
1918    CHECK(buffer->meta()->findString("mime", &mime));
1919    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
1920
1921    data = buffer->data();
1922    size = buffer->size();
1923
1924    Parcel parcel;
1925    if (size > 0) {
1926        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1927        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
1928        TextDescriptions::getParcelOfDescriptions(
1929                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
1930    }
1931
1932    if ((parcel.dataSize() > 0)) {
1933        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
1934    } else {  // send an empty timed text
1935        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
1936    }
1937}
1938////////////////////////////////////////////////////////////////////////////////
1939
1940sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
1941    sp<MetaData> meta = getFormatMeta(audio);
1942
1943    if (meta == NULL) {
1944        return NULL;
1945    }
1946
1947    sp<AMessage> msg = new AMessage;
1948
1949    if(convertMetaDataToMessage(meta, &msg) == OK) {
1950        return msg;
1951    }
1952    return NULL;
1953}
1954
1955void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
1956    sp<AMessage> notify = dupNotify();
1957    notify->setInt32("what", kWhatFlagsChanged);
1958    notify->setInt32("flags", flags);
1959    notify->post();
1960}
1961
1962void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
1963    sp<AMessage> notify = dupNotify();
1964    notify->setInt32("what", kWhatVideoSizeChanged);
1965    notify->setMessage("format", format);
1966    notify->post();
1967}
1968
1969void NuPlayer::Source::notifyPrepared(status_t err) {
1970    sp<AMessage> notify = dupNotify();
1971    notify->setInt32("what", kWhatPrepared);
1972    notify->setInt32("err", err);
1973    notify->post();
1974}
1975
1976void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
1977    TRESPASS();
1978}
1979
1980}  // namespace android
1981