NuPlayer.cpp revision fcd3e94c075e964670d946f6ec5d82d059bf9e09 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerCCDecoder.h"
25#include "NuPlayerDecoder.h"
26#include "NuPlayerDecoderBase.h"
27#include "NuPlayerDecoderPassThrough.h"
28#include "NuPlayerDriver.h"
29#include "NuPlayerRenderer.h"
30#include "NuPlayerSource.h"
31#include "RTSPSource.h"
32#include "StreamingSource.h"
33#include "GenericSource.h"
34#include "TextDescriptions.h"
35
36#include "ATSParser.h"
37
38#include <cutils/properties.h>
39
40#include <media/stagefright/foundation/hexdump.h>
41#include <media/stagefright/foundation/ABuffer.h>
42#include <media/stagefright/foundation/ADebug.h>
43#include <media/stagefright/foundation/AMessage.h>
44#include <media/stagefright/MediaBuffer.h>
45#include <media/stagefright/MediaDefs.h>
46#include <media/stagefright/MediaErrors.h>
47#include <media/stagefright/MetaData.h>
48#include <gui/IGraphicBufferProducer.h>
49
50#include "avc_utils.h"
51
52#include "ESDS.h"
53#include <media/stagefright/Utils.h>
54
55namespace android {
56
57struct NuPlayer::Action : public RefBase {
58    Action() {}
59
60    virtual void execute(NuPlayer *player) = 0;
61
62private:
63    DISALLOW_EVIL_CONSTRUCTORS(Action);
64};
65
66struct NuPlayer::SeekAction : public Action {
67    SeekAction(int64_t seekTimeUs, bool needNotify)
68        : mSeekTimeUs(seekTimeUs),
69          mNeedNotify(needNotify) {
70    }
71
72    virtual void execute(NuPlayer *player) {
73        player->performSeek(mSeekTimeUs, mNeedNotify);
74    }
75
76private:
77    int64_t mSeekTimeUs;
78    bool mNeedNotify;
79
80    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
81};
82
83struct NuPlayer::ResumeDecoderAction : public Action {
84    ResumeDecoderAction(bool needNotify)
85        : mNeedNotify(needNotify) {
86    }
87
88    virtual void execute(NuPlayer *player) {
89        player->performResumeDecoders(mNeedNotify);
90    }
91
92private:
93    bool mNeedNotify;
94
95    DISALLOW_EVIL_CONSTRUCTORS(ResumeDecoderAction);
96};
97
98struct NuPlayer::SetSurfaceAction : public Action {
99    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
100        : mWrapper(wrapper) {
101    }
102
103    virtual void execute(NuPlayer *player) {
104        player->performSetSurface(mWrapper);
105    }
106
107private:
108    sp<NativeWindowWrapper> mWrapper;
109
110    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
111};
112
113struct NuPlayer::FlushDecoderAction : public Action {
114    FlushDecoderAction(FlushCommand audio, FlushCommand video)
115        : mAudio(audio),
116          mVideo(video) {
117    }
118
119    virtual void execute(NuPlayer *player) {
120        player->performDecoderFlush(mAudio, mVideo);
121    }
122
123private:
124    FlushCommand mAudio;
125    FlushCommand mVideo;
126
127    DISALLOW_EVIL_CONSTRUCTORS(FlushDecoderAction);
128};
129
130struct NuPlayer::PostMessageAction : public Action {
131    PostMessageAction(const sp<AMessage> &msg)
132        : mMessage(msg) {
133    }
134
135    virtual void execute(NuPlayer *) {
136        mMessage->post();
137    }
138
139private:
140    sp<AMessage> mMessage;
141
142    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
143};
144
145// Use this if there's no state necessary to save in order to execute
146// the action.
147struct NuPlayer::SimpleAction : public Action {
148    typedef void (NuPlayer::*ActionFunc)();
149
150    SimpleAction(ActionFunc func)
151        : mFunc(func) {
152    }
153
154    virtual void execute(NuPlayer *player) {
155        (player->*mFunc)();
156    }
157
158private:
159    ActionFunc mFunc;
160
161    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
162};
163
164////////////////////////////////////////////////////////////////////////////////
165
166NuPlayer::NuPlayer()
167    : mUIDValid(false),
168      mSourceFlags(0),
169      mOffloadAudio(false),
170      mAudioDecoderGeneration(0),
171      mVideoDecoderGeneration(0),
172      mRendererGeneration(0),
173      mAudioEOS(false),
174      mVideoEOS(false),
175      mScanSourcesPending(false),
176      mScanSourcesGeneration(0),
177      mPollDurationGeneration(0),
178      mTimedTextGeneration(0),
179      mFlushingAudio(NONE),
180      mFlushingVideo(NONE),
181      mResumePending(false),
182      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
183      mPlaybackRate(1.0),
184      mStarted(false),
185      mPaused(false),
186      mPausedByClient(false) {
187    clearFlushComplete();
188}
189
190NuPlayer::~NuPlayer() {
191}
192
193void NuPlayer::setUID(uid_t uid) {
194    mUIDValid = true;
195    mUID = uid;
196}
197
198void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
199    mDriver = driver;
200}
201
202void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
203    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
204
205    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
206
207    msg->setObject("source", new StreamingSource(notify, source));
208    msg->post();
209}
210
211static bool IsHTTPLiveURL(const char *url) {
212    if (!strncasecmp("http://", url, 7)
213            || !strncasecmp("https://", url, 8)
214            || !strncasecmp("file://", url, 7)) {
215        size_t len = strlen(url);
216        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
217            return true;
218        }
219
220        if (strstr(url,"m3u8")) {
221            return true;
222        }
223    }
224
225    return false;
226}
227
228void NuPlayer::setDataSourceAsync(
229        const sp<IMediaHTTPService> &httpService,
230        const char *url,
231        const KeyedVector<String8, String8> *headers) {
232
233    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
234    size_t len = strlen(url);
235
236    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
237
238    sp<Source> source;
239    if (IsHTTPLiveURL(url)) {
240        source = new HTTPLiveSource(notify, httpService, url, headers);
241    } else if (!strncasecmp(url, "rtsp://", 7)) {
242        source = new RTSPSource(
243                notify, httpService, url, headers, mUIDValid, mUID);
244    } else if ((!strncasecmp(url, "http://", 7)
245                || !strncasecmp(url, "https://", 8))
246                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
247                    || strstr(url, ".sdp?"))) {
248        source = new RTSPSource(
249                notify, httpService, url, headers, mUIDValid, mUID, true);
250    } else {
251        sp<GenericSource> genericSource =
252                new GenericSource(notify, mUIDValid, mUID);
253        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
254        // The correct flags will be updated in Source::kWhatFlagsChanged
255        // handler when  GenericSource is prepared.
256
257        status_t err = genericSource->setDataSource(httpService, url, headers);
258
259        if (err == OK) {
260            source = genericSource;
261        } else {
262            ALOGE("Failed to set data source!");
263        }
264    }
265    msg->setObject("source", source);
266    msg->post();
267}
268
269void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
270    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
271
272    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
273
274    sp<GenericSource> source =
275            new GenericSource(notify, mUIDValid, mUID);
276
277    status_t err = source->setDataSource(fd, offset, length);
278
279    if (err != OK) {
280        ALOGE("Failed to set data source!");
281        source = NULL;
282    }
283
284    msg->setObject("source", source);
285    msg->post();
286}
287
288void NuPlayer::prepareAsync() {
289    (new AMessage(kWhatPrepare, this))->post();
290}
291
292void NuPlayer::setVideoSurfaceTextureAsync(
293        const sp<IGraphicBufferProducer> &bufferProducer) {
294    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, this);
295
296    if (bufferProducer == NULL) {
297        msg->setObject("native-window", NULL);
298    } else {
299        msg->setObject(
300                "native-window",
301                new NativeWindowWrapper(
302                    new Surface(bufferProducer, true /* controlledByApp */)));
303    }
304
305    msg->post();
306}
307
308void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
309    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, this);
310    msg->setObject("sink", sink);
311    msg->post();
312}
313
314void NuPlayer::start() {
315    (new AMessage(kWhatStart, this))->post();
316}
317
318void NuPlayer::setPlaybackRate(float rate) {
319    sp<AMessage> msg = new AMessage(kWhatSetRate, this);
320    msg->setFloat("rate", rate);
321    msg->post();
322}
323
324void NuPlayer::pause() {
325    (new AMessage(kWhatPause, this))->post();
326}
327
328void NuPlayer::resetAsync() {
329    if (mSource != NULL) {
330        // During a reset, the data source might be unresponsive already, we need to
331        // disconnect explicitly so that reads exit promptly.
332        // We can't queue the disconnect request to the looper, as it might be
333        // queued behind a stuck read and never gets processed.
334        // Doing a disconnect outside the looper to allows the pending reads to exit
335        // (either successfully or with error).
336        mSource->disconnect();
337    }
338
339    (new AMessage(kWhatReset, this))->post();
340}
341
342void NuPlayer::seekToAsync(int64_t seekTimeUs, bool needNotify) {
343    sp<AMessage> msg = new AMessage(kWhatSeek, this);
344    msg->setInt64("seekTimeUs", seekTimeUs);
345    msg->setInt32("needNotify", needNotify);
346    msg->post();
347}
348
349
350void NuPlayer::writeTrackInfo(
351        Parcel* reply, const sp<AMessage> format) const {
352    int32_t trackType;
353    CHECK(format->findInt32("type", &trackType));
354
355    AString lang;
356    CHECK(format->findString("language", &lang));
357
358    reply->writeInt32(2); // write something non-zero
359    reply->writeInt32(trackType);
360    reply->writeString16(String16(lang.c_str()));
361
362    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
363        AString mime;
364        CHECK(format->findString("mime", &mime));
365
366        int32_t isAuto, isDefault, isForced;
367        CHECK(format->findInt32("auto", &isAuto));
368        CHECK(format->findInt32("default", &isDefault));
369        CHECK(format->findInt32("forced", &isForced));
370
371        reply->writeString16(String16(mime.c_str()));
372        reply->writeInt32(isAuto);
373        reply->writeInt32(isDefault);
374        reply->writeInt32(isForced);
375    }
376}
377
378void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
379    switch (msg->what()) {
380        case kWhatSetDataSource:
381        {
382            ALOGV("kWhatSetDataSource");
383
384            CHECK(mSource == NULL);
385
386            status_t err = OK;
387            sp<RefBase> obj;
388            CHECK(msg->findObject("source", &obj));
389            if (obj != NULL) {
390                mSource = static_cast<Source *>(obj.get());
391            } else {
392                err = UNKNOWN_ERROR;
393            }
394
395            CHECK(mDriver != NULL);
396            sp<NuPlayerDriver> driver = mDriver.promote();
397            if (driver != NULL) {
398                driver->notifySetDataSourceCompleted(err);
399            }
400            break;
401        }
402
403        case kWhatPrepare:
404        {
405            mSource->prepareAsync();
406            break;
407        }
408
409        case kWhatGetTrackInfo:
410        {
411            sp<AReplyToken> replyID;
412            CHECK(msg->senderAwaitsResponse(&replyID));
413
414            Parcel* reply;
415            CHECK(msg->findPointer("reply", (void**)&reply));
416
417            size_t inbandTracks = 0;
418            if (mSource != NULL) {
419                inbandTracks = mSource->getTrackCount();
420            }
421
422            size_t ccTracks = 0;
423            if (mCCDecoder != NULL) {
424                ccTracks = mCCDecoder->getTrackCount();
425            }
426
427            // total track count
428            reply->writeInt32(inbandTracks + ccTracks);
429
430            // write inband tracks
431            for (size_t i = 0; i < inbandTracks; ++i) {
432                writeTrackInfo(reply, mSource->getTrackInfo(i));
433            }
434
435            // write CC track
436            for (size_t i = 0; i < ccTracks; ++i) {
437                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
438            }
439
440            sp<AMessage> response = new AMessage;
441            response->postReply(replyID);
442            break;
443        }
444
445        case kWhatGetSelectedTrack:
446        {
447            status_t err = INVALID_OPERATION;
448            if (mSource != NULL) {
449                err = OK;
450
451                int32_t type32;
452                CHECK(msg->findInt32("type", (int32_t*)&type32));
453                media_track_type type = (media_track_type)type32;
454                ssize_t selectedTrack = mSource->getSelectedTrack(type);
455
456                Parcel* reply;
457                CHECK(msg->findPointer("reply", (void**)&reply));
458                reply->writeInt32(selectedTrack);
459            }
460
461            sp<AMessage> response = new AMessage;
462            response->setInt32("err", err);
463
464            sp<AReplyToken> replyID;
465            CHECK(msg->senderAwaitsResponse(&replyID));
466            response->postReply(replyID);
467            break;
468        }
469
470        case kWhatSelectTrack:
471        {
472            sp<AReplyToken> replyID;
473            CHECK(msg->senderAwaitsResponse(&replyID));
474
475            size_t trackIndex;
476            int32_t select;
477            int64_t timeUs;
478            CHECK(msg->findSize("trackIndex", &trackIndex));
479            CHECK(msg->findInt32("select", &select));
480            CHECK(msg->findInt64("timeUs", &timeUs));
481
482            status_t err = INVALID_OPERATION;
483
484            size_t inbandTracks = 0;
485            if (mSource != NULL) {
486                inbandTracks = mSource->getTrackCount();
487            }
488            size_t ccTracks = 0;
489            if (mCCDecoder != NULL) {
490                ccTracks = mCCDecoder->getTrackCount();
491            }
492
493            if (trackIndex < inbandTracks) {
494                err = mSource->selectTrack(trackIndex, select, timeUs);
495
496                if (!select && err == OK) {
497                    int32_t type;
498                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
499                    if (info != NULL
500                            && info->findInt32("type", &type)
501                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
502                        ++mTimedTextGeneration;
503                    }
504                }
505            } else {
506                trackIndex -= inbandTracks;
507
508                if (trackIndex < ccTracks) {
509                    err = mCCDecoder->selectTrack(trackIndex, select);
510                }
511            }
512
513            sp<AMessage> response = new AMessage;
514            response->setInt32("err", err);
515
516            response->postReply(replyID);
517            break;
518        }
519
520        case kWhatPollDuration:
521        {
522            int32_t generation;
523            CHECK(msg->findInt32("generation", &generation));
524
525            if (generation != mPollDurationGeneration) {
526                // stale
527                break;
528            }
529
530            int64_t durationUs;
531            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
532                sp<NuPlayerDriver> driver = mDriver.promote();
533                if (driver != NULL) {
534                    driver->notifyDuration(durationUs);
535                }
536            }
537
538            msg->post(1000000ll);  // poll again in a second.
539            break;
540        }
541
542        case kWhatSetVideoNativeWindow:
543        {
544            ALOGV("kWhatSetVideoNativeWindow");
545
546            sp<RefBase> obj;
547            CHECK(msg->findObject("native-window", &obj));
548
549            if (mSource == NULL || mSource->getFormat(false /* audio */) == NULL) {
550                performSetSurface(static_cast<NativeWindowWrapper *>(obj.get()));
551                break;
552            }
553
554            mDeferredActions.push_back(
555                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
556                                           FLUSH_CMD_SHUTDOWN /* video */));
557
558            mDeferredActions.push_back(
559                    new SetSurfaceAction(
560                        static_cast<NativeWindowWrapper *>(obj.get())));
561
562            if (obj != NULL) {
563                if (mStarted) {
564                    // Issue a seek to refresh the video screen only if started otherwise
565                    // the extractor may not yet be started and will assert.
566                    // If the video decoder is not set (perhaps audio only in this case)
567                    // do not perform a seek as it is not needed.
568                    int64_t currentPositionUs = 0;
569                    if (getCurrentPosition(&currentPositionUs) == OK) {
570                        mDeferredActions.push_back(
571                                new SeekAction(currentPositionUs, false /* needNotify */));
572                    }
573                }
574
575                // If there is a new surface texture, instantiate decoders
576                // again if possible.
577                mDeferredActions.push_back(
578                        new SimpleAction(&NuPlayer::performScanSources));
579            }
580
581            // After a flush without shutdown, decoder is paused.
582            // Don't resume it until source seek is done, otherwise it could
583            // start pulling stale data too soon.
584            mDeferredActions.push_back(
585                    new ResumeDecoderAction(false /* needNotify */));
586
587            processDeferredActions();
588            break;
589        }
590
591        case kWhatSetAudioSink:
592        {
593            ALOGV("kWhatSetAudioSink");
594
595            sp<RefBase> obj;
596            CHECK(msg->findObject("sink", &obj));
597
598            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
599            break;
600        }
601
602        case kWhatStart:
603        {
604            ALOGV("kWhatStart");
605            if (mStarted) {
606                onResume();
607            } else {
608                onStart();
609            }
610            mPausedByClient = false;
611            break;
612        }
613
614        case kWhatSetRate:
615        {
616            ALOGV("kWhatSetRate");
617            CHECK(msg->findFloat("rate", &mPlaybackRate));
618            if (mRenderer != NULL) {
619                mRenderer->setPlaybackRate(mPlaybackRate);
620            }
621            break;
622        }
623
624        case kWhatScanSources:
625        {
626            int32_t generation;
627            CHECK(msg->findInt32("generation", &generation));
628            if (generation != mScanSourcesGeneration) {
629                // Drop obsolete msg.
630                break;
631            }
632
633            mScanSourcesPending = false;
634
635            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
636                 mAudioDecoder != NULL, mVideoDecoder != NULL);
637
638            bool mHadAnySourcesBefore =
639                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
640
641            // initialize video before audio because successful initialization of
642            // video may change deep buffer mode of audio.
643            if (mNativeWindow != NULL) {
644                instantiateDecoder(false, &mVideoDecoder);
645            }
646
647            // Don't try to re-open audio sink if there's an existing decoder.
648            if (mAudioSink != NULL && mAudioDecoder == NULL) {
649                sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
650                sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
651                audio_stream_type_t streamType = mAudioSink->getAudioStreamType();
652                const bool hasVideo = (videoFormat != NULL);
653                const bool canOffload = canOffloadStream(
654                        audioMeta, hasVideo, true /* is_streaming */, streamType);
655                if (canOffload) {
656                    if (!mOffloadAudio) {
657                        mRenderer->signalEnableOffloadAudio();
658                    }
659                    // open audio sink early under offload mode.
660                    sp<AMessage> format = mSource->getFormat(true /*audio*/);
661                    tryOpenAudioSinkForOffload(format, hasVideo);
662                }
663                instantiateDecoder(true, &mAudioDecoder);
664            }
665
666            if (!mHadAnySourcesBefore
667                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
668                // This is the first time we've found anything playable.
669
670                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
671                    schedulePollDuration();
672                }
673            }
674
675            status_t err;
676            if ((err = mSource->feedMoreTSData()) != OK) {
677                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
678                    // We're not currently decoding anything (no audio or
679                    // video tracks found) and we just ran out of input data.
680
681                    if (err == ERROR_END_OF_STREAM) {
682                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
683                    } else {
684                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
685                    }
686                }
687                break;
688            }
689
690            if ((mAudioDecoder == NULL && mAudioSink != NULL)
691                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
692                msg->post(100000ll);
693                mScanSourcesPending = true;
694            }
695            break;
696        }
697
698        case kWhatVideoNotify:
699        case kWhatAudioNotify:
700        {
701            bool audio = msg->what() == kWhatAudioNotify;
702
703            int32_t currentDecoderGeneration =
704                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
705            int32_t requesterGeneration = currentDecoderGeneration - 1;
706            CHECK(msg->findInt32("generation", &requesterGeneration));
707
708            if (requesterGeneration != currentDecoderGeneration) {
709                ALOGV("got message from old %s decoder, generation(%d:%d)",
710                        audio ? "audio" : "video", requesterGeneration,
711                        currentDecoderGeneration);
712                sp<AMessage> reply;
713                if (!(msg->findMessage("reply", &reply))) {
714                    return;
715                }
716
717                reply->setInt32("err", INFO_DISCONTINUITY);
718                reply->post();
719                return;
720            }
721
722            int32_t what;
723            CHECK(msg->findInt32("what", &what));
724
725            if (what == DecoderBase::kWhatInputDiscontinuity) {
726                int32_t formatChange;
727                CHECK(msg->findInt32("formatChange", &formatChange));
728
729                ALOGV("%s discontinuity: formatChange %d",
730                        audio ? "audio" : "video", formatChange);
731
732                if (formatChange) {
733                    mDeferredActions.push_back(
734                            new FlushDecoderAction(
735                                audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
736                                audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN));
737                }
738
739                mDeferredActions.push_back(
740                        new SimpleAction(
741                                &NuPlayer::performScanSources));
742
743                processDeferredActions();
744            } else if (what == DecoderBase::kWhatEOS) {
745                int32_t err;
746                CHECK(msg->findInt32("err", &err));
747
748                if (err == ERROR_END_OF_STREAM) {
749                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
750                } else {
751                    ALOGV("got %s decoder EOS w/ error %d",
752                         audio ? "audio" : "video",
753                         err);
754                }
755
756                mRenderer->queueEOS(audio, err);
757            } else if (what == DecoderBase::kWhatFlushCompleted) {
758                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
759
760                handleFlushComplete(audio, true /* isDecoder */);
761                finishFlushIfPossible();
762            } else if (what == DecoderBase::kWhatVideoSizeChanged) {
763                sp<AMessage> format;
764                CHECK(msg->findMessage("format", &format));
765
766                sp<AMessage> inputFormat =
767                        mSource->getFormat(false /* audio */);
768
769                updateVideoSize(inputFormat, format);
770            } else if (what == DecoderBase::kWhatShutdownCompleted) {
771                ALOGV("%s shutdown completed", audio ? "audio" : "video");
772                if (audio) {
773                    mAudioDecoder.clear();
774                    ++mAudioDecoderGeneration;
775
776                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
777                    mFlushingAudio = SHUT_DOWN;
778                } else {
779                    mVideoDecoder.clear();
780                    ++mVideoDecoderGeneration;
781
782                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
783                    mFlushingVideo = SHUT_DOWN;
784                }
785
786                finishFlushIfPossible();
787            } else if (what == DecoderBase::kWhatResumeCompleted) {
788                finishResume();
789            } else if (what == DecoderBase::kWhatError) {
790                status_t err;
791                if (!msg->findInt32("err", &err) || err == OK) {
792                    err = UNKNOWN_ERROR;
793                }
794
795                // Decoder errors can be due to Source (e.g. from streaming),
796                // or from decoding corrupted bitstreams, or from other decoder
797                // MediaCodec operations (e.g. from an ongoing reset or seek).
798                // They may also be due to openAudioSink failure at
799                // decoder start or after a format change.
800                //
801                // We try to gracefully shut down the affected decoder if possible,
802                // rather than trying to force the shutdown with something
803                // similar to performReset(). This method can lead to a hang
804                // if MediaCodec functions block after an error, but they should
805                // typically return INVALID_OPERATION instead of blocking.
806
807                FlushStatus *flushing = audio ? &mFlushingAudio : &mFlushingVideo;
808                ALOGE("received error(%#x) from %s decoder, flushing(%d), now shutting down",
809                        err, audio ? "audio" : "video", *flushing);
810
811                switch (*flushing) {
812                    case NONE:
813                        mDeferredActions.push_back(
814                                new FlushDecoderAction(
815                                    audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
816                                    audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN));
817                        processDeferredActions();
818                        break;
819                    case FLUSHING_DECODER:
820                        *flushing = FLUSHING_DECODER_SHUTDOWN; // initiate shutdown after flush.
821                        break; // Wait for flush to complete.
822                    case FLUSHING_DECODER_SHUTDOWN:
823                        break; // Wait for flush to complete.
824                    case SHUTTING_DOWN_DECODER:
825                        break; // Wait for shutdown to complete.
826                    case FLUSHED:
827                        // Widevine source reads must stop before releasing the video decoder.
828                        if (!audio && mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
829                            mSource->stop();
830                        }
831                        getDecoder(audio)->initiateShutdown(); // In the middle of a seek.
832                        *flushing = SHUTTING_DOWN_DECODER;     // Shut down.
833                        break;
834                    case SHUT_DOWN:
835                        finishFlushIfPossible();  // Should not occur.
836                        break;                    // Finish anyways.
837                }
838                notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
839            } else {
840                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
841                      what,
842                      what >> 24,
843                      (what >> 16) & 0xff,
844                      (what >> 8) & 0xff,
845                      what & 0xff);
846            }
847
848            break;
849        }
850
851        case kWhatRendererNotify:
852        {
853            int32_t requesterGeneration = mRendererGeneration - 1;
854            CHECK(msg->findInt32("generation", &requesterGeneration));
855            if (requesterGeneration != mRendererGeneration) {
856                ALOGV("got message from old renderer, generation(%d:%d)",
857                        requesterGeneration, mRendererGeneration);
858                return;
859            }
860
861            int32_t what;
862            CHECK(msg->findInt32("what", &what));
863
864            if (what == Renderer::kWhatEOS) {
865                int32_t audio;
866                CHECK(msg->findInt32("audio", &audio));
867
868                int32_t finalResult;
869                CHECK(msg->findInt32("finalResult", &finalResult));
870
871                if (audio) {
872                    mAudioEOS = true;
873                } else {
874                    mVideoEOS = true;
875                }
876
877                if (finalResult == ERROR_END_OF_STREAM) {
878                    ALOGV("reached %s EOS", audio ? "audio" : "video");
879                } else {
880                    ALOGE("%s track encountered an error (%d)",
881                         audio ? "audio" : "video", finalResult);
882
883                    notifyListener(
884                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
885                }
886
887                if ((mAudioEOS || mAudioDecoder == NULL)
888                        && (mVideoEOS || mVideoDecoder == NULL)) {
889                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
890                }
891            } else if (what == Renderer::kWhatFlushComplete) {
892                int32_t audio;
893                CHECK(msg->findInt32("audio", &audio));
894
895                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
896                handleFlushComplete(audio, false /* isDecoder */);
897                finishFlushIfPossible();
898            } else if (what == Renderer::kWhatVideoRenderingStart) {
899                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
900            } else if (what == Renderer::kWhatMediaRenderingStart) {
901                ALOGV("media rendering started");
902                notifyListener(MEDIA_STARTED, 0, 0);
903            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
904                ALOGV("Tear down audio offload, fall back to s/w path if due to error.");
905                int64_t positionUs;
906                CHECK(msg->findInt64("positionUs", &positionUs));
907                int32_t reason;
908                CHECK(msg->findInt32("reason", &reason));
909                closeAudioSink();
910                mAudioDecoder.clear();
911                ++mAudioDecoderGeneration;
912                mRenderer->flush(
913                        true /* audio */, false /* notifyComplete */);
914                if (mVideoDecoder != NULL) {
915                    mRenderer->flush(
916                            false /* audio */, false /* notifyComplete */);
917                }
918
919                performSeek(positionUs, false /* needNotify */);
920                if (reason == Renderer::kDueToError) {
921                    mRenderer->signalDisableOffloadAudio();
922                    mOffloadAudio = false;
923                    instantiateDecoder(true /* audio */, &mAudioDecoder);
924                }
925            }
926            break;
927        }
928
929        case kWhatMoreDataQueued:
930        {
931            break;
932        }
933
934        case kWhatReset:
935        {
936            ALOGV("kWhatReset");
937
938            mDeferredActions.push_back(
939                    new FlushDecoderAction(
940                        FLUSH_CMD_SHUTDOWN /* audio */,
941                        FLUSH_CMD_SHUTDOWN /* video */));
942
943            mDeferredActions.push_back(
944                    new SimpleAction(&NuPlayer::performReset));
945
946            processDeferredActions();
947            break;
948        }
949
950        case kWhatSeek:
951        {
952            int64_t seekTimeUs;
953            int32_t needNotify;
954            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
955            CHECK(msg->findInt32("needNotify", &needNotify));
956
957            ALOGV("kWhatSeek seekTimeUs=%lld us, needNotify=%d",
958                    seekTimeUs, needNotify);
959
960            mDeferredActions.push_back(
961                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
962                                           FLUSH_CMD_FLUSH /* video */));
963
964            mDeferredActions.push_back(
965                    new SeekAction(seekTimeUs, needNotify));
966
967            // After a flush without shutdown, decoder is paused.
968            // Don't resume it until source seek is done, otherwise it could
969            // start pulling stale data too soon.
970            mDeferredActions.push_back(
971                    new ResumeDecoderAction(needNotify));
972
973            processDeferredActions();
974            break;
975        }
976
977        case kWhatPause:
978        {
979            onPause();
980            mPausedByClient = true;
981            break;
982        }
983
984        case kWhatSourceNotify:
985        {
986            onSourceNotify(msg);
987            break;
988        }
989
990        case kWhatClosedCaptionNotify:
991        {
992            onClosedCaptionNotify(msg);
993            break;
994        }
995
996        default:
997            TRESPASS();
998            break;
999    }
1000}
1001
1002void NuPlayer::onResume() {
1003    if (!mPaused) {
1004        return;
1005    }
1006    mPaused = false;
1007    if (mSource != NULL) {
1008        mSource->resume();
1009    } else {
1010        ALOGW("resume called when source is gone or not set");
1011    }
1012    // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if
1013    // needed.
1014    if (audioDecoderStillNeeded() && mAudioDecoder == NULL) {
1015        instantiateDecoder(true /* audio */, &mAudioDecoder);
1016    }
1017    if (mRenderer != NULL) {
1018        mRenderer->resume();
1019    } else {
1020        ALOGW("resume called when renderer is gone or not set");
1021    }
1022}
1023
1024status_t NuPlayer::onInstantiateSecureDecoders() {
1025    status_t err;
1026    if (!(mSourceFlags & Source::FLAG_SECURE)) {
1027        return BAD_TYPE;
1028    }
1029
1030    if (mRenderer != NULL) {
1031        ALOGE("renderer should not be set when instantiating secure decoders");
1032        return UNKNOWN_ERROR;
1033    }
1034
1035    // TRICKY: We rely on mRenderer being null, so that decoder does not start requesting
1036    // data on instantiation.
1037    if (mNativeWindow != NULL) {
1038        err = instantiateDecoder(false, &mVideoDecoder);
1039        if (err != OK) {
1040            return err;
1041        }
1042    }
1043
1044    if (mAudioSink != NULL) {
1045        err = instantiateDecoder(true, &mAudioDecoder);
1046        if (err != OK) {
1047            return err;
1048        }
1049    }
1050    return OK;
1051}
1052
1053void NuPlayer::onStart() {
1054    mOffloadAudio = false;
1055    mAudioEOS = false;
1056    mVideoEOS = false;
1057    mStarted = true;
1058
1059    mSource->start();
1060
1061    uint32_t flags = 0;
1062
1063    if (mSource->isRealTime()) {
1064        flags |= Renderer::FLAG_REAL_TIME;
1065    }
1066
1067    sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
1068    audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
1069    if (mAudioSink != NULL) {
1070        streamType = mAudioSink->getAudioStreamType();
1071    }
1072
1073    sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
1074
1075    mOffloadAudio =
1076        canOffloadStream(audioMeta, (videoFormat != NULL),
1077                         true /* is_streaming */, streamType);
1078    if (mOffloadAudio) {
1079        flags |= Renderer::FLAG_OFFLOAD_AUDIO;
1080    }
1081
1082    sp<AMessage> notify = new AMessage(kWhatRendererNotify, this);
1083    ++mRendererGeneration;
1084    notify->setInt32("generation", mRendererGeneration);
1085    mRenderer = new Renderer(mAudioSink, notify, flags);
1086    mRendererLooper = new ALooper;
1087    mRendererLooper->setName("NuPlayerRenderer");
1088    mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
1089    mRendererLooper->registerHandler(mRenderer);
1090    if (mPlaybackRate != 1.0) {
1091        mRenderer->setPlaybackRate(mPlaybackRate);
1092    }
1093
1094    sp<MetaData> meta = getFileMeta();
1095    int32_t rate;
1096    if (meta != NULL
1097            && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) {
1098        mRenderer->setVideoFrameRate(rate);
1099    }
1100
1101    if (mVideoDecoder != NULL) {
1102        mVideoDecoder->setRenderer(mRenderer);
1103    }
1104    if (mAudioDecoder != NULL) {
1105        mAudioDecoder->setRenderer(mRenderer);
1106    }
1107
1108    postScanSources();
1109}
1110
1111void NuPlayer::onPause() {
1112    if (mPaused) {
1113        return;
1114    }
1115    mPaused = true;
1116    if (mSource != NULL) {
1117        mSource->pause();
1118    } else {
1119        ALOGW("pause called when source is gone or not set");
1120    }
1121    if (mRenderer != NULL) {
1122        mRenderer->pause();
1123    } else {
1124        ALOGW("pause called when renderer is gone or not set");
1125    }
1126}
1127
1128bool NuPlayer::audioDecoderStillNeeded() {
1129    // Audio decoder is no longer needed if it's in shut/shutting down status.
1130    return ((mFlushingAudio != SHUT_DOWN) && (mFlushingAudio != SHUTTING_DOWN_DECODER));
1131}
1132
1133void NuPlayer::handleFlushComplete(bool audio, bool isDecoder) {
1134    // We wait for both the decoder flush and the renderer flush to complete
1135    // before entering either the FLUSHED or the SHUTTING_DOWN_DECODER state.
1136
1137    mFlushComplete[audio][isDecoder] = true;
1138    if (!mFlushComplete[audio][!isDecoder]) {
1139        return;
1140    }
1141
1142    FlushStatus *state = audio ? &mFlushingAudio : &mFlushingVideo;
1143    switch (*state) {
1144        case FLUSHING_DECODER:
1145        {
1146            *state = FLUSHED;
1147            break;
1148        }
1149
1150        case FLUSHING_DECODER_SHUTDOWN:
1151        {
1152            *state = SHUTTING_DOWN_DECODER;
1153
1154            ALOGV("initiating %s decoder shutdown", audio ? "audio" : "video");
1155            if (!audio) {
1156                // Widevine source reads must stop before releasing the video decoder.
1157                if (mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
1158                    mSource->stop();
1159                }
1160            }
1161            getDecoder(audio)->initiateShutdown();
1162            break;
1163        }
1164
1165        default:
1166            // decoder flush completes only occur in a flushing state.
1167            LOG_ALWAYS_FATAL_IF(isDecoder, "decoder flush in invalid state %d", *state);
1168            break;
1169    }
1170}
1171
1172void NuPlayer::finishFlushIfPossible() {
1173    if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED
1174            && mFlushingAudio != SHUT_DOWN) {
1175        return;
1176    }
1177
1178    if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED
1179            && mFlushingVideo != SHUT_DOWN) {
1180        return;
1181    }
1182
1183    ALOGV("both audio and video are flushed now.");
1184
1185    mFlushingAudio = NONE;
1186    mFlushingVideo = NONE;
1187
1188    clearFlushComplete();
1189
1190    processDeferredActions();
1191}
1192
1193void NuPlayer::postScanSources() {
1194    if (mScanSourcesPending) {
1195        return;
1196    }
1197
1198    sp<AMessage> msg = new AMessage(kWhatScanSources, this);
1199    msg->setInt32("generation", mScanSourcesGeneration);
1200    msg->post();
1201
1202    mScanSourcesPending = true;
1203}
1204
1205void NuPlayer::tryOpenAudioSinkForOffload(const sp<AMessage> &format, bool hasVideo) {
1206    // Note: This is called early in NuPlayer to determine whether offloading
1207    // is possible; otherwise the decoders call the renderer openAudioSink directly.
1208
1209    status_t err = mRenderer->openAudioSink(
1210            format, true /* offloadOnly */, hasVideo, AUDIO_OUTPUT_FLAG_NONE, &mOffloadAudio);
1211    if (err != OK) {
1212        // Any failure we turn off mOffloadAudio.
1213        mOffloadAudio = false;
1214    } else if (mOffloadAudio) {
1215        sp<MetaData> audioMeta =
1216                mSource->getFormatMeta(true /* audio */);
1217        sendMetaDataToHal(mAudioSink, audioMeta);
1218    }
1219}
1220
1221void NuPlayer::closeAudioSink() {
1222    mRenderer->closeAudioSink();
1223}
1224
1225status_t NuPlayer::instantiateDecoder(bool audio, sp<DecoderBase> *decoder) {
1226    if (*decoder != NULL) {
1227        return OK;
1228    }
1229
1230    sp<AMessage> format = mSource->getFormat(audio);
1231
1232    if (format == NULL) {
1233        return -EWOULDBLOCK;
1234    }
1235
1236    if (!audio) {
1237        AString mime;
1238        CHECK(format->findString("mime", &mime));
1239
1240        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, this);
1241        if (mCCDecoder == NULL) {
1242            mCCDecoder = new CCDecoder(ccNotify);
1243        }
1244
1245        if (mSourceFlags & Source::FLAG_SECURE) {
1246            format->setInt32("secure", true);
1247        }
1248
1249        if (mSourceFlags & Source::FLAG_PROTECTED) {
1250            format->setInt32("protected", true);
1251        }
1252    }
1253
1254    if (audio) {
1255        sp<AMessage> notify = new AMessage(kWhatAudioNotify, this);
1256        ++mAudioDecoderGeneration;
1257        notify->setInt32("generation", mAudioDecoderGeneration);
1258
1259        if (mOffloadAudio) {
1260            *decoder = new DecoderPassThrough(notify, mSource, mRenderer);
1261        } else {
1262            *decoder = new Decoder(notify, mSource, mRenderer);
1263        }
1264    } else {
1265        sp<AMessage> notify = new AMessage(kWhatVideoNotify, this);
1266        ++mVideoDecoderGeneration;
1267        notify->setInt32("generation", mVideoDecoderGeneration);
1268
1269        *decoder = new Decoder(
1270                notify, mSource, mRenderer, mNativeWindow, mCCDecoder);
1271
1272        // enable FRC if high-quality AV sync is requested, even if not
1273        // queuing to native window, as this will even improve textureview
1274        // playback.
1275        {
1276            char value[PROPERTY_VALUE_MAX];
1277            if (property_get("persist.sys.media.avsync", value, NULL) &&
1278                    (!strcmp("1", value) || !strcasecmp("true", value))) {
1279                format->setInt32("auto-frc", 1);
1280            }
1281        }
1282    }
1283    (*decoder)->init();
1284    (*decoder)->configure(format);
1285
1286    // allocate buffers to decrypt widevine source buffers
1287    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1288        Vector<sp<ABuffer> > inputBufs;
1289        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1290
1291        Vector<MediaBuffer *> mediaBufs;
1292        for (size_t i = 0; i < inputBufs.size(); i++) {
1293            const sp<ABuffer> &buffer = inputBufs[i];
1294            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1295            mediaBufs.push(mbuf);
1296        }
1297
1298        status_t err = mSource->setBuffers(audio, mediaBufs);
1299        if (err != OK) {
1300            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1301                mediaBufs[i]->release();
1302            }
1303            mediaBufs.clear();
1304            ALOGE("Secure source didn't support secure mediaBufs.");
1305            return err;
1306        }
1307    }
1308    return OK;
1309}
1310
1311void NuPlayer::updateVideoSize(
1312        const sp<AMessage> &inputFormat,
1313        const sp<AMessage> &outputFormat) {
1314    if (inputFormat == NULL) {
1315        ALOGW("Unknown video size, reporting 0x0!");
1316        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1317        return;
1318    }
1319
1320    int32_t displayWidth, displayHeight;
1321    int32_t cropLeft, cropTop, cropRight, cropBottom;
1322
1323    if (outputFormat != NULL) {
1324        int32_t width, height;
1325        CHECK(outputFormat->findInt32("width", &width));
1326        CHECK(outputFormat->findInt32("height", &height));
1327
1328        int32_t cropLeft, cropTop, cropRight, cropBottom;
1329        CHECK(outputFormat->findRect(
1330                    "crop",
1331                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1332
1333        displayWidth = cropRight - cropLeft + 1;
1334        displayHeight = cropBottom - cropTop + 1;
1335
1336        ALOGV("Video output format changed to %d x %d "
1337             "(crop: %d x %d @ (%d, %d))",
1338             width, height,
1339             displayWidth,
1340             displayHeight,
1341             cropLeft, cropTop);
1342    } else {
1343        CHECK(inputFormat->findInt32("width", &displayWidth));
1344        CHECK(inputFormat->findInt32("height", &displayHeight));
1345
1346        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1347    }
1348
1349    // Take into account sample aspect ratio if necessary:
1350    int32_t sarWidth, sarHeight;
1351    if (inputFormat->findInt32("sar-width", &sarWidth)
1352            && inputFormat->findInt32("sar-height", &sarHeight)) {
1353        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1354
1355        displayWidth = (displayWidth * sarWidth) / sarHeight;
1356
1357        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1358    }
1359
1360    int32_t rotationDegrees;
1361    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1362        rotationDegrees = 0;
1363    }
1364
1365    if (rotationDegrees == 90 || rotationDegrees == 270) {
1366        int32_t tmp = displayWidth;
1367        displayWidth = displayHeight;
1368        displayHeight = tmp;
1369    }
1370
1371    notifyListener(
1372            MEDIA_SET_VIDEO_SIZE,
1373            displayWidth,
1374            displayHeight);
1375}
1376
1377void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1378    if (mDriver == NULL) {
1379        return;
1380    }
1381
1382    sp<NuPlayerDriver> driver = mDriver.promote();
1383
1384    if (driver == NULL) {
1385        return;
1386    }
1387
1388    driver->notifyListener(msg, ext1, ext2, in);
1389}
1390
1391void NuPlayer::flushDecoder(bool audio, bool needShutdown) {
1392    ALOGV("[%s] flushDecoder needShutdown=%d",
1393          audio ? "audio" : "video", needShutdown);
1394
1395    const sp<DecoderBase> &decoder = getDecoder(audio);
1396    if (decoder == NULL) {
1397        ALOGI("flushDecoder %s without decoder present",
1398             audio ? "audio" : "video");
1399        return;
1400    }
1401
1402    // Make sure we don't continue to scan sources until we finish flushing.
1403    ++mScanSourcesGeneration;
1404    mScanSourcesPending = false;
1405
1406    decoder->signalFlush();
1407
1408    FlushStatus newStatus =
1409        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1410
1411    mFlushComplete[audio][false /* isDecoder */] = (mRenderer == NULL);
1412    mFlushComplete[audio][true /* isDecoder */] = false;
1413    if (audio) {
1414        ALOGE_IF(mFlushingAudio != NONE,
1415                "audio flushDecoder() is called in state %d", mFlushingAudio);
1416        mFlushingAudio = newStatus;
1417    } else {
1418        ALOGE_IF(mFlushingVideo != NONE,
1419                "video flushDecoder() is called in state %d", mFlushingVideo);
1420        mFlushingVideo = newStatus;
1421    }
1422}
1423
1424void NuPlayer::queueDecoderShutdown(
1425        bool audio, bool video, const sp<AMessage> &reply) {
1426    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1427
1428    mDeferredActions.push_back(
1429            new FlushDecoderAction(
1430                audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
1431                video ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE));
1432
1433    mDeferredActions.push_back(
1434            new SimpleAction(&NuPlayer::performScanSources));
1435
1436    mDeferredActions.push_back(new PostMessageAction(reply));
1437
1438    processDeferredActions();
1439}
1440
1441status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1442    mVideoScalingMode = mode;
1443    if (mNativeWindow != NULL) {
1444        status_t ret = native_window_set_scaling_mode(
1445                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1446        if (ret != OK) {
1447            ALOGE("Failed to set scaling mode (%d): %s",
1448                -ret, strerror(-ret));
1449            return ret;
1450        }
1451    }
1452    return OK;
1453}
1454
1455status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1456    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, this);
1457    msg->setPointer("reply", reply);
1458
1459    sp<AMessage> response;
1460    status_t err = msg->postAndAwaitResponse(&response);
1461    return err;
1462}
1463
1464status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1465    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, this);
1466    msg->setPointer("reply", reply);
1467    msg->setInt32("type", type);
1468
1469    sp<AMessage> response;
1470    status_t err = msg->postAndAwaitResponse(&response);
1471    if (err == OK && response != NULL) {
1472        CHECK(response->findInt32("err", &err));
1473    }
1474    return err;
1475}
1476
1477status_t NuPlayer::selectTrack(size_t trackIndex, bool select, int64_t timeUs) {
1478    sp<AMessage> msg = new AMessage(kWhatSelectTrack, this);
1479    msg->setSize("trackIndex", trackIndex);
1480    msg->setInt32("select", select);
1481    msg->setInt64("timeUs", timeUs);
1482
1483    sp<AMessage> response;
1484    status_t err = msg->postAndAwaitResponse(&response);
1485
1486    if (err != OK) {
1487        return err;
1488    }
1489
1490    if (!response->findInt32("err", &err)) {
1491        err = OK;
1492    }
1493
1494    return err;
1495}
1496
1497status_t NuPlayer::getCurrentPosition(int64_t *mediaUs) {
1498    sp<Renderer> renderer = mRenderer;
1499    if (renderer == NULL) {
1500        return NO_INIT;
1501    }
1502
1503    return renderer->getCurrentPosition(mediaUs);
1504}
1505
1506void NuPlayer::getStats(int64_t *numFramesTotal, int64_t *numFramesDropped) {
1507    sp<DecoderBase> decoder = getDecoder(false /* audio */);
1508    if (decoder != NULL) {
1509        decoder->getStats(numFramesTotal, numFramesDropped);
1510    } else {
1511        *numFramesTotal = 0;
1512        *numFramesDropped = 0;
1513    }
1514}
1515
1516sp<MetaData> NuPlayer::getFileMeta() {
1517    return mSource->getFileFormatMeta();
1518}
1519
1520void NuPlayer::schedulePollDuration() {
1521    sp<AMessage> msg = new AMessage(kWhatPollDuration, this);
1522    msg->setInt32("generation", mPollDurationGeneration);
1523    msg->post();
1524}
1525
1526void NuPlayer::cancelPollDuration() {
1527    ++mPollDurationGeneration;
1528}
1529
1530void NuPlayer::processDeferredActions() {
1531    while (!mDeferredActions.empty()) {
1532        // We won't execute any deferred actions until we're no longer in
1533        // an intermediate state, i.e. one more more decoders are currently
1534        // flushing or shutting down.
1535
1536        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1537            // We're currently flushing, postpone the reset until that's
1538            // completed.
1539
1540            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1541                  mFlushingAudio, mFlushingVideo);
1542
1543            break;
1544        }
1545
1546        sp<Action> action = *mDeferredActions.begin();
1547        mDeferredActions.erase(mDeferredActions.begin());
1548
1549        action->execute(this);
1550    }
1551}
1552
1553void NuPlayer::performSeek(int64_t seekTimeUs, bool needNotify) {
1554    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs), needNotify(%d)",
1555          seekTimeUs,
1556          seekTimeUs / 1E6,
1557          needNotify);
1558
1559    if (mSource == NULL) {
1560        // This happens when reset occurs right before the loop mode
1561        // asynchronously seeks to the start of the stream.
1562        LOG_ALWAYS_FATAL_IF(mAudioDecoder != NULL || mVideoDecoder != NULL,
1563                "mSource is NULL and decoders not NULL audio(%p) video(%p)",
1564                mAudioDecoder.get(), mVideoDecoder.get());
1565        return;
1566    }
1567    mSource->seekTo(seekTimeUs);
1568    ++mTimedTextGeneration;
1569
1570    // everything's flushed, continue playback.
1571}
1572
1573void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) {
1574    ALOGV("performDecoderFlush audio=%d, video=%d", audio, video);
1575
1576    if ((audio == FLUSH_CMD_NONE || mAudioDecoder == NULL)
1577            && (video == FLUSH_CMD_NONE || mVideoDecoder == NULL)) {
1578        return;
1579    }
1580
1581    if (audio != FLUSH_CMD_NONE && mAudioDecoder != NULL) {
1582        flushDecoder(true /* audio */, (audio == FLUSH_CMD_SHUTDOWN));
1583    }
1584
1585    if (video != FLUSH_CMD_NONE && mVideoDecoder != NULL) {
1586        flushDecoder(false /* audio */, (video == FLUSH_CMD_SHUTDOWN));
1587    }
1588}
1589
1590void NuPlayer::performReset() {
1591    ALOGV("performReset");
1592
1593    CHECK(mAudioDecoder == NULL);
1594    CHECK(mVideoDecoder == NULL);
1595
1596    cancelPollDuration();
1597
1598    ++mScanSourcesGeneration;
1599    mScanSourcesPending = false;
1600
1601    if (mRendererLooper != NULL) {
1602        if (mRenderer != NULL) {
1603            mRendererLooper->unregisterHandler(mRenderer->id());
1604        }
1605        mRendererLooper->stop();
1606        mRendererLooper.clear();
1607    }
1608    mRenderer.clear();
1609    ++mRendererGeneration;
1610
1611    if (mSource != NULL) {
1612        mSource->stop();
1613
1614        mSource.clear();
1615    }
1616
1617    if (mDriver != NULL) {
1618        sp<NuPlayerDriver> driver = mDriver.promote();
1619        if (driver != NULL) {
1620            driver->notifyResetComplete();
1621        }
1622    }
1623
1624    mStarted = false;
1625}
1626
1627void NuPlayer::performScanSources() {
1628    ALOGV("performScanSources");
1629
1630    if (!mStarted) {
1631        return;
1632    }
1633
1634    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1635        postScanSources();
1636    }
1637}
1638
1639void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1640    ALOGV("performSetSurface");
1641
1642    mNativeWindow = wrapper;
1643
1644    // XXX - ignore error from setVideoScalingMode for now
1645    setVideoScalingMode(mVideoScalingMode);
1646
1647    if (mDriver != NULL) {
1648        sp<NuPlayerDriver> driver = mDriver.promote();
1649        if (driver != NULL) {
1650            driver->notifySetSurfaceComplete();
1651        }
1652    }
1653}
1654
1655void NuPlayer::performResumeDecoders(bool needNotify) {
1656    if (needNotify) {
1657        mResumePending = true;
1658        if (mVideoDecoder == NULL) {
1659            // if audio-only, we can notify seek complete now,
1660            // as the resume operation will be relatively fast.
1661            finishResume();
1662        }
1663    }
1664
1665    if (mVideoDecoder != NULL) {
1666        // When there is continuous seek, MediaPlayer will cache the seek
1667        // position, and send down new seek request when previous seek is
1668        // complete. Let's wait for at least one video output frame before
1669        // notifying seek complete, so that the video thumbnail gets updated
1670        // when seekbar is dragged.
1671        mVideoDecoder->signalResume(needNotify);
1672    }
1673
1674    if (mAudioDecoder != NULL) {
1675        mAudioDecoder->signalResume(false /* needNotify */);
1676    }
1677}
1678
1679void NuPlayer::finishResume() {
1680    if (mResumePending) {
1681        mResumePending = false;
1682        if (mDriver != NULL) {
1683            sp<NuPlayerDriver> driver = mDriver.promote();
1684            if (driver != NULL) {
1685                driver->notifySeekComplete();
1686            }
1687        }
1688    }
1689}
1690
1691void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1692    int32_t what;
1693    CHECK(msg->findInt32("what", &what));
1694
1695    switch (what) {
1696        case Source::kWhatInstantiateSecureDecoders:
1697        {
1698            if (mSource == NULL) {
1699                // This is a stale notification from a source that was
1700                // asynchronously preparing when the client called reset().
1701                // We handled the reset, the source is gone.
1702                break;
1703            }
1704
1705            sp<AMessage> reply;
1706            CHECK(msg->findMessage("reply", &reply));
1707            status_t err = onInstantiateSecureDecoders();
1708            reply->setInt32("err", err);
1709            reply->post();
1710            break;
1711        }
1712
1713        case Source::kWhatPrepared:
1714        {
1715            if (mSource == NULL) {
1716                // This is a stale notification from a source that was
1717                // asynchronously preparing when the client called reset().
1718                // We handled the reset, the source is gone.
1719                break;
1720            }
1721
1722            int32_t err;
1723            CHECK(msg->findInt32("err", &err));
1724
1725            if (err != OK) {
1726                // shut down potential secure codecs in case client never calls reset
1727                mDeferredActions.push_back(
1728                        new FlushDecoderAction(FLUSH_CMD_SHUTDOWN /* audio */,
1729                                               FLUSH_CMD_SHUTDOWN /* video */));
1730                processDeferredActions();
1731            }
1732
1733            sp<NuPlayerDriver> driver = mDriver.promote();
1734            if (driver != NULL) {
1735                // notify duration first, so that it's definitely set when
1736                // the app received the "prepare complete" callback.
1737                int64_t durationUs;
1738                if (mSource->getDuration(&durationUs) == OK) {
1739                    driver->notifyDuration(durationUs);
1740                }
1741                driver->notifyPrepareCompleted(err);
1742            }
1743
1744            break;
1745        }
1746
1747        case Source::kWhatFlagsChanged:
1748        {
1749            uint32_t flags;
1750            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1751
1752            sp<NuPlayerDriver> driver = mDriver.promote();
1753            if (driver != NULL) {
1754                if ((flags & NuPlayer::Source::FLAG_CAN_SEEK) == 0) {
1755                    driver->notifyListener(
1756                            MEDIA_INFO, MEDIA_INFO_NOT_SEEKABLE, 0);
1757                }
1758                driver->notifyFlagsChanged(flags);
1759            }
1760
1761            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1762                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
1763                cancelPollDuration();
1764            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1765                    && (flags & Source::FLAG_DYNAMIC_DURATION)
1766                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
1767                schedulePollDuration();
1768            }
1769
1770            mSourceFlags = flags;
1771            break;
1772        }
1773
1774        case Source::kWhatVideoSizeChanged:
1775        {
1776            sp<AMessage> format;
1777            CHECK(msg->findMessage("format", &format));
1778
1779            updateVideoSize(format);
1780            break;
1781        }
1782
1783        case Source::kWhatBufferingUpdate:
1784        {
1785            int32_t percentage;
1786            CHECK(msg->findInt32("percentage", &percentage));
1787
1788            notifyListener(MEDIA_BUFFERING_UPDATE, percentage, 0);
1789            break;
1790        }
1791
1792        case Source::kWhatPauseOnBufferingStart:
1793        {
1794            // ignore if not playing
1795            if (mStarted && !mPausedByClient) {
1796                ALOGI("buffer low, pausing...");
1797
1798                onPause();
1799            }
1800            // fall-thru
1801        }
1802
1803        case Source::kWhatBufferingStart:
1804        {
1805            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
1806            break;
1807        }
1808
1809        case Source::kWhatResumeOnBufferingEnd:
1810        {
1811            // ignore if not playing
1812            if (mStarted && !mPausedByClient) {
1813                ALOGI("buffer ready, resuming...");
1814
1815                onResume();
1816            }
1817            // fall-thru
1818        }
1819
1820        case Source::kWhatBufferingEnd:
1821        {
1822            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
1823            break;
1824        }
1825
1826        case Source::kWhatCacheStats:
1827        {
1828            int32_t kbps;
1829            CHECK(msg->findInt32("bandwidth", &kbps));
1830
1831            notifyListener(MEDIA_INFO, MEDIA_INFO_NETWORK_BANDWIDTH, kbps);
1832            break;
1833        }
1834
1835        case Source::kWhatSubtitleData:
1836        {
1837            sp<ABuffer> buffer;
1838            CHECK(msg->findBuffer("buffer", &buffer));
1839
1840            sendSubtitleData(buffer, 0 /* baseIndex */);
1841            break;
1842        }
1843
1844        case Source::kWhatTimedTextData:
1845        {
1846            int32_t generation;
1847            if (msg->findInt32("generation", &generation)
1848                    && generation != mTimedTextGeneration) {
1849                break;
1850            }
1851
1852            sp<ABuffer> buffer;
1853            CHECK(msg->findBuffer("buffer", &buffer));
1854
1855            sp<NuPlayerDriver> driver = mDriver.promote();
1856            if (driver == NULL) {
1857                break;
1858            }
1859
1860            int posMs;
1861            int64_t timeUs, posUs;
1862            driver->getCurrentPosition(&posMs);
1863            posUs = posMs * 1000;
1864            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1865
1866            if (posUs < timeUs) {
1867                if (!msg->findInt32("generation", &generation)) {
1868                    msg->setInt32("generation", mTimedTextGeneration);
1869                }
1870                msg->post(timeUs - posUs);
1871            } else {
1872                sendTimedTextData(buffer);
1873            }
1874            break;
1875        }
1876
1877        case Source::kWhatQueueDecoderShutdown:
1878        {
1879            int32_t audio, video;
1880            CHECK(msg->findInt32("audio", &audio));
1881            CHECK(msg->findInt32("video", &video));
1882
1883            sp<AMessage> reply;
1884            CHECK(msg->findMessage("reply", &reply));
1885
1886            queueDecoderShutdown(audio, video, reply);
1887            break;
1888        }
1889
1890        case Source::kWhatDrmNoLicense:
1891        {
1892            notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, ERROR_DRM_NO_LICENSE);
1893            break;
1894        }
1895
1896        default:
1897            TRESPASS();
1898    }
1899}
1900
1901void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
1902    int32_t what;
1903    CHECK(msg->findInt32("what", &what));
1904
1905    switch (what) {
1906        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
1907        {
1908            sp<ABuffer> buffer;
1909            CHECK(msg->findBuffer("buffer", &buffer));
1910
1911            size_t inbandTracks = 0;
1912            if (mSource != NULL) {
1913                inbandTracks = mSource->getTrackCount();
1914            }
1915
1916            sendSubtitleData(buffer, inbandTracks);
1917            break;
1918        }
1919
1920        case NuPlayer::CCDecoder::kWhatTrackAdded:
1921        {
1922            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
1923
1924            break;
1925        }
1926
1927        default:
1928            TRESPASS();
1929    }
1930
1931
1932}
1933
1934void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
1935    int32_t trackIndex;
1936    int64_t timeUs, durationUs;
1937    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
1938    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1939    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
1940
1941    Parcel in;
1942    in.writeInt32(trackIndex + baseIndex);
1943    in.writeInt64(timeUs);
1944    in.writeInt64(durationUs);
1945    in.writeInt32(buffer->size());
1946    in.writeInt32(buffer->size());
1947    in.write(buffer->data(), buffer->size());
1948
1949    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
1950}
1951
1952void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
1953    const void *data;
1954    size_t size = 0;
1955    int64_t timeUs;
1956    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
1957
1958    AString mime;
1959    CHECK(buffer->meta()->findString("mime", &mime));
1960    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
1961
1962    data = buffer->data();
1963    size = buffer->size();
1964
1965    Parcel parcel;
1966    if (size > 0) {
1967        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1968        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
1969        TextDescriptions::getParcelOfDescriptions(
1970                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
1971    }
1972
1973    if ((parcel.dataSize() > 0)) {
1974        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
1975    } else {  // send an empty timed text
1976        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
1977    }
1978}
1979////////////////////////////////////////////////////////////////////////////////
1980
1981sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
1982    sp<MetaData> meta = getFormatMeta(audio);
1983
1984    if (meta == NULL) {
1985        return NULL;
1986    }
1987
1988    sp<AMessage> msg = new AMessage;
1989
1990    if(convertMetaDataToMessage(meta, &msg) == OK) {
1991        return msg;
1992    }
1993    return NULL;
1994}
1995
1996void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
1997    sp<AMessage> notify = dupNotify();
1998    notify->setInt32("what", kWhatFlagsChanged);
1999    notify->setInt32("flags", flags);
2000    notify->post();
2001}
2002
2003void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
2004    sp<AMessage> notify = dupNotify();
2005    notify->setInt32("what", kWhatVideoSizeChanged);
2006    notify->setMessage("format", format);
2007    notify->post();
2008}
2009
2010void NuPlayer::Source::notifyPrepared(status_t err) {
2011    sp<AMessage> notify = dupNotify();
2012    notify->setInt32("what", kWhatPrepared);
2013    notify->setInt32("err", err);
2014    notify->post();
2015}
2016
2017void NuPlayer::Source::notifyInstantiateSecureDecoders(const sp<AMessage> &reply) {
2018    sp<AMessage> notify = dupNotify();
2019    notify->setInt32("what", kWhatInstantiateSecureDecoders);
2020    notify->setMessage("reply", reply);
2021    notify->post();
2022}
2023
2024void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
2025    TRESPASS();
2026}
2027
2028}  // namespace android
2029