NuPlayer.cpp revision 426c719a5f3b4d88480eb35a7b0b373f672ea3cb 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerDecoder.h"
25#include "NuPlayerDecoderPassThrough.h"
26#include "NuPlayerDriver.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29#include "RTSPSource.h"
30#include "StreamingSource.h"
31#include "GenericSource.h"
32#include "TextDescriptions.h"
33
34#include "ATSParser.h"
35
36#include <media/stagefright/foundation/hexdump.h>
37#include <media/stagefright/foundation/ABuffer.h>
38#include <media/stagefright/foundation/ADebug.h>
39#include <media/stagefright/foundation/AMessage.h>
40#include <media/stagefright/MediaBuffer.h>
41#include <media/stagefright/MediaDefs.h>
42#include <media/stagefright/MediaErrors.h>
43#include <media/stagefright/MetaData.h>
44#include <gui/IGraphicBufferProducer.h>
45
46#include "avc_utils.h"
47
48#include "ESDS.h"
49#include <media/stagefright/Utils.h>
50
51namespace android {
52
53struct NuPlayer::Action : public RefBase {
54    Action() {}
55
56    virtual void execute(NuPlayer *player) = 0;
57
58private:
59    DISALLOW_EVIL_CONSTRUCTORS(Action);
60};
61
62struct NuPlayer::SeekAction : public Action {
63    SeekAction(int64_t seekTimeUs)
64        : mSeekTimeUs(seekTimeUs) {
65    }
66
67    virtual void execute(NuPlayer *player) {
68        player->performSeek(mSeekTimeUs);
69    }
70
71private:
72    int64_t mSeekTimeUs;
73
74    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
75};
76
77struct NuPlayer::SetSurfaceAction : public Action {
78    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
79        : mWrapper(wrapper) {
80    }
81
82    virtual void execute(NuPlayer *player) {
83        player->performSetSurface(mWrapper);
84    }
85
86private:
87    sp<NativeWindowWrapper> mWrapper;
88
89    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
90};
91
92struct NuPlayer::ShutdownDecoderAction : public Action {
93    ShutdownDecoderAction(bool audio, bool video)
94        : mAudio(audio),
95          mVideo(video) {
96    }
97
98    virtual void execute(NuPlayer *player) {
99        player->performDecoderShutdown(mAudio, mVideo);
100    }
101
102private:
103    bool mAudio;
104    bool mVideo;
105
106    DISALLOW_EVIL_CONSTRUCTORS(ShutdownDecoderAction);
107};
108
109struct NuPlayer::PostMessageAction : public Action {
110    PostMessageAction(const sp<AMessage> &msg)
111        : mMessage(msg) {
112    }
113
114    virtual void execute(NuPlayer *) {
115        mMessage->post();
116    }
117
118private:
119    sp<AMessage> mMessage;
120
121    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
122};
123
124// Use this if there's no state necessary to save in order to execute
125// the action.
126struct NuPlayer::SimpleAction : public Action {
127    typedef void (NuPlayer::*ActionFunc)();
128
129    SimpleAction(ActionFunc func)
130        : mFunc(func) {
131    }
132
133    virtual void execute(NuPlayer *player) {
134        (player->*mFunc)();
135    }
136
137private:
138    ActionFunc mFunc;
139
140    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
141};
142
143////////////////////////////////////////////////////////////////////////////////
144
145NuPlayer::NuPlayer()
146    : mUIDValid(false),
147      mSourceFlags(0),
148      mVideoIsAVC(false),
149      mOffloadAudio(false),
150      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
151      mAudioEOS(false),
152      mVideoEOS(false),
153      mScanSourcesPending(false),
154      mScanSourcesGeneration(0),
155      mPollDurationGeneration(0),
156      mTimedTextGeneration(0),
157      mTimeDiscontinuityPending(false),
158      mFlushingAudio(NONE),
159      mFlushingVideo(NONE),
160      mSkipRenderingAudioUntilMediaTimeUs(-1ll),
161      mSkipRenderingVideoUntilMediaTimeUs(-1ll),
162      mVideoLateByUs(0ll),
163      mNumFramesTotal(0ll),
164      mNumFramesDropped(0ll),
165      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
166      mStarted(false) {
167}
168
169NuPlayer::~NuPlayer() {
170}
171
172void NuPlayer::setUID(uid_t uid) {
173    mUIDValid = true;
174    mUID = uid;
175}
176
177void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
178    mDriver = driver;
179}
180
181void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
182    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
183
184    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
185
186    msg->setObject("source", new StreamingSource(notify, source));
187    msg->post();
188}
189
190static bool IsHTTPLiveURL(const char *url) {
191    if (!strncasecmp("http://", url, 7)
192            || !strncasecmp("https://", url, 8)
193            || !strncasecmp("file://", url, 7)) {
194        size_t len = strlen(url);
195        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
196            return true;
197        }
198
199        if (strstr(url,"m3u8")) {
200            return true;
201        }
202    }
203
204    return false;
205}
206
207void NuPlayer::setDataSourceAsync(
208        const sp<IMediaHTTPService> &httpService,
209        const char *url,
210        const KeyedVector<String8, String8> *headers) {
211
212    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
213    size_t len = strlen(url);
214
215    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
216
217    sp<Source> source;
218    if (IsHTTPLiveURL(url)) {
219        source = new HTTPLiveSource(notify, httpService, url, headers);
220    } else if (!strncasecmp(url, "rtsp://", 7)) {
221        source = new RTSPSource(
222                notify, httpService, url, headers, mUIDValid, mUID);
223    } else if ((!strncasecmp(url, "http://", 7)
224                || !strncasecmp(url, "https://", 8))
225                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
226                    || strstr(url, ".sdp?"))) {
227        source = new RTSPSource(
228                notify, httpService, url, headers, mUIDValid, mUID, true);
229    } else {
230        sp<GenericSource> genericSource =
231                new GenericSource(notify, mUIDValid, mUID);
232        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
233        // The correct flags will be updated in Source::kWhatFlagsChanged
234        // handler when  GenericSource is prepared.
235
236        status_t err = genericSource->setDataSource(httpService, url, headers);
237
238        if (err == OK) {
239            source = genericSource;
240        } else {
241            ALOGE("Failed to set data source!");
242        }
243    }
244    msg->setObject("source", source);
245    msg->post();
246}
247
248void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
249    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
250
251    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
252
253    sp<GenericSource> source =
254            new GenericSource(notify, mUIDValid, mUID);
255
256    status_t err = source->setDataSource(fd, offset, length);
257
258    if (err != OK) {
259        ALOGE("Failed to set data source!");
260        source = NULL;
261    }
262
263    msg->setObject("source", source);
264    msg->post();
265}
266
267void NuPlayer::prepareAsync() {
268    (new AMessage(kWhatPrepare, id()))->post();
269}
270
271void NuPlayer::setVideoSurfaceTextureAsync(
272        const sp<IGraphicBufferProducer> &bufferProducer) {
273    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id());
274
275    if (bufferProducer == NULL) {
276        msg->setObject("native-window", NULL);
277    } else {
278        msg->setObject(
279                "native-window",
280                new NativeWindowWrapper(
281                    new Surface(bufferProducer)));
282    }
283
284    msg->post();
285}
286
287void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
288    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id());
289    msg->setObject("sink", sink);
290    msg->post();
291}
292
293void NuPlayer::start() {
294    (new AMessage(kWhatStart, id()))->post();
295}
296
297void NuPlayer::pause() {
298    (new AMessage(kWhatPause, id()))->post();
299}
300
301void NuPlayer::resume() {
302    (new AMessage(kWhatResume, id()))->post();
303}
304
305void NuPlayer::resetAsync() {
306    (new AMessage(kWhatReset, id()))->post();
307}
308
309void NuPlayer::seekToAsync(int64_t seekTimeUs) {
310    sp<AMessage> msg = new AMessage(kWhatSeek, id());
311    msg->setInt64("seekTimeUs", seekTimeUs);
312    msg->post();
313}
314
315// static
316bool NuPlayer::IsFlushingState(FlushStatus state, bool *needShutdown) {
317    switch (state) {
318        case FLUSHING_DECODER:
319            if (needShutdown != NULL) {
320                *needShutdown = false;
321            }
322            return true;
323
324        case FLUSHING_DECODER_SHUTDOWN:
325            if (needShutdown != NULL) {
326                *needShutdown = true;
327            }
328            return true;
329
330        default:
331            return false;
332    }
333}
334
335void NuPlayer::writeTrackInfo(
336        Parcel* reply, const sp<AMessage> format) const {
337    int32_t trackType;
338    CHECK(format->findInt32("type", &trackType));
339
340    AString lang;
341    CHECK(format->findString("language", &lang));
342
343    reply->writeInt32(2); // write something non-zero
344    reply->writeInt32(trackType);
345    reply->writeString16(String16(lang.c_str()));
346
347    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
348        AString mime;
349        CHECK(format->findString("mime", &mime));
350
351        int32_t isAuto, isDefault, isForced;
352        CHECK(format->findInt32("auto", &isAuto));
353        CHECK(format->findInt32("default", &isDefault));
354        CHECK(format->findInt32("forced", &isForced));
355
356        reply->writeString16(String16(mime.c_str()));
357        reply->writeInt32(isAuto);
358        reply->writeInt32(isDefault);
359        reply->writeInt32(isForced);
360    }
361}
362
363void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
364    switch (msg->what()) {
365        case kWhatSetDataSource:
366        {
367            ALOGV("kWhatSetDataSource");
368
369            CHECK(mSource == NULL);
370
371            status_t err = OK;
372            sp<RefBase> obj;
373            CHECK(msg->findObject("source", &obj));
374            if (obj != NULL) {
375                mSource = static_cast<Source *>(obj.get());
376            } else {
377                err = UNKNOWN_ERROR;
378            }
379
380            CHECK(mDriver != NULL);
381            sp<NuPlayerDriver> driver = mDriver.promote();
382            if (driver != NULL) {
383                driver->notifySetDataSourceCompleted(err);
384            }
385            break;
386        }
387
388        case kWhatPrepare:
389        {
390            mSource->prepareAsync();
391            break;
392        }
393
394        case kWhatGetTrackInfo:
395        {
396            uint32_t replyID;
397            CHECK(msg->senderAwaitsResponse(&replyID));
398
399            Parcel* reply;
400            CHECK(msg->findPointer("reply", (void**)&reply));
401
402            size_t inbandTracks = 0;
403            if (mSource != NULL) {
404                inbandTracks = mSource->getTrackCount();
405            }
406
407            size_t ccTracks = 0;
408            if (mCCDecoder != NULL) {
409                ccTracks = mCCDecoder->getTrackCount();
410            }
411
412            // total track count
413            reply->writeInt32(inbandTracks + ccTracks);
414
415            // write inband tracks
416            for (size_t i = 0; i < inbandTracks; ++i) {
417                writeTrackInfo(reply, mSource->getTrackInfo(i));
418            }
419
420            // write CC track
421            for (size_t i = 0; i < ccTracks; ++i) {
422                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
423            }
424
425            sp<AMessage> response = new AMessage;
426            response->postReply(replyID);
427            break;
428        }
429
430        case kWhatGetSelectedTrack:
431        {
432            status_t err = INVALID_OPERATION;
433            if (mSource != NULL) {
434                err = OK;
435
436                int32_t type32;
437                CHECK(msg->findInt32("type", (int32_t*)&type32));
438                media_track_type type = (media_track_type)type32;
439                ssize_t selectedTrack = mSource->getSelectedTrack(type);
440
441                Parcel* reply;
442                CHECK(msg->findPointer("reply", (void**)&reply));
443                reply->writeInt32(selectedTrack);
444            }
445
446            sp<AMessage> response = new AMessage;
447            response->setInt32("err", err);
448
449            uint32_t replyID;
450            CHECK(msg->senderAwaitsResponse(&replyID));
451            response->postReply(replyID);
452            break;
453        }
454
455        case kWhatSelectTrack:
456        {
457            uint32_t replyID;
458            CHECK(msg->senderAwaitsResponse(&replyID));
459
460            size_t trackIndex;
461            int32_t select;
462            CHECK(msg->findSize("trackIndex", &trackIndex));
463            CHECK(msg->findInt32("select", &select));
464
465            status_t err = INVALID_OPERATION;
466
467            size_t inbandTracks = 0;
468            if (mSource != NULL) {
469                inbandTracks = mSource->getTrackCount();
470            }
471            size_t ccTracks = 0;
472            if (mCCDecoder != NULL) {
473                ccTracks = mCCDecoder->getTrackCount();
474            }
475
476            if (trackIndex < inbandTracks) {
477                err = mSource->selectTrack(trackIndex, select);
478
479                if (!select && err == OK) {
480                    int32_t type;
481                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
482                    if (info != NULL
483                            && info->findInt32("type", &type)
484                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
485                        ++mTimedTextGeneration;
486                    }
487                }
488            } else {
489                trackIndex -= inbandTracks;
490
491                if (trackIndex < ccTracks) {
492                    err = mCCDecoder->selectTrack(trackIndex, select);
493                }
494            }
495
496            sp<AMessage> response = new AMessage;
497            response->setInt32("err", err);
498
499            response->postReply(replyID);
500            break;
501        }
502
503        case kWhatPollDuration:
504        {
505            int32_t generation;
506            CHECK(msg->findInt32("generation", &generation));
507
508            if (generation != mPollDurationGeneration) {
509                // stale
510                break;
511            }
512
513            int64_t durationUs;
514            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
515                sp<NuPlayerDriver> driver = mDriver.promote();
516                if (driver != NULL) {
517                    driver->notifyDuration(durationUs);
518                }
519            }
520
521            msg->post(1000000ll);  // poll again in a second.
522            break;
523        }
524
525        case kWhatSetVideoNativeWindow:
526        {
527            ALOGV("kWhatSetVideoNativeWindow");
528
529            mDeferredActions.push_back(
530                    new ShutdownDecoderAction(
531                        false /* audio */, true /* video */));
532
533            sp<RefBase> obj;
534            CHECK(msg->findObject("native-window", &obj));
535
536            mDeferredActions.push_back(
537                    new SetSurfaceAction(
538                        static_cast<NativeWindowWrapper *>(obj.get())));
539
540            if (obj != NULL) {
541                // If there is a new surface texture, instantiate decoders
542                // again if possible.
543                mDeferredActions.push_back(
544                        new SimpleAction(&NuPlayer::performScanSources));
545            }
546
547            processDeferredActions();
548            break;
549        }
550
551        case kWhatSetAudioSink:
552        {
553            ALOGV("kWhatSetAudioSink");
554
555            sp<RefBase> obj;
556            CHECK(msg->findObject("sink", &obj));
557
558            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
559            break;
560        }
561
562        case kWhatStart:
563        {
564            ALOGV("kWhatStart");
565
566            mVideoIsAVC = false;
567            mOffloadAudio = false;
568            mAudioEOS = false;
569            mVideoEOS = false;
570            mSkipRenderingAudioUntilMediaTimeUs = -1;
571            mSkipRenderingVideoUntilMediaTimeUs = -1;
572            mVideoLateByUs = 0;
573            mNumFramesTotal = 0;
574            mNumFramesDropped = 0;
575            mStarted = true;
576
577            /* instantiate decoders now for secure playback */
578            if (mSourceFlags & Source::FLAG_SECURE) {
579                if (mNativeWindow != NULL) {
580                    instantiateDecoder(false, &mVideoDecoder);
581                }
582
583                if (mAudioSink != NULL) {
584                    instantiateDecoder(true, &mAudioDecoder);
585                }
586            }
587
588            mSource->start();
589
590            uint32_t flags = 0;
591
592            if (mSource->isRealTime()) {
593                flags |= Renderer::FLAG_REAL_TIME;
594            }
595
596            sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
597            audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
598            if (mAudioSink != NULL) {
599                streamType = mAudioSink->getAudioStreamType();
600            }
601
602            sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
603
604            mOffloadAudio =
605                canOffloadStream(audioMeta, (videoFormat != NULL),
606                                 true /* is_streaming */, streamType);
607            if (mOffloadAudio) {
608                flags |= Renderer::FLAG_OFFLOAD_AUDIO;
609            }
610
611            mRenderer = new Renderer(
612                    mAudioSink,
613                    new AMessage(kWhatRendererNotify, id()),
614                    flags);
615
616            mRendererLooper = new ALooper;
617            mRendererLooper->setName("NuPlayerRenderer");
618            mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
619            mRendererLooper->registerHandler(mRenderer);
620
621            postScanSources();
622            break;
623        }
624
625        case kWhatScanSources:
626        {
627            int32_t generation;
628            CHECK(msg->findInt32("generation", &generation));
629            if (generation != mScanSourcesGeneration) {
630                // Drop obsolete msg.
631                break;
632            }
633
634            mScanSourcesPending = false;
635
636            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
637                 mAudioDecoder != NULL, mVideoDecoder != NULL);
638
639            bool mHadAnySourcesBefore =
640                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
641
642            // initialize video before audio because successful initialization of
643            // video may change deep buffer mode of audio.
644            if (mNativeWindow != NULL) {
645                instantiateDecoder(false, &mVideoDecoder);
646            }
647
648            if (mAudioSink != NULL) {
649                if (mOffloadAudio) {
650                    // open audio sink early under offload mode.
651                    sp<AMessage> format = mSource->getFormat(true /*audio*/);
652                    openAudioSink(format, true /*offloadOnly*/);
653                }
654                instantiateDecoder(true, &mAudioDecoder);
655            }
656
657            if (!mHadAnySourcesBefore
658                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
659                // This is the first time we've found anything playable.
660
661                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
662                    schedulePollDuration();
663                }
664            }
665
666            status_t err;
667            if ((err = mSource->feedMoreTSData()) != OK) {
668                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
669                    // We're not currently decoding anything (no audio or
670                    // video tracks found) and we just ran out of input data.
671
672                    if (err == ERROR_END_OF_STREAM) {
673                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
674                    } else {
675                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
676                    }
677                }
678                break;
679            }
680
681            if ((mAudioDecoder == NULL && mAudioSink != NULL)
682                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
683                msg->post(100000ll);
684                mScanSourcesPending = true;
685            }
686            break;
687        }
688
689        case kWhatVideoNotify:
690        case kWhatAudioNotify:
691        {
692            bool audio = msg->what() == kWhatAudioNotify;
693
694            int32_t what;
695            CHECK(msg->findInt32("what", &what));
696
697            if (what == Decoder::kWhatFillThisBuffer) {
698                status_t err = feedDecoderInputData(
699                        audio, msg);
700
701                if (err == -EWOULDBLOCK) {
702                    if (mSource->feedMoreTSData() == OK) {
703                        msg->post(10000ll);
704                    }
705                }
706            } else if (what == Decoder::kWhatEOS) {
707                int32_t err;
708                CHECK(msg->findInt32("err", &err));
709
710                if (err == ERROR_END_OF_STREAM) {
711                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
712                } else {
713                    ALOGV("got %s decoder EOS w/ error %d",
714                         audio ? "audio" : "video",
715                         err);
716                }
717
718                mRenderer->queueEOS(audio, err);
719            } else if (what == Decoder::kWhatFlushCompleted) {
720                bool needShutdown;
721
722                if (audio) {
723                    CHECK(IsFlushingState(mFlushingAudio, &needShutdown));
724                    mFlushingAudio = FLUSHED;
725                } else {
726                    CHECK(IsFlushingState(mFlushingVideo, &needShutdown));
727                    mFlushingVideo = FLUSHED;
728
729                    mVideoLateByUs = 0;
730                }
731
732                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
733
734                if (needShutdown) {
735                    ALOGV("initiating %s decoder shutdown",
736                         audio ? "audio" : "video");
737
738                    (audio ? mAudioDecoder : mVideoDecoder)->initiateShutdown();
739
740                    if (audio) {
741                        mFlushingAudio = SHUTTING_DOWN_DECODER;
742                    } else {
743                        mFlushingVideo = SHUTTING_DOWN_DECODER;
744                    }
745                }
746
747                finishFlushIfPossible();
748            } else if (what == Decoder::kWhatOutputFormatChanged) {
749                sp<AMessage> format;
750                CHECK(msg->findMessage("format", &format));
751
752                if (audio) {
753                    openAudioSink(format, false /*offloadOnly*/);
754                } else {
755                    // video
756                    sp<AMessage> inputFormat =
757                            mSource->getFormat(false /* audio */);
758
759                    updateVideoSize(inputFormat, format);
760                }
761            } else if (what == Decoder::kWhatShutdownCompleted) {
762                ALOGV("%s shutdown completed", audio ? "audio" : "video");
763                if (audio) {
764                    mAudioDecoder.clear();
765
766                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
767                    mFlushingAudio = SHUT_DOWN;
768                } else {
769                    mVideoDecoder.clear();
770
771                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
772                    mFlushingVideo = SHUT_DOWN;
773                }
774
775                finishFlushIfPossible();
776            } else if (what == Decoder::kWhatError) {
777                ALOGE("Received error from %s decoder, aborting playback.",
778                     audio ? "audio" : "video");
779
780                status_t err;
781                if (!msg->findInt32("err", &err)) {
782                    err = UNKNOWN_ERROR;
783                }
784                mRenderer->queueEOS(audio, err);
785            } else if (what == Decoder::kWhatDrainThisBuffer) {
786                renderBuffer(audio, msg);
787            } else {
788                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
789                      what,
790                      what >> 24,
791                      (what >> 16) & 0xff,
792                      (what >> 8) & 0xff,
793                      what & 0xff);
794            }
795
796            break;
797        }
798
799        case kWhatRendererNotify:
800        {
801            int32_t what;
802            CHECK(msg->findInt32("what", &what));
803
804            if (what == Renderer::kWhatEOS) {
805                int32_t audio;
806                CHECK(msg->findInt32("audio", &audio));
807
808                int32_t finalResult;
809                CHECK(msg->findInt32("finalResult", &finalResult));
810
811                if (audio) {
812                    mAudioEOS = true;
813                } else {
814                    mVideoEOS = true;
815                }
816
817                if (finalResult == ERROR_END_OF_STREAM) {
818                    ALOGV("reached %s EOS", audio ? "audio" : "video");
819                } else {
820                    ALOGE("%s track encountered an error (%d)",
821                         audio ? "audio" : "video", finalResult);
822
823                    notifyListener(
824                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
825                }
826
827                if ((mAudioEOS || mAudioDecoder == NULL)
828                        && (mVideoEOS || mVideoDecoder == NULL)) {
829                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
830                }
831            } else if (what == Renderer::kWhatPosition) {
832                int64_t positionUs;
833                CHECK(msg->findInt64("positionUs", &positionUs));
834
835                CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs));
836
837                if (mDriver != NULL) {
838                    sp<NuPlayerDriver> driver = mDriver.promote();
839                    if (driver != NULL) {
840                        driver->notifyPosition(positionUs);
841
842                        driver->notifyFrameStats(
843                                mNumFramesTotal, mNumFramesDropped);
844                    }
845                }
846            } else if (what == Renderer::kWhatFlushComplete) {
847                int32_t audio;
848                CHECK(msg->findInt32("audio", &audio));
849
850                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
851            } else if (what == Renderer::kWhatVideoRenderingStart) {
852                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
853            } else if (what == Renderer::kWhatMediaRenderingStart) {
854                ALOGV("media rendering started");
855                notifyListener(MEDIA_STARTED, 0, 0);
856            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
857                ALOGV("Tear down audio offload, fall back to s/w path");
858                int64_t positionUs;
859                CHECK(msg->findInt64("positionUs", &positionUs));
860                closeAudioSink();
861                mAudioDecoder.clear();
862                mRenderer->flush(true /* audio */);
863                if (mVideoDecoder != NULL) {
864                    mRenderer->flush(false /* audio */);
865                }
866                mRenderer->signalDisableOffloadAudio();
867                mOffloadAudio = false;
868
869                performSeek(positionUs);
870                instantiateDecoder(true /* audio */, &mAudioDecoder);
871            }
872            break;
873        }
874
875        case kWhatMoreDataQueued:
876        {
877            break;
878        }
879
880        case kWhatReset:
881        {
882            ALOGV("kWhatReset");
883
884            mDeferredActions.push_back(
885                    new ShutdownDecoderAction(
886                        true /* audio */, true /* video */));
887
888            mDeferredActions.push_back(
889                    new SimpleAction(&NuPlayer::performReset));
890
891            processDeferredActions();
892            break;
893        }
894
895        case kWhatSeek:
896        {
897            int64_t seekTimeUs;
898            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
899
900            ALOGV("kWhatSeek seekTimeUs=%lld us", seekTimeUs);
901
902            mDeferredActions.push_back(
903                    new SimpleAction(&NuPlayer::performDecoderFlush));
904
905            mDeferredActions.push_back(new SeekAction(seekTimeUs));
906
907            processDeferredActions();
908            break;
909        }
910
911        case kWhatPause:
912        {
913            CHECK(mRenderer != NULL);
914            mSource->pause();
915            mRenderer->pause();
916            break;
917        }
918
919        case kWhatResume:
920        {
921            CHECK(mRenderer != NULL);
922            mSource->resume();
923            mRenderer->resume();
924            break;
925        }
926
927        case kWhatSourceNotify:
928        {
929            onSourceNotify(msg);
930            break;
931        }
932
933        case kWhatClosedCaptionNotify:
934        {
935            onClosedCaptionNotify(msg);
936            break;
937        }
938
939        default:
940            TRESPASS();
941            break;
942    }
943}
944
945void NuPlayer::finishFlushIfPossible() {
946    if (mFlushingAudio != FLUSHED && mFlushingAudio != SHUT_DOWN) {
947        return;
948    }
949
950    if (mFlushingVideo != FLUSHED && mFlushingVideo != SHUT_DOWN) {
951        return;
952    }
953
954    ALOGV("both audio and video are flushed now.");
955
956    if (mTimeDiscontinuityPending) {
957        mRenderer->signalTimeDiscontinuity();
958        mTimeDiscontinuityPending = false;
959    }
960
961    if (mAudioDecoder != NULL) {
962        mAudioDecoder->signalResume();
963    }
964
965    if (mVideoDecoder != NULL) {
966        mVideoDecoder->signalResume();
967    }
968
969    mFlushingAudio = NONE;
970    mFlushingVideo = NONE;
971
972    processDeferredActions();
973}
974
975void NuPlayer::postScanSources() {
976    if (mScanSourcesPending) {
977        return;
978    }
979
980    sp<AMessage> msg = new AMessage(kWhatScanSources, id());
981    msg->setInt32("generation", mScanSourcesGeneration);
982    msg->post();
983
984    mScanSourcesPending = true;
985}
986
987void NuPlayer::openAudioSink(const sp<AMessage> &format, bool offloadOnly) {
988    ALOGV("openAudioSink: offloadOnly(%d) mOffloadAudio(%d)",
989            offloadOnly, mOffloadAudio);
990    bool audioSinkChanged = false;
991
992    int32_t numChannels;
993    CHECK(format->findInt32("channel-count", &numChannels));
994
995    int32_t channelMask;
996    if (!format->findInt32("channel-mask", &channelMask)) {
997        // signal to the AudioSink to derive the mask from count.
998        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
999    }
1000
1001    int32_t sampleRate;
1002    CHECK(format->findInt32("sample-rate", &sampleRate));
1003
1004    uint32_t flags;
1005    int64_t durationUs;
1006    // FIXME: we should handle the case where the video decoder
1007    // is created after we receive the format change indication.
1008    // Current code will just make that we select deep buffer
1009    // with video which should not be a problem as it should
1010    // not prevent from keeping A/V sync.
1011    if (mVideoDecoder == NULL &&
1012            mSource->getDuration(&durationUs) == OK &&
1013            durationUs
1014                > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
1015        flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1016    } else {
1017        flags = AUDIO_OUTPUT_FLAG_NONE;
1018    }
1019
1020    if (mOffloadAudio) {
1021        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1022        AString mime;
1023        CHECK(format->findString("mime", &mime));
1024        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1025
1026        if (err != OK) {
1027            ALOGE("Couldn't map mime \"%s\" to a valid "
1028                    "audio_format", mime.c_str());
1029            mOffloadAudio = false;
1030        } else {
1031            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1032                    mime.c_str(), audioFormat);
1033
1034            int avgBitRate = -1;
1035            format->findInt32("bit-rate", &avgBitRate);
1036
1037            int32_t aacProfile = -1;
1038            if (audioFormat == AUDIO_FORMAT_AAC
1039                    && format->findInt32("aac-profile", &aacProfile)) {
1040                // Redefine AAC format as per aac profile
1041                mapAACProfileToAudioFormat(
1042                        audioFormat,
1043                        aacProfile);
1044            }
1045
1046            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1047            offloadInfo.duration_us = -1;
1048            format->findInt64(
1049                    "durationUs", &offloadInfo.duration_us);
1050            offloadInfo.sample_rate = sampleRate;
1051            offloadInfo.channel_mask = channelMask;
1052            offloadInfo.format = audioFormat;
1053            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1054            offloadInfo.bit_rate = avgBitRate;
1055            offloadInfo.has_video = (mVideoDecoder != NULL);
1056            offloadInfo.is_streaming = true;
1057
1058            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1059                ALOGV("openAudioSink: no change in offload mode");
1060                return;  // no change from previous configuration, everything ok.
1061            }
1062            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1063            flags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1064            audioSinkChanged = true;
1065            mAudioSink->close();
1066            err = mAudioSink->open(
1067                    sampleRate,
1068                    numChannels,
1069                    (audio_channel_mask_t)channelMask,
1070                    audioFormat,
1071                    8 /* bufferCount */,
1072                    &NuPlayer::Renderer::AudioSinkCallback,
1073                    mRenderer.get(),
1074                    (audio_output_flags_t)flags,
1075                    &offloadInfo);
1076
1077            if (err == OK) {
1078                // If the playback is offloaded to h/w, we pass
1079                // the HAL some metadata information.
1080                // We don't want to do this for PCM because it
1081                // will be going through the AudioFlinger mixer
1082                // before reaching the hardware.
1083                sp<MetaData> audioMeta =
1084                        mSource->getFormatMeta(true /* audio */);
1085                sendMetaDataToHal(mAudioSink, audioMeta);
1086                mCurrentOffloadInfo = offloadInfo;
1087                err = mAudioSink->start();
1088                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1089            }
1090            if (err != OK) {
1091                // Clean up, fall back to non offload mode.
1092                mAudioSink->close();
1093                mRenderer->signalDisableOffloadAudio();
1094                mOffloadAudio = false;
1095                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1096                ALOGV("openAudioSink: offload failed");
1097            }
1098        }
1099    }
1100    if (!offloadOnly && !mOffloadAudio) {
1101        flags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1102        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1103
1104        audioSinkChanged = true;
1105        mAudioSink->close();
1106        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1107        CHECK_EQ(mAudioSink->open(
1108                    sampleRate,
1109                    numChannels,
1110                    (audio_channel_mask_t)channelMask,
1111                    AUDIO_FORMAT_PCM_16_BIT,
1112                    8 /* bufferCount */,
1113                    NULL,
1114                    NULL,
1115                    (audio_output_flags_t)flags),
1116                 (status_t)OK);
1117        mAudioSink->start();
1118    }
1119    if (audioSinkChanged) {
1120        mRenderer->signalAudioSinkChanged();
1121    }
1122}
1123
1124void NuPlayer::closeAudioSink() {
1125    mAudioSink->close();
1126    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1127}
1128
1129status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
1130    if (*decoder != NULL) {
1131        return OK;
1132    }
1133
1134    sp<AMessage> format = mSource->getFormat(audio);
1135
1136    if (format == NULL) {
1137        return -EWOULDBLOCK;
1138    }
1139
1140    if (!audio) {
1141        AString mime;
1142        CHECK(format->findString("mime", &mime));
1143        mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
1144
1145        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
1146        mCCDecoder = new CCDecoder(ccNotify);
1147
1148        if (mSourceFlags & Source::FLAG_SECURE) {
1149            format->setInt32("secure", true);
1150        }
1151    }
1152
1153    sp<AMessage> notify =
1154        new AMessage(audio ? kWhatAudioNotify : kWhatVideoNotify,
1155                     id());
1156
1157    if (audio) {
1158        if (mOffloadAudio) {
1159            *decoder = new DecoderPassThrough(notify);
1160        } else {
1161            *decoder = new Decoder(notify);
1162        }
1163    } else {
1164        *decoder = new Decoder(notify, mNativeWindow);
1165    }
1166    (*decoder)->init();
1167    (*decoder)->configure(format);
1168
1169    // allocate buffers to decrypt widevine source buffers
1170    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1171        Vector<sp<ABuffer> > inputBufs;
1172        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1173
1174        Vector<MediaBuffer *> mediaBufs;
1175        for (size_t i = 0; i < inputBufs.size(); i++) {
1176            const sp<ABuffer> &buffer = inputBufs[i];
1177            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1178            mediaBufs.push(mbuf);
1179        }
1180
1181        status_t err = mSource->setBuffers(audio, mediaBufs);
1182        if (err != OK) {
1183            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1184                mediaBufs[i]->release();
1185            }
1186            mediaBufs.clear();
1187            ALOGE("Secure source didn't support secure mediaBufs.");
1188            return err;
1189        }
1190    }
1191    return OK;
1192}
1193
1194status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
1195    sp<AMessage> reply;
1196    CHECK(msg->findMessage("reply", &reply));
1197
1198    if ((audio && IsFlushingState(mFlushingAudio))
1199            || (!audio && IsFlushingState(mFlushingVideo))) {
1200        reply->setInt32("err", INFO_DISCONTINUITY);
1201        reply->post();
1202        return OK;
1203    }
1204
1205    sp<ABuffer> accessUnit;
1206
1207    bool dropAccessUnit;
1208    do {
1209        status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
1210
1211        if (err == -EWOULDBLOCK) {
1212            return err;
1213        } else if (err != OK) {
1214            if (err == INFO_DISCONTINUITY) {
1215                int32_t type;
1216                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
1217
1218                bool formatChange =
1219                    (audio &&
1220                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
1221                    || (!audio &&
1222                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
1223
1224                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
1225
1226                ALOGI("%s discontinuity (formatChange=%d, time=%d)",
1227                     audio ? "audio" : "video", formatChange, timeChange);
1228
1229                if (audio) {
1230                    mSkipRenderingAudioUntilMediaTimeUs = -1;
1231                } else {
1232                    mSkipRenderingVideoUntilMediaTimeUs = -1;
1233                }
1234
1235                if (timeChange) {
1236                    sp<AMessage> extra;
1237                    if (accessUnit->meta()->findMessage("extra", &extra)
1238                            && extra != NULL) {
1239                        int64_t resumeAtMediaTimeUs;
1240                        if (extra->findInt64(
1241                                    "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
1242                            ALOGI("suppressing rendering of %s until %lld us",
1243                                    audio ? "audio" : "video", resumeAtMediaTimeUs);
1244
1245                            if (audio) {
1246                                mSkipRenderingAudioUntilMediaTimeUs =
1247                                    resumeAtMediaTimeUs;
1248                            } else {
1249                                mSkipRenderingVideoUntilMediaTimeUs =
1250                                    resumeAtMediaTimeUs;
1251                            }
1252                        }
1253                    }
1254                }
1255
1256                mTimeDiscontinuityPending =
1257                    mTimeDiscontinuityPending || timeChange;
1258
1259                if (mFlushingAudio == NONE && mFlushingVideo == NONE) {
1260                    // And we'll resume scanning sources once we're done
1261                    // flushing.
1262                    mDeferredActions.push_front(
1263                            new SimpleAction(
1264                                &NuPlayer::performScanSources));
1265                }
1266
1267                if (formatChange || timeChange) {
1268
1269                    sp<AMessage> newFormat = mSource->getFormat(audio);
1270                    sp<Decoder> &decoder = audio ? mAudioDecoder : mVideoDecoder;
1271                    if (formatChange && !decoder->supportsSeamlessFormatChange(newFormat)) {
1272                        flushDecoder(audio, /* needShutdown = */ true);
1273                    } else {
1274                        flushDecoder(audio, /* needShutdown = */ false);
1275                        err = OK;
1276                    }
1277                } else {
1278                    // This stream is unaffected by the discontinuity
1279
1280                    if (audio) {
1281                        mFlushingAudio = FLUSHED;
1282                    } else {
1283                        mFlushingVideo = FLUSHED;
1284                    }
1285
1286                    finishFlushIfPossible();
1287
1288                    return -EWOULDBLOCK;
1289                }
1290            }
1291
1292            reply->setInt32("err", err);
1293            reply->post();
1294            return OK;
1295        }
1296
1297        if (!audio) {
1298            ++mNumFramesTotal;
1299        }
1300
1301        dropAccessUnit = false;
1302        if (!audio
1303                && !(mSourceFlags & Source::FLAG_SECURE)
1304                && mVideoLateByUs > 100000ll
1305                && mVideoIsAVC
1306                && !IsAVCReferenceFrame(accessUnit)) {
1307            dropAccessUnit = true;
1308            ++mNumFramesDropped;
1309        }
1310    } while (dropAccessUnit);
1311
1312    // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
1313
1314#if 0
1315    int64_t mediaTimeUs;
1316    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
1317    ALOGV("feeding %s input buffer at media time %.2f secs",
1318         audio ? "audio" : "video",
1319         mediaTimeUs / 1E6);
1320#endif
1321
1322    if (!audio) {
1323        mCCDecoder->decode(accessUnit);
1324    }
1325
1326    reply->setBuffer("buffer", accessUnit);
1327    reply->post();
1328
1329    return OK;
1330}
1331
1332void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
1333    // ALOGV("renderBuffer %s", audio ? "audio" : "video");
1334
1335    sp<AMessage> reply;
1336    CHECK(msg->findMessage("reply", &reply));
1337
1338    if (IsFlushingState(audio ? mFlushingAudio : mFlushingVideo)) {
1339        // We're currently attempting to flush the decoder, in order
1340        // to complete this, the decoder wants all its buffers back,
1341        // so we don't want any output buffers it sent us (from before
1342        // we initiated the flush) to be stuck in the renderer's queue.
1343
1344        ALOGV("we're still flushing the %s decoder, sending its output buffer"
1345             " right back.", audio ? "audio" : "video");
1346
1347        reply->post();
1348        return;
1349    }
1350
1351    sp<ABuffer> buffer;
1352    CHECK(msg->findBuffer("buffer", &buffer));
1353
1354    int64_t mediaTimeUs;
1355    CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));
1356
1357    int64_t &skipUntilMediaTimeUs =
1358        audio
1359            ? mSkipRenderingAudioUntilMediaTimeUs
1360            : mSkipRenderingVideoUntilMediaTimeUs;
1361
1362    if (skipUntilMediaTimeUs >= 0) {
1363
1364        if (mediaTimeUs < skipUntilMediaTimeUs) {
1365            ALOGV("dropping %s buffer at time %lld as requested.",
1366                 audio ? "audio" : "video",
1367                 mediaTimeUs);
1368
1369            reply->post();
1370            return;
1371        }
1372
1373        skipUntilMediaTimeUs = -1;
1374    }
1375
1376    if (!audio && mCCDecoder->isSelected()) {
1377        mCCDecoder->display(mediaTimeUs);
1378    }
1379
1380    mRenderer->queueBuffer(audio, buffer, reply);
1381}
1382
1383void NuPlayer::updateVideoSize(
1384        const sp<AMessage> &inputFormat,
1385        const sp<AMessage> &outputFormat) {
1386    if (inputFormat == NULL) {
1387        ALOGW("Unknown video size, reporting 0x0!");
1388        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1389        return;
1390    }
1391
1392    int32_t displayWidth, displayHeight;
1393    int32_t cropLeft, cropTop, cropRight, cropBottom;
1394
1395    if (outputFormat != NULL) {
1396        int32_t width, height;
1397        CHECK(outputFormat->findInt32("width", &width));
1398        CHECK(outputFormat->findInt32("height", &height));
1399
1400        int32_t cropLeft, cropTop, cropRight, cropBottom;
1401        CHECK(outputFormat->findRect(
1402                    "crop",
1403                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1404
1405        displayWidth = cropRight - cropLeft + 1;
1406        displayHeight = cropBottom - cropTop + 1;
1407
1408        ALOGV("Video output format changed to %d x %d "
1409             "(crop: %d x %d @ (%d, %d))",
1410             width, height,
1411             displayWidth,
1412             displayHeight,
1413             cropLeft, cropTop);
1414    } else {
1415        CHECK(inputFormat->findInt32("width", &displayWidth));
1416        CHECK(inputFormat->findInt32("height", &displayHeight));
1417
1418        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1419    }
1420
1421    // Take into account sample aspect ratio if necessary:
1422    int32_t sarWidth, sarHeight;
1423    if (inputFormat->findInt32("sar-width", &sarWidth)
1424            && inputFormat->findInt32("sar-height", &sarHeight)) {
1425        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1426
1427        displayWidth = (displayWidth * sarWidth) / sarHeight;
1428
1429        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1430    }
1431
1432    int32_t rotationDegrees;
1433    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1434        rotationDegrees = 0;
1435    }
1436
1437    if (rotationDegrees == 90 || rotationDegrees == 270) {
1438        int32_t tmp = displayWidth;
1439        displayWidth = displayHeight;
1440        displayHeight = tmp;
1441    }
1442
1443    notifyListener(
1444            MEDIA_SET_VIDEO_SIZE,
1445            displayWidth,
1446            displayHeight);
1447}
1448
1449void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1450    if (mDriver == NULL) {
1451        return;
1452    }
1453
1454    sp<NuPlayerDriver> driver = mDriver.promote();
1455
1456    if (driver == NULL) {
1457        return;
1458    }
1459
1460    driver->notifyListener(msg, ext1, ext2, in);
1461}
1462
1463void NuPlayer::flushDecoder(bool audio, bool needShutdown) {
1464    ALOGV("[%s] flushDecoder needShutdown=%d",
1465          audio ? "audio" : "video", needShutdown);
1466
1467    if ((audio && mAudioDecoder == NULL) || (!audio && mVideoDecoder == NULL)) {
1468        ALOGI("flushDecoder %s without decoder present",
1469             audio ? "audio" : "video");
1470    }
1471
1472    // Make sure we don't continue to scan sources until we finish flushing.
1473    ++mScanSourcesGeneration;
1474    mScanSourcesPending = false;
1475
1476    (audio ? mAudioDecoder : mVideoDecoder)->signalFlush();
1477    mRenderer->flush(audio);
1478
1479    FlushStatus newStatus =
1480        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1481
1482    if (audio) {
1483        CHECK(mFlushingAudio == NONE
1484                || mFlushingAudio == AWAITING_DISCONTINUITY);
1485
1486        mFlushingAudio = newStatus;
1487
1488        if (mFlushingVideo == NONE) {
1489            mFlushingVideo = (mVideoDecoder != NULL)
1490                ? AWAITING_DISCONTINUITY
1491                : FLUSHED;
1492        }
1493    } else {
1494        CHECK(mFlushingVideo == NONE
1495                || mFlushingVideo == AWAITING_DISCONTINUITY);
1496
1497        mFlushingVideo = newStatus;
1498
1499        if (mFlushingAudio == NONE) {
1500            mFlushingAudio = (mAudioDecoder != NULL)
1501                ? AWAITING_DISCONTINUITY
1502                : FLUSHED;
1503        }
1504    }
1505}
1506
1507void NuPlayer::queueDecoderShutdown(
1508        bool audio, bool video, const sp<AMessage> &reply) {
1509    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1510
1511    mDeferredActions.push_back(
1512            new ShutdownDecoderAction(audio, video));
1513
1514    mDeferredActions.push_back(
1515            new SimpleAction(&NuPlayer::performScanSources));
1516
1517    mDeferredActions.push_back(new PostMessageAction(reply));
1518
1519    processDeferredActions();
1520}
1521
1522status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1523    mVideoScalingMode = mode;
1524    if (mNativeWindow != NULL) {
1525        status_t ret = native_window_set_scaling_mode(
1526                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1527        if (ret != OK) {
1528            ALOGE("Failed to set scaling mode (%d): %s",
1529                -ret, strerror(-ret));
1530            return ret;
1531        }
1532    }
1533    return OK;
1534}
1535
1536status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1537    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id());
1538    msg->setPointer("reply", reply);
1539
1540    sp<AMessage> response;
1541    status_t err = msg->postAndAwaitResponse(&response);
1542    return err;
1543}
1544
1545status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1546    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id());
1547    msg->setPointer("reply", reply);
1548    msg->setInt32("type", type);
1549
1550    sp<AMessage> response;
1551    status_t err = msg->postAndAwaitResponse(&response);
1552    if (err == OK && response != NULL) {
1553        CHECK(response->findInt32("err", &err));
1554    }
1555    return err;
1556}
1557
1558status_t NuPlayer::selectTrack(size_t trackIndex, bool select) {
1559    sp<AMessage> msg = new AMessage(kWhatSelectTrack, id());
1560    msg->setSize("trackIndex", trackIndex);
1561    msg->setInt32("select", select);
1562
1563    sp<AMessage> response;
1564    status_t err = msg->postAndAwaitResponse(&response);
1565
1566    if (err != OK) {
1567        return err;
1568    }
1569
1570    if (!response->findInt32("err", &err)) {
1571        err = OK;
1572    }
1573
1574    return err;
1575}
1576
1577void NuPlayer::schedulePollDuration() {
1578    sp<AMessage> msg = new AMessage(kWhatPollDuration, id());
1579    msg->setInt32("generation", mPollDurationGeneration);
1580    msg->post();
1581}
1582
1583void NuPlayer::cancelPollDuration() {
1584    ++mPollDurationGeneration;
1585}
1586
1587void NuPlayer::processDeferredActions() {
1588    while (!mDeferredActions.empty()) {
1589        // We won't execute any deferred actions until we're no longer in
1590        // an intermediate state, i.e. one more more decoders are currently
1591        // flushing or shutting down.
1592
1593        if (mRenderer != NULL) {
1594            // There's an edge case where the renderer owns all output
1595            // buffers and is paused, therefore the decoder will not read
1596            // more input data and will never encounter the matching
1597            // discontinuity. To avoid this, we resume the renderer.
1598
1599            if (mFlushingAudio == AWAITING_DISCONTINUITY
1600                    || mFlushingVideo == AWAITING_DISCONTINUITY) {
1601                mRenderer->resume();
1602            }
1603        }
1604
1605        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1606            // We're currently flushing, postpone the reset until that's
1607            // completed.
1608
1609            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1610                  mFlushingAudio, mFlushingVideo);
1611
1612            break;
1613        }
1614
1615        sp<Action> action = *mDeferredActions.begin();
1616        mDeferredActions.erase(mDeferredActions.begin());
1617
1618        action->execute(this);
1619    }
1620}
1621
1622void NuPlayer::performSeek(int64_t seekTimeUs) {
1623    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs)",
1624          seekTimeUs,
1625          seekTimeUs / 1E6);
1626
1627    mSource->seekTo(seekTimeUs);
1628    ++mTimedTextGeneration;
1629
1630    if (mDriver != NULL) {
1631        sp<NuPlayerDriver> driver = mDriver.promote();
1632        if (driver != NULL) {
1633            driver->notifyPosition(seekTimeUs);
1634            driver->notifySeekComplete();
1635        }
1636    }
1637
1638    // everything's flushed, continue playback.
1639}
1640
1641void NuPlayer::performDecoderFlush() {
1642    ALOGV("performDecoderFlush");
1643
1644    if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
1645        return;
1646    }
1647
1648    mTimeDiscontinuityPending = true;
1649
1650    if (mAudioDecoder != NULL) {
1651        flushDecoder(true /* audio */, false /* needShutdown */);
1652    }
1653
1654    if (mVideoDecoder != NULL) {
1655        flushDecoder(false /* audio */, false /* needShutdown */);
1656    }
1657}
1658
1659void NuPlayer::performDecoderShutdown(bool audio, bool video) {
1660    ALOGV("performDecoderShutdown audio=%d, video=%d", audio, video);
1661
1662    if ((!audio || mAudioDecoder == NULL)
1663            && (!video || mVideoDecoder == NULL)) {
1664        return;
1665    }
1666
1667    mTimeDiscontinuityPending = true;
1668
1669    if (mFlushingAudio == NONE && (!audio || mAudioDecoder == NULL)) {
1670        mFlushingAudio = FLUSHED;
1671    }
1672
1673    if (mFlushingVideo == NONE && (!video || mVideoDecoder == NULL)) {
1674        mFlushingVideo = FLUSHED;
1675    }
1676
1677    if (audio && mAudioDecoder != NULL) {
1678        flushDecoder(true /* audio */, true /* needShutdown */);
1679    }
1680
1681    if (video && mVideoDecoder != NULL) {
1682        flushDecoder(false /* audio */, true /* needShutdown */);
1683    }
1684}
1685
1686void NuPlayer::performReset() {
1687    ALOGV("performReset");
1688
1689    CHECK(mAudioDecoder == NULL);
1690    CHECK(mVideoDecoder == NULL);
1691
1692    cancelPollDuration();
1693
1694    ++mScanSourcesGeneration;
1695    mScanSourcesPending = false;
1696
1697    if (mRendererLooper != NULL) {
1698        if (mRenderer != NULL) {
1699            mRendererLooper->unregisterHandler(mRenderer->id());
1700        }
1701        mRendererLooper->stop();
1702        mRendererLooper.clear();
1703    }
1704    mRenderer.clear();
1705
1706    if (mSource != NULL) {
1707        mSource->stop();
1708
1709        mSource.clear();
1710    }
1711
1712    if (mDriver != NULL) {
1713        sp<NuPlayerDriver> driver = mDriver.promote();
1714        if (driver != NULL) {
1715            driver->notifyResetComplete();
1716        }
1717    }
1718
1719    mStarted = false;
1720}
1721
1722void NuPlayer::performScanSources() {
1723    ALOGV("performScanSources");
1724
1725    if (!mStarted) {
1726        return;
1727    }
1728
1729    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1730        postScanSources();
1731    }
1732}
1733
1734void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1735    ALOGV("performSetSurface");
1736
1737    mNativeWindow = wrapper;
1738
1739    // XXX - ignore error from setVideoScalingMode for now
1740    setVideoScalingMode(mVideoScalingMode);
1741}
1742
1743void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1744    int32_t what;
1745    CHECK(msg->findInt32("what", &what));
1746
1747    switch (what) {
1748        case Source::kWhatPrepared:
1749        {
1750            if (mSource == NULL) {
1751                // This is a stale notification from a source that was
1752                // asynchronously preparing when the client called reset().
1753                // We handled the reset, the source is gone.
1754                break;
1755            }
1756
1757            int32_t err;
1758            CHECK(msg->findInt32("err", &err));
1759
1760            sp<NuPlayerDriver> driver = mDriver.promote();
1761            if (driver != NULL) {
1762                // notify duration first, so that it's definitely set when
1763                // the app received the "prepare complete" callback.
1764                int64_t durationUs;
1765                if (mSource->getDuration(&durationUs) == OK) {
1766                    driver->notifyDuration(durationUs);
1767                }
1768                driver->notifyPrepareCompleted(err);
1769            }
1770
1771            break;
1772        }
1773
1774        case Source::kWhatFlagsChanged:
1775        {
1776            uint32_t flags;
1777            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1778
1779            sp<NuPlayerDriver> driver = mDriver.promote();
1780            if (driver != NULL) {
1781                driver->notifyFlagsChanged(flags);
1782            }
1783
1784            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1785                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
1786                cancelPollDuration();
1787            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1788                    && (flags & Source::FLAG_DYNAMIC_DURATION)
1789                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
1790                schedulePollDuration();
1791            }
1792
1793            mSourceFlags = flags;
1794            break;
1795        }
1796
1797        case Source::kWhatVideoSizeChanged:
1798        {
1799            sp<AMessage> format;
1800            CHECK(msg->findMessage("format", &format));
1801
1802            updateVideoSize(format);
1803            break;
1804        }
1805
1806        case Source::kWhatBufferingStart:
1807        {
1808            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
1809            break;
1810        }
1811
1812        case Source::kWhatBufferingEnd:
1813        {
1814            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
1815            break;
1816        }
1817
1818        case Source::kWhatSubtitleData:
1819        {
1820            sp<ABuffer> buffer;
1821            CHECK(msg->findBuffer("buffer", &buffer));
1822
1823            sendSubtitleData(buffer, 0 /* baseIndex */);
1824            break;
1825        }
1826
1827        case Source::kWhatTimedTextData:
1828        {
1829            int32_t generation;
1830            if (msg->findInt32("generation", &generation)
1831                    && generation != mTimedTextGeneration) {
1832                break;
1833            }
1834
1835            sp<ABuffer> buffer;
1836            CHECK(msg->findBuffer("buffer", &buffer));
1837
1838            sp<NuPlayerDriver> driver = mDriver.promote();
1839            if (driver == NULL) {
1840                break;
1841            }
1842
1843            int posMs;
1844            int64_t timeUs, posUs;
1845            driver->getCurrentPosition(&posMs);
1846            posUs = posMs * 1000;
1847            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1848
1849            if (posUs < timeUs) {
1850                if (!msg->findInt32("generation", &generation)) {
1851                    msg->setInt32("generation", mTimedTextGeneration);
1852                }
1853                msg->post(timeUs - posUs);
1854            } else {
1855                sendTimedTextData(buffer);
1856            }
1857            break;
1858        }
1859
1860        case Source::kWhatQueueDecoderShutdown:
1861        {
1862            int32_t audio, video;
1863            CHECK(msg->findInt32("audio", &audio));
1864            CHECK(msg->findInt32("video", &video));
1865
1866            sp<AMessage> reply;
1867            CHECK(msg->findMessage("reply", &reply));
1868
1869            queueDecoderShutdown(audio, video, reply);
1870            break;
1871        }
1872
1873        default:
1874            TRESPASS();
1875    }
1876}
1877
1878void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
1879    int32_t what;
1880    CHECK(msg->findInt32("what", &what));
1881
1882    switch (what) {
1883        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
1884        {
1885            sp<ABuffer> buffer;
1886            CHECK(msg->findBuffer("buffer", &buffer));
1887
1888            size_t inbandTracks = 0;
1889            if (mSource != NULL) {
1890                inbandTracks = mSource->getTrackCount();
1891            }
1892
1893            sendSubtitleData(buffer, inbandTracks);
1894            break;
1895        }
1896
1897        case NuPlayer::CCDecoder::kWhatTrackAdded:
1898        {
1899            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
1900
1901            break;
1902        }
1903
1904        default:
1905            TRESPASS();
1906    }
1907
1908
1909}
1910
1911void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
1912    int32_t trackIndex;
1913    int64_t timeUs, durationUs;
1914    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
1915    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1916    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
1917
1918    Parcel in;
1919    in.writeInt32(trackIndex + baseIndex);
1920    in.writeInt64(timeUs);
1921    in.writeInt64(durationUs);
1922    in.writeInt32(buffer->size());
1923    in.writeInt32(buffer->size());
1924    in.write(buffer->data(), buffer->size());
1925
1926    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
1927}
1928
1929void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
1930    const void *data;
1931    size_t size = 0;
1932    int64_t timeUs;
1933    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
1934
1935    AString mime;
1936    CHECK(buffer->meta()->findString("mime", &mime));
1937    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
1938
1939    data = buffer->data();
1940    size = buffer->size();
1941
1942    Parcel parcel;
1943    if (size > 0) {
1944        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1945        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
1946        TextDescriptions::getParcelOfDescriptions(
1947                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
1948    }
1949
1950    if ((parcel.dataSize() > 0)) {
1951        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
1952    } else {  // send an empty timed text
1953        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
1954    }
1955}
1956////////////////////////////////////////////////////////////////////////////////
1957
1958sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
1959    sp<MetaData> meta = getFormatMeta(audio);
1960
1961    if (meta == NULL) {
1962        return NULL;
1963    }
1964
1965    sp<AMessage> msg = new AMessage;
1966
1967    if(convertMetaDataToMessage(meta, &msg) == OK) {
1968        return msg;
1969    }
1970    return NULL;
1971}
1972
1973void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
1974    sp<AMessage> notify = dupNotify();
1975    notify->setInt32("what", kWhatFlagsChanged);
1976    notify->setInt32("flags", flags);
1977    notify->post();
1978}
1979
1980void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
1981    sp<AMessage> notify = dupNotify();
1982    notify->setInt32("what", kWhatVideoSizeChanged);
1983    notify->setMessage("format", format);
1984    notify->post();
1985}
1986
1987void NuPlayer::Source::notifyPrepared(status_t err) {
1988    sp<AMessage> notify = dupNotify();
1989    notify->setInt32("what", kWhatPrepared);
1990    notify->setInt32("err", err);
1991    notify->post();
1992}
1993
1994void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
1995    TRESPASS();
1996}
1997
1998}  // namespace android
1999