NuPlayer.cpp revision 8d121d41f5355b78b687f44e8d4aae4de2aa0359 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerDecoder.h"
25#include "NuPlayerDecoderPassThrough.h"
26#include "NuPlayerDriver.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29#include "RTSPSource.h"
30#include "StreamingSource.h"
31#include "GenericSource.h"
32#include "TextDescriptions.h"
33
34#include "ATSParser.h"
35
36#include <media/stagefright/foundation/hexdump.h>
37#include <media/stagefright/foundation/ABuffer.h>
38#include <media/stagefright/foundation/ADebug.h>
39#include <media/stagefright/foundation/AMessage.h>
40#include <media/stagefright/MediaBuffer.h>
41#include <media/stagefright/MediaDefs.h>
42#include <media/stagefright/MediaErrors.h>
43#include <media/stagefright/MetaData.h>
44#include <gui/IGraphicBufferProducer.h>
45
46#include "avc_utils.h"
47
48#include "ESDS.h"
49#include <media/stagefright/Utils.h>
50
51namespace android {
52
53// TODO optimize buffer size for power consumption
54// The offload read buffer size is 32 KB but 24 KB uses less power.
55const size_t NuPlayer::kAggregateBufferSizeBytes = 24 * 1024;
56
57struct NuPlayer::Action : public RefBase {
58    Action() {}
59
60    virtual void execute(NuPlayer *player) = 0;
61
62private:
63    DISALLOW_EVIL_CONSTRUCTORS(Action);
64};
65
66struct NuPlayer::SeekAction : public Action {
67    SeekAction(int64_t seekTimeUs, bool needNotify)
68        : mSeekTimeUs(seekTimeUs),
69          mNeedNotify(needNotify) {
70    }
71
72    virtual void execute(NuPlayer *player) {
73        player->performSeek(mSeekTimeUs, mNeedNotify);
74    }
75
76private:
77    int64_t mSeekTimeUs;
78    bool mNeedNotify;
79
80    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
81};
82
83struct NuPlayer::SetSurfaceAction : public Action {
84    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
85        : mWrapper(wrapper) {
86    }
87
88    virtual void execute(NuPlayer *player) {
89        player->performSetSurface(mWrapper);
90    }
91
92private:
93    sp<NativeWindowWrapper> mWrapper;
94
95    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
96};
97
98struct NuPlayer::ShutdownDecoderAction : public Action {
99    ShutdownDecoderAction(bool audio, bool video)
100        : mAudio(audio),
101          mVideo(video) {
102    }
103
104    virtual void execute(NuPlayer *player) {
105        player->performDecoderShutdown(mAudio, mVideo);
106    }
107
108private:
109    bool mAudio;
110    bool mVideo;
111
112    DISALLOW_EVIL_CONSTRUCTORS(ShutdownDecoderAction);
113};
114
115struct NuPlayer::PostMessageAction : public Action {
116    PostMessageAction(const sp<AMessage> &msg)
117        : mMessage(msg) {
118    }
119
120    virtual void execute(NuPlayer *) {
121        mMessage->post();
122    }
123
124private:
125    sp<AMessage> mMessage;
126
127    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
128};
129
130// Use this if there's no state necessary to save in order to execute
131// the action.
132struct NuPlayer::SimpleAction : public Action {
133    typedef void (NuPlayer::*ActionFunc)();
134
135    SimpleAction(ActionFunc func)
136        : mFunc(func) {
137    }
138
139    virtual void execute(NuPlayer *player) {
140        (player->*mFunc)();
141    }
142
143private:
144    ActionFunc mFunc;
145
146    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
147};
148
149////////////////////////////////////////////////////////////////////////////////
150
151NuPlayer::NuPlayer()
152    : mUIDValid(false),
153      mSourceFlags(0),
154      mCurrentPositionUs(0),
155      mVideoIsAVC(false),
156      mOffloadAudio(false),
157      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
158      mAudioDecoderGeneration(0),
159      mVideoDecoderGeneration(0),
160      mRendererGeneration(0),
161      mAudioEOS(false),
162      mVideoEOS(false),
163      mScanSourcesPending(false),
164      mScanSourcesGeneration(0),
165      mPollDurationGeneration(0),
166      mTimedTextGeneration(0),
167      mTimeDiscontinuityPending(false),
168      mFlushingAudio(NONE),
169      mFlushingVideo(NONE),
170      mSkipRenderingAudioUntilMediaTimeUs(-1ll),
171      mSkipRenderingVideoUntilMediaTimeUs(-1ll),
172      mVideoLateByUs(0ll),
173      mNumFramesTotal(0ll),
174      mNumFramesDropped(0ll),
175      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
176      mStarted(false) {
177    clearFlushComplete();
178}
179
180NuPlayer::~NuPlayer() {
181}
182
183void NuPlayer::setUID(uid_t uid) {
184    mUIDValid = true;
185    mUID = uid;
186}
187
188void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
189    mDriver = driver;
190}
191
192void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
193    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
194
195    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
196
197    msg->setObject("source", new StreamingSource(notify, source));
198    msg->post();
199}
200
201static bool IsHTTPLiveURL(const char *url) {
202    if (!strncasecmp("http://", url, 7)
203            || !strncasecmp("https://", url, 8)
204            || !strncasecmp("file://", url, 7)) {
205        size_t len = strlen(url);
206        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
207            return true;
208        }
209
210        if (strstr(url,"m3u8")) {
211            return true;
212        }
213    }
214
215    return false;
216}
217
218void NuPlayer::setDataSourceAsync(
219        const sp<IMediaHTTPService> &httpService,
220        const char *url,
221        const KeyedVector<String8, String8> *headers) {
222
223    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
224    size_t len = strlen(url);
225
226    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
227
228    sp<Source> source;
229    if (IsHTTPLiveURL(url)) {
230        source = new HTTPLiveSource(notify, httpService, url, headers);
231    } else if (!strncasecmp(url, "rtsp://", 7)) {
232        source = new RTSPSource(
233                notify, httpService, url, headers, mUIDValid, mUID);
234    } else if ((!strncasecmp(url, "http://", 7)
235                || !strncasecmp(url, "https://", 8))
236                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
237                    || strstr(url, ".sdp?"))) {
238        source = new RTSPSource(
239                notify, httpService, url, headers, mUIDValid, mUID, true);
240    } else {
241        sp<GenericSource> genericSource =
242                new GenericSource(notify, mUIDValid, mUID);
243        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
244        // The correct flags will be updated in Source::kWhatFlagsChanged
245        // handler when  GenericSource is prepared.
246
247        status_t err = genericSource->setDataSource(httpService, url, headers);
248
249        if (err == OK) {
250            source = genericSource;
251        } else {
252            ALOGE("Failed to set data source!");
253        }
254    }
255    msg->setObject("source", source);
256    msg->post();
257}
258
259void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
260    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
261
262    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
263
264    sp<GenericSource> source =
265            new GenericSource(notify, mUIDValid, mUID);
266
267    status_t err = source->setDataSource(fd, offset, length);
268
269    if (err != OK) {
270        ALOGE("Failed to set data source!");
271        source = NULL;
272    }
273
274    msg->setObject("source", source);
275    msg->post();
276}
277
278void NuPlayer::prepareAsync() {
279    (new AMessage(kWhatPrepare, id()))->post();
280}
281
282void NuPlayer::setVideoSurfaceTextureAsync(
283        const sp<IGraphicBufferProducer> &bufferProducer) {
284    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id());
285
286    if (bufferProducer == NULL) {
287        msg->setObject("native-window", NULL);
288    } else {
289        msg->setObject(
290                "native-window",
291                new NativeWindowWrapper(
292                    new Surface(bufferProducer, true /* controlledByApp */)));
293    }
294
295    msg->post();
296}
297
298void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
299    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id());
300    msg->setObject("sink", sink);
301    msg->post();
302}
303
304void NuPlayer::start() {
305    (new AMessage(kWhatStart, id()))->post();
306}
307
308void NuPlayer::pause() {
309    (new AMessage(kWhatPause, id()))->post();
310}
311
312void NuPlayer::resume() {
313    (new AMessage(kWhatResume, id()))->post();
314}
315
316void NuPlayer::resetAsync() {
317    if (mSource != NULL) {
318        // During a reset, the data source might be unresponsive already, we need to
319        // disconnect explicitly so that reads exit promptly.
320        // We can't queue the disconnect request to the looper, as it might be
321        // queued behind a stuck read and never gets processed.
322        // Doing a disconnect outside the looper to allows the pending reads to exit
323        // (either successfully or with error).
324        mSource->disconnect();
325    }
326
327    (new AMessage(kWhatReset, id()))->post();
328}
329
330void NuPlayer::seekToAsync(int64_t seekTimeUs, bool needNotify) {
331    sp<AMessage> msg = new AMessage(kWhatSeek, id());
332    msg->setInt64("seekTimeUs", seekTimeUs);
333    msg->setInt32("needNotify", needNotify);
334    msg->post();
335}
336
337
338void NuPlayer::writeTrackInfo(
339        Parcel* reply, const sp<AMessage> format) const {
340    int32_t trackType;
341    CHECK(format->findInt32("type", &trackType));
342
343    AString lang;
344    CHECK(format->findString("language", &lang));
345
346    reply->writeInt32(2); // write something non-zero
347    reply->writeInt32(trackType);
348    reply->writeString16(String16(lang.c_str()));
349
350    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
351        AString mime;
352        CHECK(format->findString("mime", &mime));
353
354        int32_t isAuto, isDefault, isForced;
355        CHECK(format->findInt32("auto", &isAuto));
356        CHECK(format->findInt32("default", &isDefault));
357        CHECK(format->findInt32("forced", &isForced));
358
359        reply->writeString16(String16(mime.c_str()));
360        reply->writeInt32(isAuto);
361        reply->writeInt32(isDefault);
362        reply->writeInt32(isForced);
363    }
364}
365
366void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
367    switch (msg->what()) {
368        case kWhatSetDataSource:
369        {
370            ALOGV("kWhatSetDataSource");
371
372            CHECK(mSource == NULL);
373
374            status_t err = OK;
375            sp<RefBase> obj;
376            CHECK(msg->findObject("source", &obj));
377            if (obj != NULL) {
378                mSource = static_cast<Source *>(obj.get());
379            } else {
380                err = UNKNOWN_ERROR;
381            }
382
383            CHECK(mDriver != NULL);
384            sp<NuPlayerDriver> driver = mDriver.promote();
385            if (driver != NULL) {
386                driver->notifySetDataSourceCompleted(err);
387            }
388            break;
389        }
390
391        case kWhatPrepare:
392        {
393            mSource->prepareAsync();
394            break;
395        }
396
397        case kWhatGetTrackInfo:
398        {
399            uint32_t replyID;
400            CHECK(msg->senderAwaitsResponse(&replyID));
401
402            Parcel* reply;
403            CHECK(msg->findPointer("reply", (void**)&reply));
404
405            size_t inbandTracks = 0;
406            if (mSource != NULL) {
407                inbandTracks = mSource->getTrackCount();
408            }
409
410            size_t ccTracks = 0;
411            if (mCCDecoder != NULL) {
412                ccTracks = mCCDecoder->getTrackCount();
413            }
414
415            // total track count
416            reply->writeInt32(inbandTracks + ccTracks);
417
418            // write inband tracks
419            for (size_t i = 0; i < inbandTracks; ++i) {
420                writeTrackInfo(reply, mSource->getTrackInfo(i));
421            }
422
423            // write CC track
424            for (size_t i = 0; i < ccTracks; ++i) {
425                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
426            }
427
428            sp<AMessage> response = new AMessage;
429            response->postReply(replyID);
430            break;
431        }
432
433        case kWhatGetSelectedTrack:
434        {
435            status_t err = INVALID_OPERATION;
436            if (mSource != NULL) {
437                err = OK;
438
439                int32_t type32;
440                CHECK(msg->findInt32("type", (int32_t*)&type32));
441                media_track_type type = (media_track_type)type32;
442                ssize_t selectedTrack = mSource->getSelectedTrack(type);
443
444                Parcel* reply;
445                CHECK(msg->findPointer("reply", (void**)&reply));
446                reply->writeInt32(selectedTrack);
447            }
448
449            sp<AMessage> response = new AMessage;
450            response->setInt32("err", err);
451
452            uint32_t replyID;
453            CHECK(msg->senderAwaitsResponse(&replyID));
454            response->postReply(replyID);
455            break;
456        }
457
458        case kWhatSelectTrack:
459        {
460            uint32_t replyID;
461            CHECK(msg->senderAwaitsResponse(&replyID));
462
463            size_t trackIndex;
464            int32_t select;
465            CHECK(msg->findSize("trackIndex", &trackIndex));
466            CHECK(msg->findInt32("select", &select));
467
468            status_t err = INVALID_OPERATION;
469
470            size_t inbandTracks = 0;
471            if (mSource != NULL) {
472                inbandTracks = mSource->getTrackCount();
473            }
474            size_t ccTracks = 0;
475            if (mCCDecoder != NULL) {
476                ccTracks = mCCDecoder->getTrackCount();
477            }
478
479            if (trackIndex < inbandTracks) {
480                err = mSource->selectTrack(trackIndex, select);
481
482                if (!select && err == OK) {
483                    int32_t type;
484                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
485                    if (info != NULL
486                            && info->findInt32("type", &type)
487                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
488                        ++mTimedTextGeneration;
489                    }
490                }
491            } else {
492                trackIndex -= inbandTracks;
493
494                if (trackIndex < ccTracks) {
495                    err = mCCDecoder->selectTrack(trackIndex, select);
496                }
497            }
498
499            sp<AMessage> response = new AMessage;
500            response->setInt32("err", err);
501
502            response->postReply(replyID);
503            break;
504        }
505
506        case kWhatPollDuration:
507        {
508            int32_t generation;
509            CHECK(msg->findInt32("generation", &generation));
510
511            if (generation != mPollDurationGeneration) {
512                // stale
513                break;
514            }
515
516            int64_t durationUs;
517            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
518                sp<NuPlayerDriver> driver = mDriver.promote();
519                if (driver != NULL) {
520                    driver->notifyDuration(durationUs);
521                }
522            }
523
524            msg->post(1000000ll);  // poll again in a second.
525            break;
526        }
527
528        case kWhatSetVideoNativeWindow:
529        {
530            ALOGV("kWhatSetVideoNativeWindow");
531
532            mDeferredActions.push_back(
533                    new ShutdownDecoderAction(
534                        false /* audio */, true /* video */));
535
536            sp<RefBase> obj;
537            CHECK(msg->findObject("native-window", &obj));
538
539            mDeferredActions.push_back(
540                    new SetSurfaceAction(
541                        static_cast<NativeWindowWrapper *>(obj.get())));
542
543            if (obj != NULL) {
544                if (mStarted && mSource->getFormat(false /* audio */) != NULL) {
545                    // Issue a seek to refresh the video screen only if started otherwise
546                    // the extractor may not yet be started and will assert.
547                    // If the video decoder is not set (perhaps audio only in this case)
548                    // do not perform a seek as it is not needed.
549                    mDeferredActions.push_back(
550                            new SeekAction(mCurrentPositionUs, false /* needNotify */));
551                }
552
553                // If there is a new surface texture, instantiate decoders
554                // again if possible.
555                mDeferredActions.push_back(
556                        new SimpleAction(&NuPlayer::performScanSources));
557            }
558
559            processDeferredActions();
560            break;
561        }
562
563        case kWhatSetAudioSink:
564        {
565            ALOGV("kWhatSetAudioSink");
566
567            sp<RefBase> obj;
568            CHECK(msg->findObject("sink", &obj));
569
570            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
571            break;
572        }
573
574        case kWhatStart:
575        {
576            ALOGV("kWhatStart");
577
578            mVideoIsAVC = false;
579            mOffloadAudio = false;
580            mAudioEOS = false;
581            mVideoEOS = false;
582            mSkipRenderingAudioUntilMediaTimeUs = -1;
583            mSkipRenderingVideoUntilMediaTimeUs = -1;
584            mVideoLateByUs = 0;
585            mNumFramesTotal = 0;
586            mNumFramesDropped = 0;
587            mStarted = true;
588
589            /* instantiate decoders now for secure playback */
590            if (mSourceFlags & Source::FLAG_SECURE) {
591                if (mNativeWindow != NULL) {
592                    instantiateDecoder(false, &mVideoDecoder);
593                }
594
595                if (mAudioSink != NULL) {
596                    instantiateDecoder(true, &mAudioDecoder);
597                }
598            }
599
600            mSource->start();
601
602            uint32_t flags = 0;
603
604            if (mSource->isRealTime()) {
605                flags |= Renderer::FLAG_REAL_TIME;
606            }
607
608            sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
609            audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
610            if (mAudioSink != NULL) {
611                streamType = mAudioSink->getAudioStreamType();
612            }
613
614            sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
615
616            mOffloadAudio =
617                canOffloadStream(audioMeta, (videoFormat != NULL),
618                                 true /* is_streaming */, streamType);
619            if (mOffloadAudio) {
620                flags |= Renderer::FLAG_OFFLOAD_AUDIO;
621            }
622
623            sp<AMessage> notify = new AMessage(kWhatRendererNotify, id());
624            ++mRendererGeneration;
625            notify->setInt32("generation", mRendererGeneration);
626            mRenderer = new Renderer(mAudioSink, notify, flags);
627
628            mRendererLooper = new ALooper;
629            mRendererLooper->setName("NuPlayerRenderer");
630            mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
631            mRendererLooper->registerHandler(mRenderer);
632
633            sp<MetaData> meta = getFileMeta();
634            int32_t rate;
635            if (meta != NULL
636                    && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) {
637                mRenderer->setVideoFrameRate(rate);
638            }
639
640            postScanSources();
641            break;
642        }
643
644        case kWhatScanSources:
645        {
646            int32_t generation;
647            CHECK(msg->findInt32("generation", &generation));
648            if (generation != mScanSourcesGeneration) {
649                // Drop obsolete msg.
650                break;
651            }
652
653            mScanSourcesPending = false;
654
655            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
656                 mAudioDecoder != NULL, mVideoDecoder != NULL);
657
658            bool mHadAnySourcesBefore =
659                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
660
661            // initialize video before audio because successful initialization of
662            // video may change deep buffer mode of audio.
663            if (mNativeWindow != NULL) {
664                instantiateDecoder(false, &mVideoDecoder);
665            }
666
667            if (mAudioSink != NULL) {
668                if (mOffloadAudio) {
669                    // open audio sink early under offload mode.
670                    sp<AMessage> format = mSource->getFormat(true /*audio*/);
671                    openAudioSink(format, true /*offloadOnly*/);
672                }
673                instantiateDecoder(true, &mAudioDecoder);
674            }
675
676            if (!mHadAnySourcesBefore
677                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
678                // This is the first time we've found anything playable.
679
680                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
681                    schedulePollDuration();
682                }
683            }
684
685            status_t err;
686            if ((err = mSource->feedMoreTSData()) != OK) {
687                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
688                    // We're not currently decoding anything (no audio or
689                    // video tracks found) and we just ran out of input data.
690
691                    if (err == ERROR_END_OF_STREAM) {
692                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
693                    } else {
694                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
695                    }
696                }
697                break;
698            }
699
700            if ((mAudioDecoder == NULL && mAudioSink != NULL)
701                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
702                msg->post(100000ll);
703                mScanSourcesPending = true;
704            }
705            break;
706        }
707
708        case kWhatVideoNotify:
709        case kWhatAudioNotify:
710        {
711            bool audio = msg->what() == kWhatAudioNotify;
712
713            int32_t currentDecoderGeneration =
714                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
715            int32_t requesterGeneration = currentDecoderGeneration - 1;
716            CHECK(msg->findInt32("generation", &requesterGeneration));
717
718            if (requesterGeneration != currentDecoderGeneration) {
719                ALOGV("got message from old %s decoder, generation(%d:%d)",
720                        audio ? "audio" : "video", requesterGeneration,
721                        currentDecoderGeneration);
722                sp<AMessage> reply;
723                if (!(msg->findMessage("reply", &reply))) {
724                    return;
725                }
726
727                reply->setInt32("err", INFO_DISCONTINUITY);
728                reply->post();
729                return;
730            }
731
732            int32_t what;
733            CHECK(msg->findInt32("what", &what));
734
735            if (what == Decoder::kWhatFillThisBuffer) {
736                status_t err = feedDecoderInputData(
737                        audio, msg);
738
739                if (err == -EWOULDBLOCK) {
740                    if (mSource->feedMoreTSData() == OK) {
741                        msg->post(10 * 1000ll);
742                    }
743                }
744            } else if (what == Decoder::kWhatEOS) {
745                int32_t err;
746                CHECK(msg->findInt32("err", &err));
747
748                if (err == ERROR_END_OF_STREAM) {
749                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
750                } else {
751                    ALOGV("got %s decoder EOS w/ error %d",
752                         audio ? "audio" : "video",
753                         err);
754                }
755
756                mRenderer->queueEOS(audio, err);
757            } else if (what == Decoder::kWhatFlushCompleted) {
758                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
759
760                handleFlushComplete(audio, true /* isDecoder */);
761                finishFlushIfPossible();
762            } else if (what == Decoder::kWhatOutputFormatChanged) {
763                sp<AMessage> format;
764                CHECK(msg->findMessage("format", &format));
765
766                if (audio) {
767                    openAudioSink(format, false /*offloadOnly*/);
768                } else {
769                    // video
770                    sp<AMessage> inputFormat =
771                            mSource->getFormat(false /* audio */);
772
773                    updateVideoSize(inputFormat, format);
774                }
775            } else if (what == Decoder::kWhatShutdownCompleted) {
776                ALOGV("%s shutdown completed", audio ? "audio" : "video");
777                if (audio) {
778                    mAudioDecoder.clear();
779                    ++mAudioDecoderGeneration;
780
781                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
782                    mFlushingAudio = SHUT_DOWN;
783                } else {
784                    mVideoDecoder.clear();
785                    ++mVideoDecoderGeneration;
786
787                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
788                    mFlushingVideo = SHUT_DOWN;
789                }
790
791                finishFlushIfPossible();
792            } else if (what == Decoder::kWhatError) {
793                status_t err;
794                if (!msg->findInt32("err", &err) || err == OK) {
795                    err = UNKNOWN_ERROR;
796                }
797
798                // Decoder errors can be due to Source (e.g. from streaming),
799                // or from decoding corrupted bitstreams, or from other decoder
800                // MediaCodec operations (e.g. from an ongoing reset or seek).
801                //
802                // We try to gracefully shut down the affected decoder if possible,
803                // rather than trying to force the shutdown with something
804                // similar to performReset(). This method can lead to a hang
805                // if MediaCodec functions block after an error, but they should
806                // typically return INVALID_OPERATION instead of blocking.
807
808                FlushStatus *flushing = audio ? &mFlushingAudio : &mFlushingVideo;
809                ALOGE("received error(%#x) from %s decoder, flushing(%d), now shutting down",
810                        err, audio ? "audio" : "video", *flushing);
811
812                switch (*flushing) {
813                    case NONE:
814                        mDeferredActions.push_back(
815                                new ShutdownDecoderAction(audio, !audio /* video */));
816                        processDeferredActions();
817                        break;
818                    case FLUSHING_DECODER:
819                        *flushing = FLUSHING_DECODER_SHUTDOWN; // initiate shutdown after flush.
820                        break; // Wait for flush to complete.
821                    case FLUSHING_DECODER_SHUTDOWN:
822                        break; // Wait for flush to complete.
823                    case SHUTTING_DOWN_DECODER:
824                        break; // Wait for shutdown to complete.
825                    case FLUSHED:
826                        // Widevine source reads must stop before releasing the video decoder.
827                        if (!audio && mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
828                            mSource->stop();
829                        }
830                        getDecoder(audio)->initiateShutdown(); // In the middle of a seek.
831                        *flushing = SHUTTING_DOWN_DECODER;     // Shut down.
832                        break;
833                    case SHUT_DOWN:
834                        finishFlushIfPossible();  // Should not occur.
835                        break;                    // Finish anyways.
836                }
837                notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
838            } else if (what == Decoder::kWhatDrainThisBuffer) {
839                renderBuffer(audio, msg);
840            } else {
841                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
842                      what,
843                      what >> 24,
844                      (what >> 16) & 0xff,
845                      (what >> 8) & 0xff,
846                      what & 0xff);
847            }
848
849            break;
850        }
851
852        case kWhatRendererNotify:
853        {
854            int32_t requesterGeneration = mRendererGeneration - 1;
855            CHECK(msg->findInt32("generation", &requesterGeneration));
856            if (requesterGeneration != mRendererGeneration) {
857                ALOGV("got message from old renderer, generation(%d:%d)",
858                        requesterGeneration, mRendererGeneration);
859                return;
860            }
861
862            int32_t what;
863            CHECK(msg->findInt32("what", &what));
864
865            if (what == Renderer::kWhatEOS) {
866                int32_t audio;
867                CHECK(msg->findInt32("audio", &audio));
868
869                int32_t finalResult;
870                CHECK(msg->findInt32("finalResult", &finalResult));
871
872                if (audio) {
873                    mAudioEOS = true;
874                } else {
875                    mVideoEOS = true;
876                }
877
878                if (finalResult == ERROR_END_OF_STREAM) {
879                    ALOGV("reached %s EOS", audio ? "audio" : "video");
880                } else {
881                    ALOGE("%s track encountered an error (%d)",
882                         audio ? "audio" : "video", finalResult);
883
884                    notifyListener(
885                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
886                }
887
888                if ((mAudioEOS || mAudioDecoder == NULL)
889                        && (mVideoEOS || mVideoDecoder == NULL)) {
890                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
891                }
892            } else if (what == Renderer::kWhatPosition) {
893                int64_t positionUs;
894                CHECK(msg->findInt64("positionUs", &positionUs));
895                mCurrentPositionUs = positionUs;
896
897                CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs));
898
899                if (mDriver != NULL) {
900                    sp<NuPlayerDriver> driver = mDriver.promote();
901                    if (driver != NULL) {
902                        driver->notifyPosition(positionUs);
903
904                        driver->notifyFrameStats(
905                                mNumFramesTotal, mNumFramesDropped);
906                    }
907                }
908            } else if (what == Renderer::kWhatFlushComplete) {
909                int32_t audio;
910                CHECK(msg->findInt32("audio", &audio));
911
912                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
913                handleFlushComplete(audio, false /* isDecoder */);
914                finishFlushIfPossible();
915            } else if (what == Renderer::kWhatVideoRenderingStart) {
916                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
917            } else if (what == Renderer::kWhatMediaRenderingStart) {
918                ALOGV("media rendering started");
919                notifyListener(MEDIA_STARTED, 0, 0);
920            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
921                ALOGV("Tear down audio offload, fall back to s/w path");
922                int64_t positionUs;
923                CHECK(msg->findInt64("positionUs", &positionUs));
924                int32_t reason;
925                CHECK(msg->findInt32("reason", &reason));
926                closeAudioSink();
927                mAudioDecoder.clear();
928                ++mAudioDecoderGeneration;
929                mRenderer->flush(true /* audio */);
930                if (mVideoDecoder != NULL) {
931                    mRenderer->flush(false /* audio */);
932                }
933                mRenderer->signalDisableOffloadAudio();
934                mOffloadAudio = false;
935
936                performSeek(positionUs, false /* needNotify */);
937                if (reason == Renderer::kDueToError) {
938                    instantiateDecoder(true /* audio */, &mAudioDecoder);
939                }
940            }
941            break;
942        }
943
944        case kWhatMoreDataQueued:
945        {
946            break;
947        }
948
949        case kWhatReset:
950        {
951            ALOGV("kWhatReset");
952
953            mDeferredActions.push_back(
954                    new ShutdownDecoderAction(
955                        true /* audio */, true /* video */));
956
957            mDeferredActions.push_back(
958                    new SimpleAction(&NuPlayer::performReset));
959
960            processDeferredActions();
961            break;
962        }
963
964        case kWhatSeek:
965        {
966            int64_t seekTimeUs;
967            int32_t needNotify;
968            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
969            CHECK(msg->findInt32("needNotify", &needNotify));
970
971            ALOGV("kWhatSeek seekTimeUs=%lld us, needNotify=%d",
972                    seekTimeUs, needNotify);
973
974            mDeferredActions.push_back(
975                    new SimpleAction(&NuPlayer::performDecoderFlush));
976
977            mDeferredActions.push_back(
978                    new SeekAction(seekTimeUs, needNotify));
979
980            processDeferredActions();
981            break;
982        }
983
984        case kWhatPause:
985        {
986            if (mSource != NULL) {
987                mSource->pause();
988            } else {
989                ALOGW("pause called when source is gone or not set");
990            }
991            if (mRenderer != NULL) {
992                mRenderer->pause();
993            } else {
994                ALOGW("pause called when renderer is gone or not set");
995            }
996            break;
997        }
998
999        case kWhatResume:
1000        {
1001            if (mSource != NULL) {
1002                mSource->resume();
1003            } else {
1004                ALOGW("resume called when source is gone or not set");
1005            }
1006            // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if
1007            // needed.
1008            if (audioDecoderStillNeeded() && mAudioDecoder == NULL) {
1009                instantiateDecoder(true /* audio */, &mAudioDecoder);
1010            }
1011            if (mRenderer != NULL) {
1012                mRenderer->resume();
1013            } else {
1014                ALOGW("resume called when renderer is gone or not set");
1015            }
1016            break;
1017        }
1018
1019        case kWhatSourceNotify:
1020        {
1021            onSourceNotify(msg);
1022            break;
1023        }
1024
1025        case kWhatClosedCaptionNotify:
1026        {
1027            onClosedCaptionNotify(msg);
1028            break;
1029        }
1030
1031        default:
1032            TRESPASS();
1033            break;
1034    }
1035}
1036
1037bool NuPlayer::audioDecoderStillNeeded() {
1038    // Audio decoder is no longer needed if it's in shut/shutting down status.
1039    return ((mFlushingAudio != SHUT_DOWN) && (mFlushingAudio != SHUTTING_DOWN_DECODER));
1040}
1041
1042void NuPlayer::handleFlushComplete(bool audio, bool isDecoder) {
1043    // We wait for both the decoder flush and the renderer flush to complete
1044    // before entering either the FLUSHED or the SHUTTING_DOWN_DECODER state.
1045
1046    mFlushComplete[audio][isDecoder] = true;
1047    if (!mFlushComplete[audio][!isDecoder]) {
1048        return;
1049    }
1050
1051    FlushStatus *state = audio ? &mFlushingAudio : &mFlushingVideo;
1052    switch (*state) {
1053        case FLUSHING_DECODER:
1054        {
1055            *state = FLUSHED;
1056
1057            if (!audio) {
1058                mVideoLateByUs = 0;
1059            }
1060            break;
1061        }
1062
1063        case FLUSHING_DECODER_SHUTDOWN:
1064        {
1065            *state = SHUTTING_DOWN_DECODER;
1066
1067            ALOGV("initiating %s decoder shutdown", audio ? "audio" : "video");
1068            if (!audio) {
1069                mVideoLateByUs = 0;
1070                // Widevine source reads must stop before releasing the video decoder.
1071                if (mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
1072                    mSource->stop();
1073                }
1074            }
1075            getDecoder(audio)->initiateShutdown();
1076            break;
1077        }
1078
1079        default:
1080            // decoder flush completes only occur in a flushing state.
1081            LOG_ALWAYS_FATAL_IF(isDecoder, "decoder flush in invalid state %d", *state);
1082            break;
1083    }
1084}
1085
1086void NuPlayer::finishFlushIfPossible() {
1087    if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED
1088            && mFlushingAudio != SHUT_DOWN) {
1089        return;
1090    }
1091
1092    if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED
1093            && mFlushingVideo != SHUT_DOWN) {
1094        return;
1095    }
1096
1097    ALOGV("both audio and video are flushed now.");
1098
1099    mPendingAudioAccessUnit.clear();
1100    mAggregateBuffer.clear();
1101
1102    if (mTimeDiscontinuityPending) {
1103        mRenderer->signalTimeDiscontinuity();
1104        mTimeDiscontinuityPending = false;
1105    }
1106
1107    if (mAudioDecoder != NULL && mFlushingAudio == FLUSHED) {
1108        mAudioDecoder->signalResume();
1109    }
1110
1111    if (mVideoDecoder != NULL && mFlushingVideo == FLUSHED) {
1112        mVideoDecoder->signalResume();
1113    }
1114
1115    mFlushingAudio = NONE;
1116    mFlushingVideo = NONE;
1117
1118    clearFlushComplete();
1119
1120    processDeferredActions();
1121}
1122
1123void NuPlayer::postScanSources() {
1124    if (mScanSourcesPending) {
1125        return;
1126    }
1127
1128    sp<AMessage> msg = new AMessage(kWhatScanSources, id());
1129    msg->setInt32("generation", mScanSourcesGeneration);
1130    msg->post();
1131
1132    mScanSourcesPending = true;
1133}
1134
1135void NuPlayer::openAudioSink(const sp<AMessage> &format, bool offloadOnly) {
1136    ALOGV("openAudioSink: offloadOnly(%d) mOffloadAudio(%d)",
1137            offloadOnly, mOffloadAudio);
1138    bool audioSinkChanged = false;
1139
1140    int32_t numChannels;
1141    CHECK(format->findInt32("channel-count", &numChannels));
1142
1143    int32_t channelMask;
1144    if (!format->findInt32("channel-mask", &channelMask)) {
1145        // signal to the AudioSink to derive the mask from count.
1146        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1147    }
1148
1149    int32_t sampleRate;
1150    CHECK(format->findInt32("sample-rate", &sampleRate));
1151
1152    uint32_t flags;
1153    int64_t durationUs;
1154    // FIXME: we should handle the case where the video decoder
1155    // is created after we receive the format change indication.
1156    // Current code will just make that we select deep buffer
1157    // with video which should not be a problem as it should
1158    // not prevent from keeping A/V sync.
1159    if (mVideoDecoder == NULL &&
1160            mSource->getDuration(&durationUs) == OK &&
1161            durationUs
1162                > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
1163        flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1164    } else {
1165        flags = AUDIO_OUTPUT_FLAG_NONE;
1166    }
1167
1168    if (mOffloadAudio) {
1169        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1170        AString mime;
1171        CHECK(format->findString("mime", &mime));
1172        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1173
1174        if (err != OK) {
1175            ALOGE("Couldn't map mime \"%s\" to a valid "
1176                    "audio_format", mime.c_str());
1177            mOffloadAudio = false;
1178        } else {
1179            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1180                    mime.c_str(), audioFormat);
1181
1182            int avgBitRate = -1;
1183            format->findInt32("bit-rate", &avgBitRate);
1184
1185            int32_t aacProfile = -1;
1186            if (audioFormat == AUDIO_FORMAT_AAC
1187                    && format->findInt32("aac-profile", &aacProfile)) {
1188                // Redefine AAC format as per aac profile
1189                mapAACProfileToAudioFormat(
1190                        audioFormat,
1191                        aacProfile);
1192            }
1193
1194            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1195            offloadInfo.duration_us = -1;
1196            format->findInt64(
1197                    "durationUs", &offloadInfo.duration_us);
1198            offloadInfo.sample_rate = sampleRate;
1199            offloadInfo.channel_mask = channelMask;
1200            offloadInfo.format = audioFormat;
1201            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1202            offloadInfo.bit_rate = avgBitRate;
1203            offloadInfo.has_video = (mVideoDecoder != NULL);
1204            offloadInfo.is_streaming = true;
1205
1206            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1207                ALOGV("openAudioSink: no change in offload mode");
1208                return;  // no change from previous configuration, everything ok.
1209            }
1210            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1211            flags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1212            flags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1213            audioSinkChanged = true;
1214            mAudioSink->close();
1215            err = mAudioSink->open(
1216                    sampleRate,
1217                    numChannels,
1218                    (audio_channel_mask_t)channelMask,
1219                    audioFormat,
1220                    8 /* bufferCount */,
1221                    &NuPlayer::Renderer::AudioSinkCallback,
1222                    mRenderer.get(),
1223                    (audio_output_flags_t)flags,
1224                    &offloadInfo);
1225
1226            if (err == OK) {
1227                // If the playback is offloaded to h/w, we pass
1228                // the HAL some metadata information.
1229                // We don't want to do this for PCM because it
1230                // will be going through the AudioFlinger mixer
1231                // before reaching the hardware.
1232                sp<MetaData> audioMeta =
1233                        mSource->getFormatMeta(true /* audio */);
1234                sendMetaDataToHal(mAudioSink, audioMeta);
1235                mCurrentOffloadInfo = offloadInfo;
1236                err = mAudioSink->start();
1237                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1238            }
1239            if (err != OK) {
1240                // Clean up, fall back to non offload mode.
1241                mAudioSink->close();
1242                mRenderer->signalDisableOffloadAudio();
1243                mOffloadAudio = false;
1244                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1245                ALOGV("openAudioSink: offload failed");
1246            }
1247        }
1248    }
1249    if (!offloadOnly && !mOffloadAudio) {
1250        flags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1251        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1252
1253        audioSinkChanged = true;
1254        mAudioSink->close();
1255        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1256        CHECK_EQ(mAudioSink->open(
1257                    sampleRate,
1258                    numChannels,
1259                    (audio_channel_mask_t)channelMask,
1260                    AUDIO_FORMAT_PCM_16_BIT,
1261                    8 /* bufferCount */,
1262                    NULL,
1263                    NULL,
1264                    (audio_output_flags_t)flags),
1265                 (status_t)OK);
1266        mAudioSink->start();
1267    }
1268    if (audioSinkChanged) {
1269        mRenderer->signalAudioSinkChanged();
1270    }
1271}
1272
1273void NuPlayer::closeAudioSink() {
1274    mAudioSink->close();
1275    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1276}
1277
1278status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
1279    if (*decoder != NULL) {
1280        return OK;
1281    }
1282
1283    sp<AMessage> format = mSource->getFormat(audio);
1284
1285    if (format == NULL) {
1286        return -EWOULDBLOCK;
1287    }
1288
1289    if (!audio) {
1290        AString mime;
1291        CHECK(format->findString("mime", &mime));
1292        mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
1293
1294        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
1295        mCCDecoder = new CCDecoder(ccNotify);
1296
1297        if (mSourceFlags & Source::FLAG_SECURE) {
1298            format->setInt32("secure", true);
1299        }
1300    }
1301
1302    if (audio) {
1303        sp<AMessage> notify = new AMessage(kWhatAudioNotify, id());
1304        ++mAudioDecoderGeneration;
1305        notify->setInt32("generation", mAudioDecoderGeneration);
1306
1307        if (mOffloadAudio) {
1308            *decoder = new DecoderPassThrough(notify);
1309        } else {
1310            *decoder = new Decoder(notify);
1311        }
1312    } else {
1313        sp<AMessage> notify = new AMessage(kWhatVideoNotify, id());
1314        ++mVideoDecoderGeneration;
1315        notify->setInt32("generation", mVideoDecoderGeneration);
1316
1317        *decoder = new Decoder(notify, mNativeWindow);
1318    }
1319    (*decoder)->init();
1320    (*decoder)->configure(format);
1321
1322    // allocate buffers to decrypt widevine source buffers
1323    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1324        Vector<sp<ABuffer> > inputBufs;
1325        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1326
1327        Vector<MediaBuffer *> mediaBufs;
1328        for (size_t i = 0; i < inputBufs.size(); i++) {
1329            const sp<ABuffer> &buffer = inputBufs[i];
1330            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1331            mediaBufs.push(mbuf);
1332        }
1333
1334        status_t err = mSource->setBuffers(audio, mediaBufs);
1335        if (err != OK) {
1336            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1337                mediaBufs[i]->release();
1338            }
1339            mediaBufs.clear();
1340            ALOGE("Secure source didn't support secure mediaBufs.");
1341            return err;
1342        }
1343    }
1344    return OK;
1345}
1346
1347status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
1348    sp<AMessage> reply;
1349    CHECK(msg->findMessage("reply", &reply));
1350
1351    if ((audio && mFlushingAudio != NONE)
1352            || (!audio && mFlushingVideo != NONE)
1353            || mSource == NULL) {
1354        reply->setInt32("err", INFO_DISCONTINUITY);
1355        reply->post();
1356        return OK;
1357    }
1358
1359    sp<ABuffer> accessUnit;
1360
1361    // Aggregate smaller buffers into a larger buffer.
1362    // The goal is to reduce power consumption.
1363    // Note this will not work if the decoder requires one frame per buffer.
1364    bool doBufferAggregation = (audio && mOffloadAudio);
1365    bool needMoreData = false;
1366
1367    bool dropAccessUnit;
1368    do {
1369        status_t err;
1370        // Did we save an accessUnit earlier because of a discontinuity?
1371        if (audio && (mPendingAudioAccessUnit != NULL)) {
1372            accessUnit = mPendingAudioAccessUnit;
1373            mPendingAudioAccessUnit.clear();
1374            err = mPendingAudioErr;
1375            ALOGV("feedDecoderInputData() use mPendingAudioAccessUnit");
1376        } else {
1377            err = mSource->dequeueAccessUnit(audio, &accessUnit);
1378        }
1379
1380        if (err == -EWOULDBLOCK) {
1381            return err;
1382        } else if (err != OK) {
1383            if (err == INFO_DISCONTINUITY) {
1384                if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1385                    // We already have some data so save this for later.
1386                    mPendingAudioErr = err;
1387                    mPendingAudioAccessUnit = accessUnit;
1388                    accessUnit.clear();
1389                    ALOGD("feedDecoderInputData() save discontinuity for later");
1390                    break;
1391                }
1392                int32_t type;
1393                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
1394
1395                bool formatChange =
1396                    (audio &&
1397                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
1398                    || (!audio &&
1399                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
1400
1401                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
1402
1403                ALOGI("%s discontinuity (formatChange=%d, time=%d)",
1404                     audio ? "audio" : "video", formatChange, timeChange);
1405
1406                if (audio) {
1407                    mSkipRenderingAudioUntilMediaTimeUs = -1;
1408                } else {
1409                    mSkipRenderingVideoUntilMediaTimeUs = -1;
1410                }
1411
1412                if (timeChange) {
1413                    sp<AMessage> extra;
1414                    if (accessUnit->meta()->findMessage("extra", &extra)
1415                            && extra != NULL) {
1416                        int64_t resumeAtMediaTimeUs;
1417                        if (extra->findInt64(
1418                                    "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
1419                            ALOGI("suppressing rendering of %s until %lld us",
1420                                    audio ? "audio" : "video", resumeAtMediaTimeUs);
1421
1422                            if (audio) {
1423                                mSkipRenderingAudioUntilMediaTimeUs =
1424                                    resumeAtMediaTimeUs;
1425                            } else {
1426                                mSkipRenderingVideoUntilMediaTimeUs =
1427                                    resumeAtMediaTimeUs;
1428                            }
1429                        }
1430                    }
1431                }
1432
1433                mTimeDiscontinuityPending =
1434                    mTimeDiscontinuityPending || timeChange;
1435
1436                bool seamlessFormatChange = false;
1437                sp<AMessage> newFormat = mSource->getFormat(audio);
1438                if (formatChange) {
1439                    seamlessFormatChange =
1440                        getDecoder(audio)->supportsSeamlessFormatChange(newFormat);
1441                    // treat seamless format change separately
1442                    formatChange = !seamlessFormatChange;
1443                }
1444                bool shutdownOrFlush = formatChange || timeChange;
1445
1446                // We want to queue up scan-sources only once per discontinuity.
1447                // We control this by doing it only if neither audio nor video are
1448                // flushing or shutting down.  (After handling 1st discontinuity, one
1449                // of the flushing states will not be NONE.)
1450                // No need to scan sources if this discontinuity does not result
1451                // in a flush or shutdown, as the flushing state will stay NONE.
1452                if (mFlushingAudio == NONE && mFlushingVideo == NONE &&
1453                        shutdownOrFlush) {
1454                    // And we'll resume scanning sources once we're done
1455                    // flushing.
1456                    mDeferredActions.push_front(
1457                            new SimpleAction(
1458                                &NuPlayer::performScanSources));
1459                }
1460
1461                if (formatChange /* not seamless */) {
1462                    // must change decoder
1463                    flushDecoder(audio, /* needShutdown = */ true);
1464                } else if (timeChange) {
1465                    // need to flush
1466                    flushDecoder(audio, /* needShutdown = */ false, newFormat);
1467                    err = OK;
1468                } else if (seamlessFormatChange) {
1469                    // reuse existing decoder and don't flush
1470                    updateDecoderFormatWithoutFlush(audio, newFormat);
1471                    err = OK;
1472                } else {
1473                    // This stream is unaffected by the discontinuity
1474                    return -EWOULDBLOCK;
1475                }
1476            }
1477
1478            reply->setInt32("err", err);
1479            reply->post();
1480            return OK;
1481        }
1482
1483        if (!audio) {
1484            ++mNumFramesTotal;
1485        }
1486
1487        dropAccessUnit = false;
1488        if (!audio
1489                && !(mSourceFlags & Source::FLAG_SECURE)
1490                && mVideoLateByUs > 100000ll
1491                && mVideoIsAVC
1492                && !IsAVCReferenceFrame(accessUnit)) {
1493            dropAccessUnit = true;
1494            ++mNumFramesDropped;
1495        }
1496
1497        size_t smallSize = accessUnit->size();
1498        needMoreData = false;
1499        if (doBufferAggregation && (mAggregateBuffer == NULL)
1500                // Don't bother if only room for a few small buffers.
1501                && (smallSize < (kAggregateBufferSizeBytes / 3))) {
1502            // Create a larger buffer for combining smaller buffers from the extractor.
1503            mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes);
1504            mAggregateBuffer->setRange(0, 0); // start empty
1505        }
1506
1507        if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1508            int64_t timeUs;
1509            int64_t dummy;
1510            bool smallTimestampValid = accessUnit->meta()->findInt64("timeUs", &timeUs);
1511            bool bigTimestampValid = mAggregateBuffer->meta()->findInt64("timeUs", &dummy);
1512            // Will the smaller buffer fit?
1513            size_t bigSize = mAggregateBuffer->size();
1514            size_t roomLeft = mAggregateBuffer->capacity() - bigSize;
1515            // Should we save this small buffer for the next big buffer?
1516            // If the first small buffer did not have a timestamp then save
1517            // any buffer that does have a timestamp until the next big buffer.
1518            if ((smallSize > roomLeft)
1519                || (!bigTimestampValid && (bigSize > 0) && smallTimestampValid)) {
1520                mPendingAudioErr = err;
1521                mPendingAudioAccessUnit = accessUnit;
1522                accessUnit.clear();
1523            } else {
1524                // Grab time from first small buffer if available.
1525                if ((bigSize == 0) && smallTimestampValid) {
1526                    mAggregateBuffer->meta()->setInt64("timeUs", timeUs);
1527                }
1528                // Append small buffer to the bigger buffer.
1529                memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize);
1530                bigSize += smallSize;
1531                mAggregateBuffer->setRange(0, bigSize);
1532
1533                // Keep looping until we run out of room in the mAggregateBuffer.
1534                needMoreData = true;
1535
1536                ALOGV("feedDecoderInputData() smallSize = %zu, bigSize = %zu, capacity = %zu",
1537                        smallSize, bigSize, mAggregateBuffer->capacity());
1538            }
1539        }
1540    } while (dropAccessUnit || needMoreData);
1541
1542    // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
1543
1544#if 0
1545    int64_t mediaTimeUs;
1546    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
1547    ALOGV("feeding %s input buffer at media time %.2f secs",
1548         audio ? "audio" : "video",
1549         mediaTimeUs / 1E6);
1550#endif
1551
1552    if (!audio) {
1553        mCCDecoder->decode(accessUnit);
1554    }
1555
1556    if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1557        ALOGV("feedDecoderInputData() reply with aggregated buffer, %zu",
1558                mAggregateBuffer->size());
1559        reply->setBuffer("buffer", mAggregateBuffer);
1560        mAggregateBuffer.clear();
1561    } else {
1562        reply->setBuffer("buffer", accessUnit);
1563    }
1564
1565    reply->post();
1566
1567    return OK;
1568}
1569
1570void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
1571    // ALOGV("renderBuffer %s", audio ? "audio" : "video");
1572
1573    sp<AMessage> reply;
1574    CHECK(msg->findMessage("reply", &reply));
1575
1576    if ((audio && mFlushingAudio != NONE)
1577            || (!audio && mFlushingVideo != NONE)) {
1578        // We're currently attempting to flush the decoder, in order
1579        // to complete this, the decoder wants all its buffers back,
1580        // so we don't want any output buffers it sent us (from before
1581        // we initiated the flush) to be stuck in the renderer's queue.
1582
1583        ALOGV("we're still flushing the %s decoder, sending its output buffer"
1584             " right back.", audio ? "audio" : "video");
1585
1586        reply->post();
1587        return;
1588    }
1589
1590    sp<ABuffer> buffer;
1591    CHECK(msg->findBuffer("buffer", &buffer));
1592
1593    int64_t mediaTimeUs;
1594    CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));
1595
1596    int64_t &skipUntilMediaTimeUs =
1597        audio
1598            ? mSkipRenderingAudioUntilMediaTimeUs
1599            : mSkipRenderingVideoUntilMediaTimeUs;
1600
1601    if (skipUntilMediaTimeUs >= 0) {
1602
1603        if (mediaTimeUs < skipUntilMediaTimeUs) {
1604            ALOGV("dropping %s buffer at time %lld as requested.",
1605                 audio ? "audio" : "video",
1606                 mediaTimeUs);
1607
1608            reply->post();
1609            return;
1610        }
1611
1612        skipUntilMediaTimeUs = -1;
1613    }
1614
1615    if (!audio && mCCDecoder->isSelected()) {
1616        mCCDecoder->display(mediaTimeUs);
1617    }
1618
1619    mRenderer->queueBuffer(audio, buffer, reply);
1620}
1621
1622void NuPlayer::updateVideoSize(
1623        const sp<AMessage> &inputFormat,
1624        const sp<AMessage> &outputFormat) {
1625    if (inputFormat == NULL) {
1626        ALOGW("Unknown video size, reporting 0x0!");
1627        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1628        return;
1629    }
1630
1631    int32_t displayWidth, displayHeight;
1632    int32_t cropLeft, cropTop, cropRight, cropBottom;
1633
1634    if (outputFormat != NULL) {
1635        int32_t width, height;
1636        CHECK(outputFormat->findInt32("width", &width));
1637        CHECK(outputFormat->findInt32("height", &height));
1638
1639        int32_t cropLeft, cropTop, cropRight, cropBottom;
1640        CHECK(outputFormat->findRect(
1641                    "crop",
1642                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1643
1644        displayWidth = cropRight - cropLeft + 1;
1645        displayHeight = cropBottom - cropTop + 1;
1646
1647        ALOGV("Video output format changed to %d x %d "
1648             "(crop: %d x %d @ (%d, %d))",
1649             width, height,
1650             displayWidth,
1651             displayHeight,
1652             cropLeft, cropTop);
1653    } else {
1654        CHECK(inputFormat->findInt32("width", &displayWidth));
1655        CHECK(inputFormat->findInt32("height", &displayHeight));
1656
1657        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1658    }
1659
1660    // Take into account sample aspect ratio if necessary:
1661    int32_t sarWidth, sarHeight;
1662    if (inputFormat->findInt32("sar-width", &sarWidth)
1663            && inputFormat->findInt32("sar-height", &sarHeight)) {
1664        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1665
1666        displayWidth = (displayWidth * sarWidth) / sarHeight;
1667
1668        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1669    }
1670
1671    int32_t rotationDegrees;
1672    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1673        rotationDegrees = 0;
1674    }
1675
1676    if (rotationDegrees == 90 || rotationDegrees == 270) {
1677        int32_t tmp = displayWidth;
1678        displayWidth = displayHeight;
1679        displayHeight = tmp;
1680    }
1681
1682    notifyListener(
1683            MEDIA_SET_VIDEO_SIZE,
1684            displayWidth,
1685            displayHeight);
1686}
1687
1688void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1689    if (mDriver == NULL) {
1690        return;
1691    }
1692
1693    sp<NuPlayerDriver> driver = mDriver.promote();
1694
1695    if (driver == NULL) {
1696        return;
1697    }
1698
1699    driver->notifyListener(msg, ext1, ext2, in);
1700}
1701
1702void NuPlayer::flushDecoder(
1703        bool audio, bool needShutdown, const sp<AMessage> &newFormat) {
1704    ALOGV("[%s] flushDecoder needShutdown=%d",
1705          audio ? "audio" : "video", needShutdown);
1706
1707    const sp<Decoder> &decoder = getDecoder(audio);
1708    if (decoder == NULL) {
1709        ALOGI("flushDecoder %s without decoder present",
1710             audio ? "audio" : "video");
1711        return;
1712    }
1713
1714    // Make sure we don't continue to scan sources until we finish flushing.
1715    ++mScanSourcesGeneration;
1716    mScanSourcesPending = false;
1717
1718    decoder->signalFlush(newFormat);
1719    mRenderer->flush(audio);
1720
1721    FlushStatus newStatus =
1722        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1723
1724    mFlushComplete[audio][false /* isDecoder */] = false;
1725    mFlushComplete[audio][true /* isDecoder */] = false;
1726    if (audio) {
1727        ALOGE_IF(mFlushingAudio != NONE,
1728                "audio flushDecoder() is called in state %d", mFlushingAudio);
1729        mFlushingAudio = newStatus;
1730    } else {
1731        ALOGE_IF(mFlushingVideo != NONE,
1732                "video flushDecoder() is called in state %d", mFlushingVideo);
1733        mFlushingVideo = newStatus;
1734
1735        if (mCCDecoder != NULL) {
1736            mCCDecoder->flush();
1737        }
1738    }
1739}
1740
1741void NuPlayer::updateDecoderFormatWithoutFlush(
1742        bool audio, const sp<AMessage> &format) {
1743    ALOGV("[%s] updateDecoderFormatWithoutFlush", audio ? "audio" : "video");
1744
1745    const sp<Decoder> &decoder = getDecoder(audio);
1746    if (decoder == NULL) {
1747        ALOGI("updateDecoderFormatWithoutFlush %s without decoder present",
1748             audio ? "audio" : "video");
1749        return;
1750    }
1751
1752    decoder->signalUpdateFormat(format);
1753}
1754
1755void NuPlayer::queueDecoderShutdown(
1756        bool audio, bool video, const sp<AMessage> &reply) {
1757    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1758
1759    mDeferredActions.push_back(
1760            new ShutdownDecoderAction(audio, video));
1761
1762    mDeferredActions.push_back(
1763            new SimpleAction(&NuPlayer::performScanSources));
1764
1765    mDeferredActions.push_back(new PostMessageAction(reply));
1766
1767    processDeferredActions();
1768}
1769
1770status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1771    mVideoScalingMode = mode;
1772    if (mNativeWindow != NULL) {
1773        status_t ret = native_window_set_scaling_mode(
1774                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1775        if (ret != OK) {
1776            ALOGE("Failed to set scaling mode (%d): %s",
1777                -ret, strerror(-ret));
1778            return ret;
1779        }
1780    }
1781    return OK;
1782}
1783
1784status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1785    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id());
1786    msg->setPointer("reply", reply);
1787
1788    sp<AMessage> response;
1789    status_t err = msg->postAndAwaitResponse(&response);
1790    return err;
1791}
1792
1793status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1794    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id());
1795    msg->setPointer("reply", reply);
1796    msg->setInt32("type", type);
1797
1798    sp<AMessage> response;
1799    status_t err = msg->postAndAwaitResponse(&response);
1800    if (err == OK && response != NULL) {
1801        CHECK(response->findInt32("err", &err));
1802    }
1803    return err;
1804}
1805
1806status_t NuPlayer::selectTrack(size_t trackIndex, bool select) {
1807    sp<AMessage> msg = new AMessage(kWhatSelectTrack, id());
1808    msg->setSize("trackIndex", trackIndex);
1809    msg->setInt32("select", select);
1810
1811    sp<AMessage> response;
1812    status_t err = msg->postAndAwaitResponse(&response);
1813
1814    if (err != OK) {
1815        return err;
1816    }
1817
1818    if (!response->findInt32("err", &err)) {
1819        err = OK;
1820    }
1821
1822    return err;
1823}
1824
1825sp<MetaData> NuPlayer::getFileMeta() {
1826    return mSource->getFileFormatMeta();
1827}
1828
1829void NuPlayer::schedulePollDuration() {
1830    sp<AMessage> msg = new AMessage(kWhatPollDuration, id());
1831    msg->setInt32("generation", mPollDurationGeneration);
1832    msg->post();
1833}
1834
1835void NuPlayer::cancelPollDuration() {
1836    ++mPollDurationGeneration;
1837}
1838
1839void NuPlayer::processDeferredActions() {
1840    while (!mDeferredActions.empty()) {
1841        // We won't execute any deferred actions until we're no longer in
1842        // an intermediate state, i.e. one more more decoders are currently
1843        // flushing or shutting down.
1844
1845        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1846            // We're currently flushing, postpone the reset until that's
1847            // completed.
1848
1849            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1850                  mFlushingAudio, mFlushingVideo);
1851
1852            break;
1853        }
1854
1855        sp<Action> action = *mDeferredActions.begin();
1856        mDeferredActions.erase(mDeferredActions.begin());
1857
1858        action->execute(this);
1859    }
1860}
1861
1862void NuPlayer::performSeek(int64_t seekTimeUs, bool needNotify) {
1863    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs), needNotify(%d)",
1864          seekTimeUs,
1865          seekTimeUs / 1E6,
1866          needNotify);
1867
1868    if (mSource == NULL) {
1869        // This happens when reset occurs right before the loop mode
1870        // asynchronously seeks to the start of the stream.
1871        LOG_ALWAYS_FATAL_IF(mAudioDecoder != NULL || mVideoDecoder != NULL,
1872                "mSource is NULL and decoders not NULL audio(%p) video(%p)",
1873                mAudioDecoder.get(), mVideoDecoder.get());
1874        return;
1875    }
1876    mSource->seekTo(seekTimeUs);
1877    ++mTimedTextGeneration;
1878
1879    if (mDriver != NULL) {
1880        sp<NuPlayerDriver> driver = mDriver.promote();
1881        if (driver != NULL) {
1882            driver->notifyPosition(seekTimeUs);
1883            if (needNotify) {
1884                driver->notifySeekComplete();
1885            }
1886        }
1887    }
1888
1889    // everything's flushed, continue playback.
1890}
1891
1892void NuPlayer::performDecoderFlush() {
1893    ALOGV("performDecoderFlush");
1894
1895    if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
1896        return;
1897    }
1898
1899    mTimeDiscontinuityPending = true;
1900
1901    if (mAudioDecoder != NULL) {
1902        flushDecoder(true /* audio */, false /* needShutdown */);
1903    }
1904
1905    if (mVideoDecoder != NULL) {
1906        flushDecoder(false /* audio */, false /* needShutdown */);
1907    }
1908}
1909
1910void NuPlayer::performDecoderShutdown(bool audio, bool video) {
1911    ALOGV("performDecoderShutdown audio=%d, video=%d", audio, video);
1912
1913    if ((!audio || mAudioDecoder == NULL)
1914            && (!video || mVideoDecoder == NULL)) {
1915        return;
1916    }
1917
1918    mTimeDiscontinuityPending = true;
1919
1920    if (audio && mAudioDecoder != NULL) {
1921        flushDecoder(true /* audio */, true /* needShutdown */);
1922    }
1923
1924    if (video && mVideoDecoder != NULL) {
1925        flushDecoder(false /* audio */, true /* needShutdown */);
1926    }
1927}
1928
1929void NuPlayer::performReset() {
1930    ALOGV("performReset");
1931
1932    CHECK(mAudioDecoder == NULL);
1933    CHECK(mVideoDecoder == NULL);
1934
1935    cancelPollDuration();
1936
1937    ++mScanSourcesGeneration;
1938    mScanSourcesPending = false;
1939
1940    if (mRendererLooper != NULL) {
1941        if (mRenderer != NULL) {
1942            mRendererLooper->unregisterHandler(mRenderer->id());
1943        }
1944        mRendererLooper->stop();
1945        mRendererLooper.clear();
1946    }
1947    mRenderer.clear();
1948    ++mRendererGeneration;
1949
1950    if (mSource != NULL) {
1951        mSource->stop();
1952
1953        mSource.clear();
1954    }
1955
1956    if (mDriver != NULL) {
1957        sp<NuPlayerDriver> driver = mDriver.promote();
1958        if (driver != NULL) {
1959            driver->notifyResetComplete();
1960        }
1961    }
1962
1963    mStarted = false;
1964}
1965
1966void NuPlayer::performScanSources() {
1967    ALOGV("performScanSources");
1968
1969    if (!mStarted) {
1970        return;
1971    }
1972
1973    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1974        postScanSources();
1975    }
1976}
1977
1978void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1979    ALOGV("performSetSurface");
1980
1981    mNativeWindow = wrapper;
1982
1983    // XXX - ignore error from setVideoScalingMode for now
1984    setVideoScalingMode(mVideoScalingMode);
1985
1986    if (mDriver != NULL) {
1987        sp<NuPlayerDriver> driver = mDriver.promote();
1988        if (driver != NULL) {
1989            driver->notifySetSurfaceComplete();
1990        }
1991    }
1992}
1993
1994void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1995    int32_t what;
1996    CHECK(msg->findInt32("what", &what));
1997
1998    switch (what) {
1999        case Source::kWhatPrepared:
2000        {
2001            if (mSource == NULL) {
2002                // This is a stale notification from a source that was
2003                // asynchronously preparing when the client called reset().
2004                // We handled the reset, the source is gone.
2005                break;
2006            }
2007
2008            int32_t err;
2009            CHECK(msg->findInt32("err", &err));
2010
2011            sp<NuPlayerDriver> driver = mDriver.promote();
2012            if (driver != NULL) {
2013                // notify duration first, so that it's definitely set when
2014                // the app received the "prepare complete" callback.
2015                int64_t durationUs;
2016                if (mSource->getDuration(&durationUs) == OK) {
2017                    driver->notifyDuration(durationUs);
2018                }
2019                driver->notifyPrepareCompleted(err);
2020            }
2021
2022            break;
2023        }
2024
2025        case Source::kWhatFlagsChanged:
2026        {
2027            uint32_t flags;
2028            CHECK(msg->findInt32("flags", (int32_t *)&flags));
2029
2030            sp<NuPlayerDriver> driver = mDriver.promote();
2031            if (driver != NULL) {
2032                driver->notifyFlagsChanged(flags);
2033            }
2034
2035            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
2036                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
2037                cancelPollDuration();
2038            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
2039                    && (flags & Source::FLAG_DYNAMIC_DURATION)
2040                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
2041                schedulePollDuration();
2042            }
2043
2044            mSourceFlags = flags;
2045            break;
2046        }
2047
2048        case Source::kWhatVideoSizeChanged:
2049        {
2050            sp<AMessage> format;
2051            CHECK(msg->findMessage("format", &format));
2052
2053            updateVideoSize(format);
2054            break;
2055        }
2056
2057        case Source::kWhatBufferingUpdate:
2058        {
2059            int32_t percentage;
2060            CHECK(msg->findInt32("percentage", &percentage));
2061
2062            notifyListener(MEDIA_BUFFERING_UPDATE, percentage, 0);
2063            break;
2064        }
2065
2066        case Source::kWhatBufferingStart:
2067        {
2068            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
2069            break;
2070        }
2071
2072        case Source::kWhatBufferingEnd:
2073        {
2074            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
2075            break;
2076        }
2077
2078        case Source::kWhatSubtitleData:
2079        {
2080            sp<ABuffer> buffer;
2081            CHECK(msg->findBuffer("buffer", &buffer));
2082
2083            sendSubtitleData(buffer, 0 /* baseIndex */);
2084            break;
2085        }
2086
2087        case Source::kWhatTimedTextData:
2088        {
2089            int32_t generation;
2090            if (msg->findInt32("generation", &generation)
2091                    && generation != mTimedTextGeneration) {
2092                break;
2093            }
2094
2095            sp<ABuffer> buffer;
2096            CHECK(msg->findBuffer("buffer", &buffer));
2097
2098            sp<NuPlayerDriver> driver = mDriver.promote();
2099            if (driver == NULL) {
2100                break;
2101            }
2102
2103            int posMs;
2104            int64_t timeUs, posUs;
2105            driver->getCurrentPosition(&posMs);
2106            posUs = posMs * 1000;
2107            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2108
2109            if (posUs < timeUs) {
2110                if (!msg->findInt32("generation", &generation)) {
2111                    msg->setInt32("generation", mTimedTextGeneration);
2112                }
2113                msg->post(timeUs - posUs);
2114            } else {
2115                sendTimedTextData(buffer);
2116            }
2117            break;
2118        }
2119
2120        case Source::kWhatQueueDecoderShutdown:
2121        {
2122            int32_t audio, video;
2123            CHECK(msg->findInt32("audio", &audio));
2124            CHECK(msg->findInt32("video", &video));
2125
2126            sp<AMessage> reply;
2127            CHECK(msg->findMessage("reply", &reply));
2128
2129            queueDecoderShutdown(audio, video, reply);
2130            break;
2131        }
2132
2133        case Source::kWhatDrmNoLicense:
2134        {
2135            notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, ERROR_DRM_NO_LICENSE);
2136            break;
2137        }
2138
2139        default:
2140            TRESPASS();
2141    }
2142}
2143
2144void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
2145    int32_t what;
2146    CHECK(msg->findInt32("what", &what));
2147
2148    switch (what) {
2149        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
2150        {
2151            sp<ABuffer> buffer;
2152            CHECK(msg->findBuffer("buffer", &buffer));
2153
2154            size_t inbandTracks = 0;
2155            if (mSource != NULL) {
2156                inbandTracks = mSource->getTrackCount();
2157            }
2158
2159            sendSubtitleData(buffer, inbandTracks);
2160            break;
2161        }
2162
2163        case NuPlayer::CCDecoder::kWhatTrackAdded:
2164        {
2165            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
2166
2167            break;
2168        }
2169
2170        default:
2171            TRESPASS();
2172    }
2173
2174
2175}
2176
2177void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
2178    int32_t trackIndex;
2179    int64_t timeUs, durationUs;
2180    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
2181    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2182    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
2183
2184    Parcel in;
2185    in.writeInt32(trackIndex + baseIndex);
2186    in.writeInt64(timeUs);
2187    in.writeInt64(durationUs);
2188    in.writeInt32(buffer->size());
2189    in.writeInt32(buffer->size());
2190    in.write(buffer->data(), buffer->size());
2191
2192    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
2193}
2194
2195void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
2196    const void *data;
2197    size_t size = 0;
2198    int64_t timeUs;
2199    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
2200
2201    AString mime;
2202    CHECK(buffer->meta()->findString("mime", &mime));
2203    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
2204
2205    data = buffer->data();
2206    size = buffer->size();
2207
2208    Parcel parcel;
2209    if (size > 0) {
2210        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2211        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
2212        TextDescriptions::getParcelOfDescriptions(
2213                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
2214    }
2215
2216    if ((parcel.dataSize() > 0)) {
2217        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
2218    } else {  // send an empty timed text
2219        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
2220    }
2221}
2222////////////////////////////////////////////////////////////////////////////////
2223
2224sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
2225    sp<MetaData> meta = getFormatMeta(audio);
2226
2227    if (meta == NULL) {
2228        return NULL;
2229    }
2230
2231    sp<AMessage> msg = new AMessage;
2232
2233    if(convertMetaDataToMessage(meta, &msg) == OK) {
2234        return msg;
2235    }
2236    return NULL;
2237}
2238
2239void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
2240    sp<AMessage> notify = dupNotify();
2241    notify->setInt32("what", kWhatFlagsChanged);
2242    notify->setInt32("flags", flags);
2243    notify->post();
2244}
2245
2246void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
2247    sp<AMessage> notify = dupNotify();
2248    notify->setInt32("what", kWhatVideoSizeChanged);
2249    notify->setMessage("format", format);
2250    notify->post();
2251}
2252
2253void NuPlayer::Source::notifyPrepared(status_t err) {
2254    sp<AMessage> notify = dupNotify();
2255    notify->setInt32("what", kWhatPrepared);
2256    notify->setInt32("err", err);
2257    notify->post();
2258}
2259
2260void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
2261    TRESPASS();
2262}
2263
2264}  // namespace android
2265