NuPlayer.cpp revision 6ffb1fd67eb8f00f130a6db914ba42a8432aec70 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerDecoder.h"
25#include "NuPlayerDecoderPassThrough.h"
26#include "NuPlayerDriver.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29#include "RTSPSource.h"
30#include "StreamingSource.h"
31#include "GenericSource.h"
32#include "TextDescriptions.h"
33
34#include "ATSParser.h"
35
36#include <media/stagefright/foundation/hexdump.h>
37#include <media/stagefright/foundation/ABuffer.h>
38#include <media/stagefright/foundation/ADebug.h>
39#include <media/stagefright/foundation/AMessage.h>
40#include <media/stagefright/MediaBuffer.h>
41#include <media/stagefright/MediaDefs.h>
42#include <media/stagefright/MediaErrors.h>
43#include <media/stagefright/MetaData.h>
44#include <gui/IGraphicBufferProducer.h>
45
46#include "avc_utils.h"
47
48#include "ESDS.h"
49#include <media/stagefright/Utils.h>
50
51namespace android {
52
53// TODO optimize buffer size for power consumption
54// The offload read buffer size is 32 KB but 24 KB uses less power.
55const size_t NuPlayer::kAggregateBufferSizeBytes = 24 * 1024;
56
57struct NuPlayer::Action : public RefBase {
58    Action() {}
59
60    virtual void execute(NuPlayer *player) = 0;
61
62private:
63    DISALLOW_EVIL_CONSTRUCTORS(Action);
64};
65
66struct NuPlayer::SeekAction : public Action {
67    SeekAction(int64_t seekTimeUs, bool needNotify)
68        : mSeekTimeUs(seekTimeUs),
69          mNeedNotify(needNotify) {
70    }
71
72    virtual void execute(NuPlayer *player) {
73        player->performSeek(mSeekTimeUs, mNeedNotify);
74    }
75
76private:
77    int64_t mSeekTimeUs;
78    bool mNeedNotify;
79
80    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
81};
82
83struct NuPlayer::SetSurfaceAction : public Action {
84    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
85        : mWrapper(wrapper) {
86    }
87
88    virtual void execute(NuPlayer *player) {
89        player->performSetSurface(mWrapper);
90    }
91
92private:
93    sp<NativeWindowWrapper> mWrapper;
94
95    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
96};
97
98struct NuPlayer::FlushDecoderAction : public Action {
99    FlushDecoderAction(FlushCommand audio, FlushCommand video)
100        : mAudio(audio),
101          mVideo(video) {
102    }
103
104    virtual void execute(NuPlayer *player) {
105        player->performDecoderFlush(mAudio, mVideo);
106    }
107
108private:
109    FlushCommand mAudio;
110    FlushCommand mVideo;
111
112    DISALLOW_EVIL_CONSTRUCTORS(FlushDecoderAction);
113};
114
115struct NuPlayer::PostMessageAction : public Action {
116    PostMessageAction(const sp<AMessage> &msg)
117        : mMessage(msg) {
118    }
119
120    virtual void execute(NuPlayer *) {
121        mMessage->post();
122    }
123
124private:
125    sp<AMessage> mMessage;
126
127    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
128};
129
130// Use this if there's no state necessary to save in order to execute
131// the action.
132struct NuPlayer::SimpleAction : public Action {
133    typedef void (NuPlayer::*ActionFunc)();
134
135    SimpleAction(ActionFunc func)
136        : mFunc(func) {
137    }
138
139    virtual void execute(NuPlayer *player) {
140        (player->*mFunc)();
141    }
142
143private:
144    ActionFunc mFunc;
145
146    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
147};
148
149////////////////////////////////////////////////////////////////////////////////
150
151NuPlayer::NuPlayer()
152    : mUIDValid(false),
153      mSourceFlags(0),
154      mVideoIsAVC(false),
155      mOffloadAudio(false),
156      mAudioDecoderGeneration(0),
157      mVideoDecoderGeneration(0),
158      mRendererGeneration(0),
159      mAudioEOS(false),
160      mVideoEOS(false),
161      mScanSourcesPending(false),
162      mScanSourcesGeneration(0),
163      mPollDurationGeneration(0),
164      mTimedTextGeneration(0),
165      mTimeDiscontinuityPending(false),
166      mFlushingAudio(NONE),
167      mFlushingVideo(NONE),
168      mNumFramesTotal(0ll),
169      mNumFramesDropped(0ll),
170      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
171      mStarted(false) {
172    clearFlushComplete();
173}
174
175NuPlayer::~NuPlayer() {
176}
177
178void NuPlayer::setUID(uid_t uid) {
179    mUIDValid = true;
180    mUID = uid;
181}
182
183void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
184    mDriver = driver;
185}
186
187void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
188    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
189
190    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
191
192    msg->setObject("source", new StreamingSource(notify, source));
193    msg->post();
194}
195
196static bool IsHTTPLiveURL(const char *url) {
197    if (!strncasecmp("http://", url, 7)
198            || !strncasecmp("https://", url, 8)
199            || !strncasecmp("file://", url, 7)) {
200        size_t len = strlen(url);
201        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
202            return true;
203        }
204
205        if (strstr(url,"m3u8")) {
206            return true;
207        }
208    }
209
210    return false;
211}
212
213void NuPlayer::setDataSourceAsync(
214        const sp<IMediaHTTPService> &httpService,
215        const char *url,
216        const KeyedVector<String8, String8> *headers) {
217
218    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
219    size_t len = strlen(url);
220
221    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
222
223    sp<Source> source;
224    if (IsHTTPLiveURL(url)) {
225        source = new HTTPLiveSource(notify, httpService, url, headers);
226    } else if (!strncasecmp(url, "rtsp://", 7)) {
227        source = new RTSPSource(
228                notify, httpService, url, headers, mUIDValid, mUID);
229    } else if ((!strncasecmp(url, "http://", 7)
230                || !strncasecmp(url, "https://", 8))
231                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
232                    || strstr(url, ".sdp?"))) {
233        source = new RTSPSource(
234                notify, httpService, url, headers, mUIDValid, mUID, true);
235    } else {
236        sp<GenericSource> genericSource =
237                new GenericSource(notify, mUIDValid, mUID);
238        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
239        // The correct flags will be updated in Source::kWhatFlagsChanged
240        // handler when  GenericSource is prepared.
241
242        status_t err = genericSource->setDataSource(httpService, url, headers);
243
244        if (err == OK) {
245            source = genericSource;
246        } else {
247            ALOGE("Failed to set data source!");
248        }
249    }
250    msg->setObject("source", source);
251    msg->post();
252}
253
254void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
255    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
256
257    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
258
259    sp<GenericSource> source =
260            new GenericSource(notify, mUIDValid, mUID);
261
262    status_t err = source->setDataSource(fd, offset, length);
263
264    if (err != OK) {
265        ALOGE("Failed to set data source!");
266        source = NULL;
267    }
268
269    msg->setObject("source", source);
270    msg->post();
271}
272
273void NuPlayer::prepareAsync() {
274    (new AMessage(kWhatPrepare, id()))->post();
275}
276
277void NuPlayer::setVideoSurfaceTextureAsync(
278        const sp<IGraphicBufferProducer> &bufferProducer) {
279    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id());
280
281    if (bufferProducer == NULL) {
282        msg->setObject("native-window", NULL);
283    } else {
284        msg->setObject(
285                "native-window",
286                new NativeWindowWrapper(
287                    new Surface(bufferProducer, true /* controlledByApp */)));
288    }
289
290    msg->post();
291}
292
293void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
294    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id());
295    msg->setObject("sink", sink);
296    msg->post();
297}
298
299void NuPlayer::start() {
300    (new AMessage(kWhatStart, id()))->post();
301}
302
303void NuPlayer::pause() {
304    (new AMessage(kWhatPause, id()))->post();
305}
306
307void NuPlayer::resetAsync() {
308    if (mSource != NULL) {
309        // During a reset, the data source might be unresponsive already, we need to
310        // disconnect explicitly so that reads exit promptly.
311        // We can't queue the disconnect request to the looper, as it might be
312        // queued behind a stuck read and never gets processed.
313        // Doing a disconnect outside the looper to allows the pending reads to exit
314        // (either successfully or with error).
315        mSource->disconnect();
316    }
317
318    (new AMessage(kWhatReset, id()))->post();
319}
320
321void NuPlayer::seekToAsync(int64_t seekTimeUs, bool needNotify) {
322    sp<AMessage> msg = new AMessage(kWhatSeek, id());
323    msg->setInt64("seekTimeUs", seekTimeUs);
324    msg->setInt32("needNotify", needNotify);
325    msg->post();
326}
327
328
329void NuPlayer::writeTrackInfo(
330        Parcel* reply, const sp<AMessage> format) const {
331    int32_t trackType;
332    CHECK(format->findInt32("type", &trackType));
333
334    AString lang;
335    CHECK(format->findString("language", &lang));
336
337    reply->writeInt32(2); // write something non-zero
338    reply->writeInt32(trackType);
339    reply->writeString16(String16(lang.c_str()));
340
341    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
342        AString mime;
343        CHECK(format->findString("mime", &mime));
344
345        int32_t isAuto, isDefault, isForced;
346        CHECK(format->findInt32("auto", &isAuto));
347        CHECK(format->findInt32("default", &isDefault));
348        CHECK(format->findInt32("forced", &isForced));
349
350        reply->writeString16(String16(mime.c_str()));
351        reply->writeInt32(isAuto);
352        reply->writeInt32(isDefault);
353        reply->writeInt32(isForced);
354    }
355}
356
357void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
358    switch (msg->what()) {
359        case kWhatSetDataSource:
360        {
361            ALOGV("kWhatSetDataSource");
362
363            CHECK(mSource == NULL);
364
365            status_t err = OK;
366            sp<RefBase> obj;
367            CHECK(msg->findObject("source", &obj));
368            if (obj != NULL) {
369                mSource = static_cast<Source *>(obj.get());
370            } else {
371                err = UNKNOWN_ERROR;
372            }
373
374            CHECK(mDriver != NULL);
375            sp<NuPlayerDriver> driver = mDriver.promote();
376            if (driver != NULL) {
377                driver->notifySetDataSourceCompleted(err);
378            }
379            break;
380        }
381
382        case kWhatPrepare:
383        {
384            mSource->prepareAsync();
385            break;
386        }
387
388        case kWhatGetTrackInfo:
389        {
390            uint32_t replyID;
391            CHECK(msg->senderAwaitsResponse(&replyID));
392
393            Parcel* reply;
394            CHECK(msg->findPointer("reply", (void**)&reply));
395
396            size_t inbandTracks = 0;
397            if (mSource != NULL) {
398                inbandTracks = mSource->getTrackCount();
399            }
400
401            size_t ccTracks = 0;
402            if (mCCDecoder != NULL) {
403                ccTracks = mCCDecoder->getTrackCount();
404            }
405
406            // total track count
407            reply->writeInt32(inbandTracks + ccTracks);
408
409            // write inband tracks
410            for (size_t i = 0; i < inbandTracks; ++i) {
411                writeTrackInfo(reply, mSource->getTrackInfo(i));
412            }
413
414            // write CC track
415            for (size_t i = 0; i < ccTracks; ++i) {
416                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
417            }
418
419            sp<AMessage> response = new AMessage;
420            response->postReply(replyID);
421            break;
422        }
423
424        case kWhatGetSelectedTrack:
425        {
426            status_t err = INVALID_OPERATION;
427            if (mSource != NULL) {
428                err = OK;
429
430                int32_t type32;
431                CHECK(msg->findInt32("type", (int32_t*)&type32));
432                media_track_type type = (media_track_type)type32;
433                ssize_t selectedTrack = mSource->getSelectedTrack(type);
434
435                Parcel* reply;
436                CHECK(msg->findPointer("reply", (void**)&reply));
437                reply->writeInt32(selectedTrack);
438            }
439
440            sp<AMessage> response = new AMessage;
441            response->setInt32("err", err);
442
443            uint32_t replyID;
444            CHECK(msg->senderAwaitsResponse(&replyID));
445            response->postReply(replyID);
446            break;
447        }
448
449        case kWhatSelectTrack:
450        {
451            uint32_t replyID;
452            CHECK(msg->senderAwaitsResponse(&replyID));
453
454            size_t trackIndex;
455            int32_t select;
456            int64_t timeUs;
457            CHECK(msg->findSize("trackIndex", &trackIndex));
458            CHECK(msg->findInt32("select", &select));
459            CHECK(msg->findInt64("timeUs", &timeUs));
460
461            status_t err = INVALID_OPERATION;
462
463            size_t inbandTracks = 0;
464            if (mSource != NULL) {
465                inbandTracks = mSource->getTrackCount();
466            }
467            size_t ccTracks = 0;
468            if (mCCDecoder != NULL) {
469                ccTracks = mCCDecoder->getTrackCount();
470            }
471
472            if (trackIndex < inbandTracks) {
473                err = mSource->selectTrack(trackIndex, select, timeUs);
474
475                if (!select && err == OK) {
476                    int32_t type;
477                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
478                    if (info != NULL
479                            && info->findInt32("type", &type)
480                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
481                        ++mTimedTextGeneration;
482                    }
483                }
484            } else {
485                trackIndex -= inbandTracks;
486
487                if (trackIndex < ccTracks) {
488                    err = mCCDecoder->selectTrack(trackIndex, select);
489                }
490            }
491
492            sp<AMessage> response = new AMessage;
493            response->setInt32("err", err);
494
495            response->postReply(replyID);
496            break;
497        }
498
499        case kWhatPollDuration:
500        {
501            int32_t generation;
502            CHECK(msg->findInt32("generation", &generation));
503
504            if (generation != mPollDurationGeneration) {
505                // stale
506                break;
507            }
508
509            int64_t durationUs;
510            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
511                sp<NuPlayerDriver> driver = mDriver.promote();
512                if (driver != NULL) {
513                    driver->notifyDuration(durationUs);
514                }
515            }
516
517            msg->post(1000000ll);  // poll again in a second.
518            break;
519        }
520
521        case kWhatSetVideoNativeWindow:
522        {
523            ALOGV("kWhatSetVideoNativeWindow");
524
525            sp<RefBase> obj;
526            CHECK(msg->findObject("native-window", &obj));
527
528            if (mSource->getFormat(false /* audio */) == NULL) {
529                performSetSurface(static_cast<NativeWindowWrapper *>(obj.get()));
530                break;
531            }
532
533            mDeferredActions.push_back(
534                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
535                                           FLUSH_CMD_SHUTDOWN /* video */));
536
537            mDeferredActions.push_back(
538                    new SetSurfaceAction(
539                        static_cast<NativeWindowWrapper *>(obj.get())));
540
541            if (obj != NULL) {
542                if (mStarted) {
543                    // Issue a seek to refresh the video screen only if started otherwise
544                    // the extractor may not yet be started and will assert.
545                    // If the video decoder is not set (perhaps audio only in this case)
546                    // do not perform a seek as it is not needed.
547                    int64_t currentPositionUs = 0;
548                    if (getCurrentPosition(&currentPositionUs) == OK) {
549                        mDeferredActions.push_back(
550                                new SeekAction(currentPositionUs, false /* needNotify */));
551                    }
552                }
553
554                // If there is a new surface texture, instantiate decoders
555                // again if possible.
556                mDeferredActions.push_back(
557                        new SimpleAction(&NuPlayer::performScanSources));
558            }
559
560            processDeferredActions();
561            break;
562        }
563
564        case kWhatSetAudioSink:
565        {
566            ALOGV("kWhatSetAudioSink");
567
568            sp<RefBase> obj;
569            CHECK(msg->findObject("sink", &obj));
570
571            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
572            break;
573        }
574
575        case kWhatStart:
576        {
577            ALOGV("kWhatStart");
578            if (mStarted) {
579                onResume();
580            } else {
581                onStart();
582            }
583            break;
584        }
585
586        case kWhatScanSources:
587        {
588            int32_t generation;
589            CHECK(msg->findInt32("generation", &generation));
590            if (generation != mScanSourcesGeneration) {
591                // Drop obsolete msg.
592                break;
593            }
594
595            mScanSourcesPending = false;
596
597            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
598                 mAudioDecoder != NULL, mVideoDecoder != NULL);
599
600            bool mHadAnySourcesBefore =
601                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
602
603            // initialize video before audio because successful initialization of
604            // video may change deep buffer mode of audio.
605            if (mNativeWindow != NULL) {
606                instantiateDecoder(false, &mVideoDecoder);
607            }
608
609            if (mAudioSink != NULL) {
610                if (mOffloadAudio) {
611                    // open audio sink early under offload mode.
612                    sp<AMessage> format = mSource->getFormat(true /*audio*/);
613                    openAudioSink(format, true /*offloadOnly*/);
614                }
615                instantiateDecoder(true, &mAudioDecoder);
616            }
617
618            if (!mHadAnySourcesBefore
619                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
620                // This is the first time we've found anything playable.
621
622                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
623                    schedulePollDuration();
624                }
625            }
626
627            status_t err;
628            if ((err = mSource->feedMoreTSData()) != OK) {
629                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
630                    // We're not currently decoding anything (no audio or
631                    // video tracks found) and we just ran out of input data.
632
633                    if (err == ERROR_END_OF_STREAM) {
634                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
635                    } else {
636                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
637                    }
638                }
639                break;
640            }
641
642            if ((mAudioDecoder == NULL && mAudioSink != NULL)
643                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
644                msg->post(100000ll);
645                mScanSourcesPending = true;
646            }
647            break;
648        }
649
650        case kWhatVideoNotify:
651        case kWhatAudioNotify:
652        {
653            bool audio = msg->what() == kWhatAudioNotify;
654
655            int32_t currentDecoderGeneration =
656                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
657            int32_t requesterGeneration = currentDecoderGeneration - 1;
658            CHECK(msg->findInt32("generation", &requesterGeneration));
659
660            if (requesterGeneration != currentDecoderGeneration) {
661                ALOGV("got message from old %s decoder, generation(%d:%d)",
662                        audio ? "audio" : "video", requesterGeneration,
663                        currentDecoderGeneration);
664                sp<AMessage> reply;
665                if (!(msg->findMessage("reply", &reply))) {
666                    return;
667                }
668
669                reply->setInt32("err", INFO_DISCONTINUITY);
670                reply->post();
671                return;
672            }
673
674            int32_t what;
675            CHECK(msg->findInt32("what", &what));
676
677            if (what == Decoder::kWhatFillThisBuffer) {
678                status_t err = feedDecoderInputData(
679                        audio, msg);
680
681                if (err == -EWOULDBLOCK) {
682                    if (mSource->feedMoreTSData() == OK) {
683                        msg->post(10 * 1000ll);
684                    }
685                }
686            } else if (what == Decoder::kWhatEOS) {
687                int32_t err;
688                CHECK(msg->findInt32("err", &err));
689
690                if (err == ERROR_END_OF_STREAM) {
691                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
692                } else {
693                    ALOGV("got %s decoder EOS w/ error %d",
694                         audio ? "audio" : "video",
695                         err);
696                }
697
698                mRenderer->queueEOS(audio, err);
699            } else if (what == Decoder::kWhatFlushCompleted) {
700                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
701
702                handleFlushComplete(audio, true /* isDecoder */);
703                finishFlushIfPossible();
704            } else if (what == Decoder::kWhatVideoSizeChanged) {
705                sp<AMessage> format;
706                CHECK(msg->findMessage("format", &format));
707
708                sp<AMessage> inputFormat =
709                        mSource->getFormat(false /* audio */);
710
711                updateVideoSize(inputFormat, format);
712            } else if (what == Decoder::kWhatShutdownCompleted) {
713                ALOGV("%s shutdown completed", audio ? "audio" : "video");
714                if (audio) {
715                    mAudioDecoder.clear();
716                    ++mAudioDecoderGeneration;
717
718                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
719                    mFlushingAudio = SHUT_DOWN;
720                } else {
721                    mVideoDecoder.clear();
722                    ++mVideoDecoderGeneration;
723
724                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
725                    mFlushingVideo = SHUT_DOWN;
726                }
727
728                finishFlushIfPossible();
729            } else if (what == Decoder::kWhatError) {
730                status_t err;
731                if (!msg->findInt32("err", &err) || err == OK) {
732                    err = UNKNOWN_ERROR;
733                }
734
735                // Decoder errors can be due to Source (e.g. from streaming),
736                // or from decoding corrupted bitstreams, or from other decoder
737                // MediaCodec operations (e.g. from an ongoing reset or seek).
738                //
739                // We try to gracefully shut down the affected decoder if possible,
740                // rather than trying to force the shutdown with something
741                // similar to performReset(). This method can lead to a hang
742                // if MediaCodec functions block after an error, but they should
743                // typically return INVALID_OPERATION instead of blocking.
744
745                FlushStatus *flushing = audio ? &mFlushingAudio : &mFlushingVideo;
746                ALOGE("received error(%#x) from %s decoder, flushing(%d), now shutting down",
747                        err, audio ? "audio" : "video", *flushing);
748
749                switch (*flushing) {
750                    case NONE:
751                        mDeferredActions.push_back(
752                                new FlushDecoderAction(
753                                    audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
754                                    audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN));
755                        processDeferredActions();
756                        break;
757                    case FLUSHING_DECODER:
758                        *flushing = FLUSHING_DECODER_SHUTDOWN; // initiate shutdown after flush.
759                        break; // Wait for flush to complete.
760                    case FLUSHING_DECODER_SHUTDOWN:
761                        break; // Wait for flush to complete.
762                    case SHUTTING_DOWN_DECODER:
763                        break; // Wait for shutdown to complete.
764                    case FLUSHED:
765                        // Widevine source reads must stop before releasing the video decoder.
766                        if (!audio && mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
767                            mSource->stop();
768                        }
769                        getDecoder(audio)->initiateShutdown(); // In the middle of a seek.
770                        *flushing = SHUTTING_DOWN_DECODER;     // Shut down.
771                        break;
772                    case SHUT_DOWN:
773                        finishFlushIfPossible();  // Should not occur.
774                        break;                    // Finish anyways.
775                }
776                notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
777            } else if (what == Decoder::kWhatRenderBufferTime) {
778                renderBuffer(audio, msg);
779            } else {
780                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
781                      what,
782                      what >> 24,
783                      (what >> 16) & 0xff,
784                      (what >> 8) & 0xff,
785                      what & 0xff);
786            }
787
788            break;
789        }
790
791        case kWhatRendererNotify:
792        {
793            int32_t requesterGeneration = mRendererGeneration - 1;
794            CHECK(msg->findInt32("generation", &requesterGeneration));
795            if (requesterGeneration != mRendererGeneration) {
796                ALOGV("got message from old renderer, generation(%d:%d)",
797                        requesterGeneration, mRendererGeneration);
798                return;
799            }
800
801            int32_t what;
802            CHECK(msg->findInt32("what", &what));
803
804            if (what == Renderer::kWhatEOS) {
805                int32_t audio;
806                CHECK(msg->findInt32("audio", &audio));
807
808                int32_t finalResult;
809                CHECK(msg->findInt32("finalResult", &finalResult));
810
811                if (audio) {
812                    mAudioEOS = true;
813                } else {
814                    mVideoEOS = true;
815                }
816
817                if (finalResult == ERROR_END_OF_STREAM) {
818                    ALOGV("reached %s EOS", audio ? "audio" : "video");
819                } else {
820                    ALOGE("%s track encountered an error (%d)",
821                         audio ? "audio" : "video", finalResult);
822
823                    notifyListener(
824                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
825                }
826
827                if ((mAudioEOS || mAudioDecoder == NULL)
828                        && (mVideoEOS || mVideoDecoder == NULL)) {
829                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
830                }
831            } else if (what == Renderer::kWhatFlushComplete) {
832                int32_t audio;
833                CHECK(msg->findInt32("audio", &audio));
834
835                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
836                handleFlushComplete(audio, false /* isDecoder */);
837                finishFlushIfPossible();
838            } else if (what == Renderer::kWhatVideoRenderingStart) {
839                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
840            } else if (what == Renderer::kWhatMediaRenderingStart) {
841                ALOGV("media rendering started");
842                notifyListener(MEDIA_STARTED, 0, 0);
843            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
844                ALOGV("Tear down audio offload, fall back to s/w path");
845                int64_t positionUs;
846                CHECK(msg->findInt64("positionUs", &positionUs));
847                int32_t reason;
848                CHECK(msg->findInt32("reason", &reason));
849                closeAudioSink();
850                mAudioDecoder.clear();
851                ++mAudioDecoderGeneration;
852                mRenderer->flush(true /* audio */);
853                if (mVideoDecoder != NULL) {
854                    mRenderer->flush(false /* audio */);
855                }
856                mRenderer->signalDisableOffloadAudio();
857                mOffloadAudio = false;
858
859                performSeek(positionUs, false /* needNotify */);
860                if (reason == Renderer::kDueToError) {
861                    instantiateDecoder(true /* audio */, &mAudioDecoder);
862                }
863            }
864            break;
865        }
866
867        case kWhatMoreDataQueued:
868        {
869            break;
870        }
871
872        case kWhatReset:
873        {
874            ALOGV("kWhatReset");
875
876            mDeferredActions.push_back(
877                    new FlushDecoderAction(
878                        FLUSH_CMD_SHUTDOWN /* audio */,
879                        FLUSH_CMD_SHUTDOWN /* video */));
880
881            mDeferredActions.push_back(
882                    new SimpleAction(&NuPlayer::performReset));
883
884            processDeferredActions();
885            break;
886        }
887
888        case kWhatSeek:
889        {
890            int64_t seekTimeUs;
891            int32_t needNotify;
892            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
893            CHECK(msg->findInt32("needNotify", &needNotify));
894
895            ALOGV("kWhatSeek seekTimeUs=%lld us, needNotify=%d",
896                    seekTimeUs, needNotify);
897
898            mDeferredActions.push_back(
899                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
900                                           FLUSH_CMD_FLUSH /* video */));
901
902            mDeferredActions.push_back(
903                    new SeekAction(seekTimeUs, needNotify));
904
905            processDeferredActions();
906            break;
907        }
908
909        case kWhatPause:
910        {
911            if (mSource != NULL) {
912                mSource->pause();
913            } else {
914                ALOGW("pause called when source is gone or not set");
915            }
916            if (mRenderer != NULL) {
917                mRenderer->pause();
918            } else {
919                ALOGW("pause called when renderer is gone or not set");
920            }
921            break;
922        }
923
924        case kWhatSourceNotify:
925        {
926            onSourceNotify(msg);
927            break;
928        }
929
930        case kWhatClosedCaptionNotify:
931        {
932            onClosedCaptionNotify(msg);
933            break;
934        }
935
936        default:
937            TRESPASS();
938            break;
939    }
940}
941
942void NuPlayer::onResume() {
943    if (mSource != NULL) {
944        mSource->resume();
945    } else {
946        ALOGW("resume called when source is gone or not set");
947    }
948    // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if
949    // needed.
950    if (audioDecoderStillNeeded() && mAudioDecoder == NULL) {
951        instantiateDecoder(true /* audio */, &mAudioDecoder);
952    }
953    if (mRenderer != NULL) {
954        mRenderer->resume();
955    } else {
956        ALOGW("resume called when renderer is gone or not set");
957    }
958}
959
960void NuPlayer::onStart() {
961    mVideoIsAVC = false;
962    mOffloadAudio = false;
963    mAudioEOS = false;
964    mVideoEOS = false;
965    mNumFramesTotal = 0;
966    mNumFramesDropped = 0;
967    mStarted = true;
968
969    /* instantiate decoders now for secure playback */
970    if (mSourceFlags & Source::FLAG_SECURE) {
971        if (mNativeWindow != NULL) {
972            instantiateDecoder(false, &mVideoDecoder);
973        }
974
975        if (mAudioSink != NULL) {
976            instantiateDecoder(true, &mAudioDecoder);
977        }
978    }
979
980    mSource->start();
981
982    uint32_t flags = 0;
983
984    if (mSource->isRealTime()) {
985        flags |= Renderer::FLAG_REAL_TIME;
986    }
987
988    sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
989    audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
990    if (mAudioSink != NULL) {
991        streamType = mAudioSink->getAudioStreamType();
992    }
993
994    sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
995
996    mOffloadAudio =
997        canOffloadStream(audioMeta, (videoFormat != NULL),
998                         true /* is_streaming */, streamType);
999    if (mOffloadAudio) {
1000        flags |= Renderer::FLAG_OFFLOAD_AUDIO;
1001    }
1002
1003    sp<AMessage> notify = new AMessage(kWhatRendererNotify, id());
1004    ++mRendererGeneration;
1005    notify->setInt32("generation", mRendererGeneration);
1006    mRenderer = new Renderer(mAudioSink, notify, flags);
1007
1008    mRendererLooper = new ALooper;
1009    mRendererLooper->setName("NuPlayerRenderer");
1010    mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
1011    mRendererLooper->registerHandler(mRenderer);
1012
1013    sp<MetaData> meta = getFileMeta();
1014    int32_t rate;
1015    if (meta != NULL
1016            && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) {
1017        mRenderer->setVideoFrameRate(rate);
1018    }
1019
1020    if (mVideoDecoder != NULL) {
1021        mVideoDecoder->setRenderer(mRenderer);
1022    }
1023    if (mAudioDecoder != NULL) {
1024        mAudioDecoder->setRenderer(mRenderer);
1025    }
1026
1027    postScanSources();
1028}
1029
1030bool NuPlayer::audioDecoderStillNeeded() {
1031    // Audio decoder is no longer needed if it's in shut/shutting down status.
1032    return ((mFlushingAudio != SHUT_DOWN) && (mFlushingAudio != SHUTTING_DOWN_DECODER));
1033}
1034
1035void NuPlayer::handleFlushComplete(bool audio, bool isDecoder) {
1036    // We wait for both the decoder flush and the renderer flush to complete
1037    // before entering either the FLUSHED or the SHUTTING_DOWN_DECODER state.
1038
1039    mFlushComplete[audio][isDecoder] = true;
1040    if (!mFlushComplete[audio][!isDecoder]) {
1041        return;
1042    }
1043
1044    FlushStatus *state = audio ? &mFlushingAudio : &mFlushingVideo;
1045    switch (*state) {
1046        case FLUSHING_DECODER:
1047        {
1048            *state = FLUSHED;
1049            break;
1050        }
1051
1052        case FLUSHING_DECODER_SHUTDOWN:
1053        {
1054            *state = SHUTTING_DOWN_DECODER;
1055
1056            ALOGV("initiating %s decoder shutdown", audio ? "audio" : "video");
1057            if (!audio) {
1058                // Widevine source reads must stop before releasing the video decoder.
1059                if (mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
1060                    mSource->stop();
1061                }
1062            }
1063            getDecoder(audio)->initiateShutdown();
1064            break;
1065        }
1066
1067        default:
1068            // decoder flush completes only occur in a flushing state.
1069            LOG_ALWAYS_FATAL_IF(isDecoder, "decoder flush in invalid state %d", *state);
1070            break;
1071    }
1072}
1073
1074void NuPlayer::finishFlushIfPossible() {
1075    if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED
1076            && mFlushingAudio != SHUT_DOWN) {
1077        return;
1078    }
1079
1080    if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED
1081            && mFlushingVideo != SHUT_DOWN) {
1082        return;
1083    }
1084
1085    ALOGV("both audio and video are flushed now.");
1086
1087    mPendingAudioAccessUnit.clear();
1088    mAggregateBuffer.clear();
1089
1090    if (mTimeDiscontinuityPending) {
1091        mRenderer->signalTimeDiscontinuity();
1092        mTimeDiscontinuityPending = false;
1093    }
1094
1095    if (mAudioDecoder != NULL && mFlushingAudio == FLUSHED) {
1096        mAudioDecoder->signalResume();
1097    }
1098
1099    if (mVideoDecoder != NULL && mFlushingVideo == FLUSHED) {
1100        mVideoDecoder->signalResume();
1101    }
1102
1103    mFlushingAudio = NONE;
1104    mFlushingVideo = NONE;
1105
1106    clearFlushComplete();
1107
1108    processDeferredActions();
1109}
1110
1111void NuPlayer::postScanSources() {
1112    if (mScanSourcesPending) {
1113        return;
1114    }
1115
1116    sp<AMessage> msg = new AMessage(kWhatScanSources, id());
1117    msg->setInt32("generation", mScanSourcesGeneration);
1118    msg->post();
1119
1120    mScanSourcesPending = true;
1121}
1122
1123void NuPlayer::openAudioSink(const sp<AMessage> &format, bool offloadOnly) {
1124    uint32_t flags;
1125    int64_t durationUs;
1126    bool hasVideo = (mVideoDecoder != NULL);
1127    // FIXME: we should handle the case where the video decoder
1128    // is created after we receive the format change indication.
1129    // Current code will just make that we select deep buffer
1130    // with video which should not be a problem as it should
1131    // not prevent from keeping A/V sync.
1132    if (!hasVideo &&
1133            mSource->getDuration(&durationUs) == OK &&
1134            durationUs
1135                > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
1136        flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1137    } else {
1138        flags = AUDIO_OUTPUT_FLAG_NONE;
1139    }
1140
1141    mOffloadAudio = mRenderer->openAudioSink(
1142            format, offloadOnly, hasVideo, flags);
1143
1144    if (mOffloadAudio) {
1145        sp<MetaData> audioMeta =
1146                mSource->getFormatMeta(true /* audio */);
1147        sendMetaDataToHal(mAudioSink, audioMeta);
1148    }
1149}
1150
1151void NuPlayer::closeAudioSink() {
1152    mRenderer->closeAudioSink();
1153}
1154
1155status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
1156    if (*decoder != NULL) {
1157        return OK;
1158    }
1159
1160    sp<AMessage> format = mSource->getFormat(audio);
1161
1162    if (format == NULL) {
1163        return -EWOULDBLOCK;
1164    }
1165
1166    if (!audio) {
1167        AString mime;
1168        CHECK(format->findString("mime", &mime));
1169        mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
1170
1171        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
1172        mCCDecoder = new CCDecoder(ccNotify);
1173
1174        if (mSourceFlags & Source::FLAG_SECURE) {
1175            format->setInt32("secure", true);
1176        }
1177    }
1178
1179    if (audio) {
1180        sp<AMessage> notify = new AMessage(kWhatAudioNotify, id());
1181        ++mAudioDecoderGeneration;
1182        notify->setInt32("generation", mAudioDecoderGeneration);
1183
1184        if (mOffloadAudio) {
1185            *decoder = new DecoderPassThrough(notify, mSource, mRenderer);
1186        } else {
1187            *decoder = new Decoder(notify, mSource, mRenderer);
1188        }
1189    } else {
1190        sp<AMessage> notify = new AMessage(kWhatVideoNotify, id());
1191        ++mVideoDecoderGeneration;
1192        notify->setInt32("generation", mVideoDecoderGeneration);
1193
1194        *decoder = new Decoder(notify, mSource, mRenderer, mNativeWindow);
1195    }
1196    (*decoder)->init();
1197    (*decoder)->configure(format);
1198
1199    // allocate buffers to decrypt widevine source buffers
1200    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1201        Vector<sp<ABuffer> > inputBufs;
1202        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1203
1204        Vector<MediaBuffer *> mediaBufs;
1205        for (size_t i = 0; i < inputBufs.size(); i++) {
1206            const sp<ABuffer> &buffer = inputBufs[i];
1207            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1208            mediaBufs.push(mbuf);
1209        }
1210
1211        status_t err = mSource->setBuffers(audio, mediaBufs);
1212        if (err != OK) {
1213            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1214                mediaBufs[i]->release();
1215            }
1216            mediaBufs.clear();
1217            ALOGE("Secure source didn't support secure mediaBufs.");
1218            return err;
1219        }
1220    }
1221    return OK;
1222}
1223
1224status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
1225    sp<AMessage> reply;
1226    CHECK(msg->findMessage("reply", &reply));
1227
1228    if ((audio && mFlushingAudio != NONE)
1229            || (!audio && mFlushingVideo != NONE)
1230            || mSource == NULL) {
1231        reply->setInt32("err", INFO_DISCONTINUITY);
1232        reply->post();
1233        return OK;
1234    }
1235
1236    sp<ABuffer> accessUnit;
1237
1238    // Aggregate smaller buffers into a larger buffer.
1239    // The goal is to reduce power consumption.
1240    // Note this will not work if the decoder requires one frame per buffer.
1241    bool doBufferAggregation = (audio && mOffloadAudio);
1242    bool needMoreData = false;
1243
1244    bool dropAccessUnit;
1245    do {
1246        status_t err;
1247        // Did we save an accessUnit earlier because of a discontinuity?
1248        if (audio && (mPendingAudioAccessUnit != NULL)) {
1249            accessUnit = mPendingAudioAccessUnit;
1250            mPendingAudioAccessUnit.clear();
1251            err = mPendingAudioErr;
1252            ALOGV("feedDecoderInputData() use mPendingAudioAccessUnit");
1253        } else {
1254            err = mSource->dequeueAccessUnit(audio, &accessUnit);
1255        }
1256
1257        if (err == -EWOULDBLOCK) {
1258            return err;
1259        } else if (err != OK) {
1260            if (err == INFO_DISCONTINUITY) {
1261                if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1262                    // We already have some data so save this for later.
1263                    mPendingAudioErr = err;
1264                    mPendingAudioAccessUnit = accessUnit;
1265                    accessUnit.clear();
1266                    ALOGD("feedDecoderInputData() save discontinuity for later");
1267                    break;
1268                }
1269                int32_t type;
1270                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
1271
1272                bool formatChange =
1273                    (audio &&
1274                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
1275                    || (!audio &&
1276                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
1277
1278                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
1279
1280                ALOGI("%s discontinuity (formatChange=%d, time=%d)",
1281                     audio ? "audio" : "video", formatChange, timeChange);
1282
1283                mTimeDiscontinuityPending =
1284                    mTimeDiscontinuityPending || timeChange;
1285
1286                bool seamlessFormatChange = false;
1287                sp<AMessage> newFormat = mSource->getFormat(audio);
1288                if (formatChange) {
1289                    seamlessFormatChange =
1290                        getDecoder(audio)->supportsSeamlessFormatChange(newFormat);
1291                    // treat seamless format change separately
1292                    formatChange = !seamlessFormatChange;
1293                }
1294                bool shutdownOrFlush = formatChange || timeChange;
1295
1296                // We want to queue up scan-sources only once per discontinuity.
1297                // We control this by doing it only if neither audio nor video are
1298                // flushing or shutting down.  (After handling 1st discontinuity, one
1299                // of the flushing states will not be NONE.)
1300                // No need to scan sources if this discontinuity does not result
1301                // in a flush or shutdown, as the flushing state will stay NONE.
1302                if (mFlushingAudio == NONE && mFlushingVideo == NONE &&
1303                        shutdownOrFlush) {
1304                    // And we'll resume scanning sources once we're done
1305                    // flushing.
1306                    mDeferredActions.push_front(
1307                            new SimpleAction(
1308                                &NuPlayer::performScanSources));
1309                }
1310
1311                if (formatChange /* not seamless */) {
1312                    // must change decoder
1313                    flushDecoder(audio, /* needShutdown = */ true);
1314                } else if (timeChange) {
1315                    // need to flush
1316                    flushDecoder(audio, /* needShutdown = */ false, newFormat);
1317                    err = OK;
1318                } else if (seamlessFormatChange) {
1319                    // reuse existing decoder and don't flush
1320                    updateDecoderFormatWithoutFlush(audio, newFormat);
1321                    err = OK;
1322                } else {
1323                    // This stream is unaffected by the discontinuity
1324                    return -EWOULDBLOCK;
1325                }
1326            }
1327
1328            reply->setInt32("err", err);
1329            reply->post();
1330            return OK;
1331        }
1332
1333        if (!audio) {
1334            ++mNumFramesTotal;
1335        }
1336
1337        dropAccessUnit = false;
1338        if (!audio
1339                && !(mSourceFlags & Source::FLAG_SECURE)
1340                && mRenderer->getVideoLateByUs() > 100000ll
1341                && mVideoIsAVC
1342                && !IsAVCReferenceFrame(accessUnit)) {
1343            dropAccessUnit = true;
1344            ++mNumFramesDropped;
1345        }
1346
1347        size_t smallSize = accessUnit->size();
1348        needMoreData = false;
1349        if (doBufferAggregation && (mAggregateBuffer == NULL)
1350                // Don't bother if only room for a few small buffers.
1351                && (smallSize < (kAggregateBufferSizeBytes / 3))) {
1352            // Create a larger buffer for combining smaller buffers from the extractor.
1353            mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes);
1354            mAggregateBuffer->setRange(0, 0); // start empty
1355        }
1356
1357        if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1358            int64_t timeUs;
1359            int64_t dummy;
1360            bool smallTimestampValid = accessUnit->meta()->findInt64("timeUs", &timeUs);
1361            bool bigTimestampValid = mAggregateBuffer->meta()->findInt64("timeUs", &dummy);
1362            // Will the smaller buffer fit?
1363            size_t bigSize = mAggregateBuffer->size();
1364            size_t roomLeft = mAggregateBuffer->capacity() - bigSize;
1365            // Should we save this small buffer for the next big buffer?
1366            // If the first small buffer did not have a timestamp then save
1367            // any buffer that does have a timestamp until the next big buffer.
1368            if ((smallSize > roomLeft)
1369                || (!bigTimestampValid && (bigSize > 0) && smallTimestampValid)) {
1370                mPendingAudioErr = err;
1371                mPendingAudioAccessUnit = accessUnit;
1372                accessUnit.clear();
1373            } else {
1374                // Grab time from first small buffer if available.
1375                if ((bigSize == 0) && smallTimestampValid) {
1376                    mAggregateBuffer->meta()->setInt64("timeUs", timeUs);
1377                }
1378                // Append small buffer to the bigger buffer.
1379                memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize);
1380                bigSize += smallSize;
1381                mAggregateBuffer->setRange(0, bigSize);
1382
1383                // Keep looping until we run out of room in the mAggregateBuffer.
1384                needMoreData = true;
1385
1386                ALOGV("feedDecoderInputData() smallSize = %zu, bigSize = %zu, capacity = %zu",
1387                        smallSize, bigSize, mAggregateBuffer->capacity());
1388            }
1389        }
1390    } while (dropAccessUnit || needMoreData);
1391
1392    // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
1393
1394#if 0
1395    int64_t mediaTimeUs;
1396    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
1397    ALOGV("feeding %s input buffer at media time %.2f secs",
1398         audio ? "audio" : "video",
1399         mediaTimeUs / 1E6);
1400#endif
1401
1402    if (!audio) {
1403        mCCDecoder->decode(accessUnit);
1404    }
1405
1406    if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1407        ALOGV("feedDecoderInputData() reply with aggregated buffer, %zu",
1408                mAggregateBuffer->size());
1409        reply->setBuffer("buffer", mAggregateBuffer);
1410        mAggregateBuffer.clear();
1411    } else {
1412        reply->setBuffer("buffer", accessUnit);
1413    }
1414
1415    reply->post();
1416
1417    return OK;
1418}
1419
1420void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
1421    // ALOGV("renderBuffer %s", audio ? "audio" : "video");
1422
1423    if ((audio && mFlushingAudio != NONE)
1424            || (!audio && mFlushingVideo != NONE)) {
1425        // We're currently attempting to flush the decoder, in order
1426        // to complete this, the decoder wants all its buffers back,
1427        // so we don't want any output buffers it sent us (from before
1428        // we initiated the flush) to be stuck in the renderer's queue.
1429
1430        ALOGV("we're still flushing the %s decoder, sending its output buffer"
1431             " right back.", audio ? "audio" : "video");
1432
1433        return;
1434    }
1435
1436    int64_t mediaTimeUs;
1437    CHECK(msg->findInt64("timeUs", &mediaTimeUs));
1438
1439    if (!audio && mCCDecoder->isSelected()) {
1440        mCCDecoder->display(mediaTimeUs);
1441    }
1442}
1443
1444void NuPlayer::updateVideoSize(
1445        const sp<AMessage> &inputFormat,
1446        const sp<AMessage> &outputFormat) {
1447    if (inputFormat == NULL) {
1448        ALOGW("Unknown video size, reporting 0x0!");
1449        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1450        return;
1451    }
1452
1453    int32_t displayWidth, displayHeight;
1454    int32_t cropLeft, cropTop, cropRight, cropBottom;
1455
1456    if (outputFormat != NULL) {
1457        int32_t width, height;
1458        CHECK(outputFormat->findInt32("width", &width));
1459        CHECK(outputFormat->findInt32("height", &height));
1460
1461        int32_t cropLeft, cropTop, cropRight, cropBottom;
1462        CHECK(outputFormat->findRect(
1463                    "crop",
1464                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1465
1466        displayWidth = cropRight - cropLeft + 1;
1467        displayHeight = cropBottom - cropTop + 1;
1468
1469        ALOGV("Video output format changed to %d x %d "
1470             "(crop: %d x %d @ (%d, %d))",
1471             width, height,
1472             displayWidth,
1473             displayHeight,
1474             cropLeft, cropTop);
1475    } else {
1476        CHECK(inputFormat->findInt32("width", &displayWidth));
1477        CHECK(inputFormat->findInt32("height", &displayHeight));
1478
1479        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1480    }
1481
1482    // Take into account sample aspect ratio if necessary:
1483    int32_t sarWidth, sarHeight;
1484    if (inputFormat->findInt32("sar-width", &sarWidth)
1485            && inputFormat->findInt32("sar-height", &sarHeight)) {
1486        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1487
1488        displayWidth = (displayWidth * sarWidth) / sarHeight;
1489
1490        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1491    }
1492
1493    int32_t rotationDegrees;
1494    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1495        rotationDegrees = 0;
1496    }
1497
1498    if (rotationDegrees == 90 || rotationDegrees == 270) {
1499        int32_t tmp = displayWidth;
1500        displayWidth = displayHeight;
1501        displayHeight = tmp;
1502    }
1503
1504    notifyListener(
1505            MEDIA_SET_VIDEO_SIZE,
1506            displayWidth,
1507            displayHeight);
1508}
1509
1510void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1511    if (mDriver == NULL) {
1512        return;
1513    }
1514
1515    sp<NuPlayerDriver> driver = mDriver.promote();
1516
1517    if (driver == NULL) {
1518        return;
1519    }
1520
1521    driver->notifyListener(msg, ext1, ext2, in);
1522}
1523
1524void NuPlayer::flushDecoder(
1525        bool audio, bool needShutdown, const sp<AMessage> &newFormat) {
1526    ALOGV("[%s] flushDecoder needShutdown=%d",
1527          audio ? "audio" : "video", needShutdown);
1528
1529    const sp<Decoder> &decoder = getDecoder(audio);
1530    if (decoder == NULL) {
1531        ALOGI("flushDecoder %s without decoder present",
1532             audio ? "audio" : "video");
1533        return;
1534    }
1535
1536    // Make sure we don't continue to scan sources until we finish flushing.
1537    ++mScanSourcesGeneration;
1538    mScanSourcesPending = false;
1539
1540    decoder->signalFlush(newFormat);
1541
1542    FlushStatus newStatus =
1543        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1544
1545    mFlushComplete[audio][false /* isDecoder */] = false;
1546    mFlushComplete[audio][true /* isDecoder */] = false;
1547    if (audio) {
1548        ALOGE_IF(mFlushingAudio != NONE,
1549                "audio flushDecoder() is called in state %d", mFlushingAudio);
1550        mFlushingAudio = newStatus;
1551    } else {
1552        ALOGE_IF(mFlushingVideo != NONE,
1553                "video flushDecoder() is called in state %d", mFlushingVideo);
1554        mFlushingVideo = newStatus;
1555
1556        if (mCCDecoder != NULL) {
1557            mCCDecoder->flush();
1558        }
1559    }
1560}
1561
1562void NuPlayer::updateDecoderFormatWithoutFlush(
1563        bool audio, const sp<AMessage> &format) {
1564    ALOGV("[%s] updateDecoderFormatWithoutFlush", audio ? "audio" : "video");
1565
1566    const sp<Decoder> &decoder = getDecoder(audio);
1567    if (decoder == NULL) {
1568        ALOGI("updateDecoderFormatWithoutFlush %s without decoder present",
1569             audio ? "audio" : "video");
1570        return;
1571    }
1572
1573    decoder->signalUpdateFormat(format);
1574}
1575
1576void NuPlayer::queueDecoderShutdown(
1577        bool audio, bool video, const sp<AMessage> &reply) {
1578    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1579
1580    mDeferredActions.push_back(
1581            new FlushDecoderAction(
1582                audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
1583                video ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE));
1584
1585    mDeferredActions.push_back(
1586            new SimpleAction(&NuPlayer::performScanSources));
1587
1588    mDeferredActions.push_back(new PostMessageAction(reply));
1589
1590    processDeferredActions();
1591}
1592
1593status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1594    mVideoScalingMode = mode;
1595    if (mNativeWindow != NULL) {
1596        status_t ret = native_window_set_scaling_mode(
1597                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1598        if (ret != OK) {
1599            ALOGE("Failed to set scaling mode (%d): %s",
1600                -ret, strerror(-ret));
1601            return ret;
1602        }
1603    }
1604    return OK;
1605}
1606
1607status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1608    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id());
1609    msg->setPointer("reply", reply);
1610
1611    sp<AMessage> response;
1612    status_t err = msg->postAndAwaitResponse(&response);
1613    return err;
1614}
1615
1616status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1617    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id());
1618    msg->setPointer("reply", reply);
1619    msg->setInt32("type", type);
1620
1621    sp<AMessage> response;
1622    status_t err = msg->postAndAwaitResponse(&response);
1623    if (err == OK && response != NULL) {
1624        CHECK(response->findInt32("err", &err));
1625    }
1626    return err;
1627}
1628
1629status_t NuPlayer::selectTrack(size_t trackIndex, bool select, int64_t timeUs) {
1630    sp<AMessage> msg = new AMessage(kWhatSelectTrack, id());
1631    msg->setSize("trackIndex", trackIndex);
1632    msg->setInt32("select", select);
1633    msg->setInt64("timeUs", timeUs);
1634
1635    sp<AMessage> response;
1636    status_t err = msg->postAndAwaitResponse(&response);
1637
1638    if (err != OK) {
1639        return err;
1640    }
1641
1642    if (!response->findInt32("err", &err)) {
1643        err = OK;
1644    }
1645
1646    return err;
1647}
1648
1649status_t NuPlayer::getCurrentPosition(int64_t *mediaUs) {
1650    sp<Renderer> renderer = mRenderer;
1651    if (renderer == NULL) {
1652        return NO_INIT;
1653    }
1654
1655    return renderer->getCurrentPosition(mediaUs);
1656}
1657
1658void NuPlayer::getStats(int64_t *numFramesTotal, int64_t *numFramesDropped) {
1659    *numFramesTotal = mNumFramesTotal;
1660    *numFramesDropped = mNumFramesDropped;
1661}
1662
1663sp<MetaData> NuPlayer::getFileMeta() {
1664    return mSource->getFileFormatMeta();
1665}
1666
1667void NuPlayer::schedulePollDuration() {
1668    sp<AMessage> msg = new AMessage(kWhatPollDuration, id());
1669    msg->setInt32("generation", mPollDurationGeneration);
1670    msg->post();
1671}
1672
1673void NuPlayer::cancelPollDuration() {
1674    ++mPollDurationGeneration;
1675}
1676
1677void NuPlayer::processDeferredActions() {
1678    while (!mDeferredActions.empty()) {
1679        // We won't execute any deferred actions until we're no longer in
1680        // an intermediate state, i.e. one more more decoders are currently
1681        // flushing or shutting down.
1682
1683        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1684            // We're currently flushing, postpone the reset until that's
1685            // completed.
1686
1687            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1688                  mFlushingAudio, mFlushingVideo);
1689
1690            break;
1691        }
1692
1693        sp<Action> action = *mDeferredActions.begin();
1694        mDeferredActions.erase(mDeferredActions.begin());
1695
1696        action->execute(this);
1697    }
1698}
1699
1700void NuPlayer::performSeek(int64_t seekTimeUs, bool needNotify) {
1701    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs), needNotify(%d)",
1702          seekTimeUs,
1703          seekTimeUs / 1E6,
1704          needNotify);
1705
1706    if (mSource == NULL) {
1707        // This happens when reset occurs right before the loop mode
1708        // asynchronously seeks to the start of the stream.
1709        LOG_ALWAYS_FATAL_IF(mAudioDecoder != NULL || mVideoDecoder != NULL,
1710                "mSource is NULL and decoders not NULL audio(%p) video(%p)",
1711                mAudioDecoder.get(), mVideoDecoder.get());
1712        return;
1713    }
1714    mSource->seekTo(seekTimeUs);
1715    ++mTimedTextGeneration;
1716
1717    if (mDriver != NULL) {
1718        sp<NuPlayerDriver> driver = mDriver.promote();
1719        if (driver != NULL) {
1720            if (needNotify) {
1721                driver->notifySeekComplete();
1722            }
1723        }
1724    }
1725
1726    // everything's flushed, continue playback.
1727}
1728
1729void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) {
1730    ALOGV("performDecoderFlush audio=%d, video=%d", audio, video);
1731
1732    if ((audio == FLUSH_CMD_NONE || mAudioDecoder == NULL)
1733            && (video == FLUSH_CMD_NONE || mVideoDecoder == NULL)) {
1734        return;
1735    }
1736
1737    mTimeDiscontinuityPending = true;
1738
1739    if (audio != FLUSH_CMD_NONE && mAudioDecoder != NULL) {
1740        flushDecoder(true /* audio */, (audio == FLUSH_CMD_SHUTDOWN));
1741    }
1742
1743    if (video != FLUSH_CMD_NONE && mVideoDecoder != NULL) {
1744        flushDecoder(false /* audio */, (video == FLUSH_CMD_SHUTDOWN));
1745    }
1746}
1747
1748void NuPlayer::performReset() {
1749    ALOGV("performReset");
1750
1751    CHECK(mAudioDecoder == NULL);
1752    CHECK(mVideoDecoder == NULL);
1753
1754    cancelPollDuration();
1755
1756    ++mScanSourcesGeneration;
1757    mScanSourcesPending = false;
1758
1759    if (mRendererLooper != NULL) {
1760        if (mRenderer != NULL) {
1761            mRendererLooper->unregisterHandler(mRenderer->id());
1762        }
1763        mRendererLooper->stop();
1764        mRendererLooper.clear();
1765    }
1766    mRenderer.clear();
1767    ++mRendererGeneration;
1768
1769    if (mSource != NULL) {
1770        mSource->stop();
1771
1772        mSource.clear();
1773    }
1774
1775    if (mDriver != NULL) {
1776        sp<NuPlayerDriver> driver = mDriver.promote();
1777        if (driver != NULL) {
1778            driver->notifyResetComplete();
1779        }
1780    }
1781
1782    mStarted = false;
1783}
1784
1785void NuPlayer::performScanSources() {
1786    ALOGV("performScanSources");
1787
1788    if (!mStarted) {
1789        return;
1790    }
1791
1792    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1793        postScanSources();
1794    }
1795}
1796
1797void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1798    ALOGV("performSetSurface");
1799
1800    mNativeWindow = wrapper;
1801
1802    // XXX - ignore error from setVideoScalingMode for now
1803    setVideoScalingMode(mVideoScalingMode);
1804
1805    if (mDriver != NULL) {
1806        sp<NuPlayerDriver> driver = mDriver.promote();
1807        if (driver != NULL) {
1808            driver->notifySetSurfaceComplete();
1809        }
1810    }
1811}
1812
1813void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1814    int32_t what;
1815    CHECK(msg->findInt32("what", &what));
1816
1817    switch (what) {
1818        case Source::kWhatPrepared:
1819        {
1820            if (mSource == NULL) {
1821                // This is a stale notification from a source that was
1822                // asynchronously preparing when the client called reset().
1823                // We handled the reset, the source is gone.
1824                break;
1825            }
1826
1827            int32_t err;
1828            CHECK(msg->findInt32("err", &err));
1829
1830            sp<NuPlayerDriver> driver = mDriver.promote();
1831            if (driver != NULL) {
1832                // notify duration first, so that it's definitely set when
1833                // the app received the "prepare complete" callback.
1834                int64_t durationUs;
1835                if (mSource->getDuration(&durationUs) == OK) {
1836                    driver->notifyDuration(durationUs);
1837                }
1838                driver->notifyPrepareCompleted(err);
1839            }
1840
1841            break;
1842        }
1843
1844        case Source::kWhatFlagsChanged:
1845        {
1846            uint32_t flags;
1847            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1848
1849            sp<NuPlayerDriver> driver = mDriver.promote();
1850            if (driver != NULL) {
1851                driver->notifyFlagsChanged(flags);
1852            }
1853
1854            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1855                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
1856                cancelPollDuration();
1857            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1858                    && (flags & Source::FLAG_DYNAMIC_DURATION)
1859                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
1860                schedulePollDuration();
1861            }
1862
1863            mSourceFlags = flags;
1864            break;
1865        }
1866
1867        case Source::kWhatVideoSizeChanged:
1868        {
1869            sp<AMessage> format;
1870            CHECK(msg->findMessage("format", &format));
1871
1872            updateVideoSize(format);
1873            break;
1874        }
1875
1876        case Source::kWhatBufferingUpdate:
1877        {
1878            int32_t percentage;
1879            CHECK(msg->findInt32("percentage", &percentage));
1880
1881            notifyListener(MEDIA_BUFFERING_UPDATE, percentage, 0);
1882            break;
1883        }
1884
1885        case Source::kWhatBufferingStart:
1886        {
1887            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
1888            break;
1889        }
1890
1891        case Source::kWhatBufferingEnd:
1892        {
1893            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
1894            break;
1895        }
1896
1897        case Source::kWhatSubtitleData:
1898        {
1899            sp<ABuffer> buffer;
1900            CHECK(msg->findBuffer("buffer", &buffer));
1901
1902            sendSubtitleData(buffer, 0 /* baseIndex */);
1903            break;
1904        }
1905
1906        case Source::kWhatTimedTextData:
1907        {
1908            int32_t generation;
1909            if (msg->findInt32("generation", &generation)
1910                    && generation != mTimedTextGeneration) {
1911                break;
1912            }
1913
1914            sp<ABuffer> buffer;
1915            CHECK(msg->findBuffer("buffer", &buffer));
1916
1917            sp<NuPlayerDriver> driver = mDriver.promote();
1918            if (driver == NULL) {
1919                break;
1920            }
1921
1922            int posMs;
1923            int64_t timeUs, posUs;
1924            driver->getCurrentPosition(&posMs);
1925            posUs = posMs * 1000;
1926            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1927
1928            if (posUs < timeUs) {
1929                if (!msg->findInt32("generation", &generation)) {
1930                    msg->setInt32("generation", mTimedTextGeneration);
1931                }
1932                msg->post(timeUs - posUs);
1933            } else {
1934                sendTimedTextData(buffer);
1935            }
1936            break;
1937        }
1938
1939        case Source::kWhatQueueDecoderShutdown:
1940        {
1941            int32_t audio, video;
1942            CHECK(msg->findInt32("audio", &audio));
1943            CHECK(msg->findInt32("video", &video));
1944
1945            sp<AMessage> reply;
1946            CHECK(msg->findMessage("reply", &reply));
1947
1948            queueDecoderShutdown(audio, video, reply);
1949            break;
1950        }
1951
1952        case Source::kWhatDrmNoLicense:
1953        {
1954            notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, ERROR_DRM_NO_LICENSE);
1955            break;
1956        }
1957
1958        default:
1959            TRESPASS();
1960    }
1961}
1962
1963void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
1964    int32_t what;
1965    CHECK(msg->findInt32("what", &what));
1966
1967    switch (what) {
1968        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
1969        {
1970            sp<ABuffer> buffer;
1971            CHECK(msg->findBuffer("buffer", &buffer));
1972
1973            size_t inbandTracks = 0;
1974            if (mSource != NULL) {
1975                inbandTracks = mSource->getTrackCount();
1976            }
1977
1978            sendSubtitleData(buffer, inbandTracks);
1979            break;
1980        }
1981
1982        case NuPlayer::CCDecoder::kWhatTrackAdded:
1983        {
1984            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
1985
1986            break;
1987        }
1988
1989        default:
1990            TRESPASS();
1991    }
1992
1993
1994}
1995
1996void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
1997    int32_t trackIndex;
1998    int64_t timeUs, durationUs;
1999    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
2000    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2001    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
2002
2003    Parcel in;
2004    in.writeInt32(trackIndex + baseIndex);
2005    in.writeInt64(timeUs);
2006    in.writeInt64(durationUs);
2007    in.writeInt32(buffer->size());
2008    in.writeInt32(buffer->size());
2009    in.write(buffer->data(), buffer->size());
2010
2011    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
2012}
2013
2014void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
2015    const void *data;
2016    size_t size = 0;
2017    int64_t timeUs;
2018    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
2019
2020    AString mime;
2021    CHECK(buffer->meta()->findString("mime", &mime));
2022    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
2023
2024    data = buffer->data();
2025    size = buffer->size();
2026
2027    Parcel parcel;
2028    if (size > 0) {
2029        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2030        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
2031        TextDescriptions::getParcelOfDescriptions(
2032                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
2033    }
2034
2035    if ((parcel.dataSize() > 0)) {
2036        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
2037    } else {  // send an empty timed text
2038        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
2039    }
2040}
2041////////////////////////////////////////////////////////////////////////////////
2042
2043sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
2044    sp<MetaData> meta = getFormatMeta(audio);
2045
2046    if (meta == NULL) {
2047        return NULL;
2048    }
2049
2050    sp<AMessage> msg = new AMessage;
2051
2052    if(convertMetaDataToMessage(meta, &msg) == OK) {
2053        return msg;
2054    }
2055    return NULL;
2056}
2057
2058void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
2059    sp<AMessage> notify = dupNotify();
2060    notify->setInt32("what", kWhatFlagsChanged);
2061    notify->setInt32("flags", flags);
2062    notify->post();
2063}
2064
2065void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
2066    sp<AMessage> notify = dupNotify();
2067    notify->setInt32("what", kWhatVideoSizeChanged);
2068    notify->setMessage("format", format);
2069    notify->post();
2070}
2071
2072void NuPlayer::Source::notifyPrepared(status_t err) {
2073    sp<AMessage> notify = dupNotify();
2074    notify->setInt32("what", kWhatPrepared);
2075    notify->setInt32("err", err);
2076    notify->post();
2077}
2078
2079void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
2080    TRESPASS();
2081}
2082
2083}  // namespace android
2084