NuPlayer.cpp revision a10fd23bb9fcf16e778c639ea5638e2917dacd89 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerDecoder.h"
25#include "NuPlayerDecoderPassThrough.h"
26#include "NuPlayerDriver.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29#include "RTSPSource.h"
30#include "StreamingSource.h"
31#include "GenericSource.h"
32#include "TextDescriptions.h"
33
34#include "ATSParser.h"
35
36#include <media/stagefright/foundation/hexdump.h>
37#include <media/stagefright/foundation/ABuffer.h>
38#include <media/stagefright/foundation/ADebug.h>
39#include <media/stagefright/foundation/AMessage.h>
40#include <media/stagefright/MediaBuffer.h>
41#include <media/stagefright/MediaDefs.h>
42#include <media/stagefright/MediaErrors.h>
43#include <media/stagefright/MetaData.h>
44#include <gui/IGraphicBufferProducer.h>
45
46#include "avc_utils.h"
47
48#include "ESDS.h"
49#include <media/stagefright/Utils.h>
50
51namespace android {
52
53// TODO optimize buffer size for power consumption
54// The offload read buffer size is 32 KB but 24 KB uses less power.
55const size_t NuPlayer::kAggregateBufferSizeBytes = 24 * 1024;
56
57struct NuPlayer::Action : public RefBase {
58    Action() {}
59
60    virtual void execute(NuPlayer *player) = 0;
61
62private:
63    DISALLOW_EVIL_CONSTRUCTORS(Action);
64};
65
66struct NuPlayer::SeekAction : public Action {
67    SeekAction(int64_t seekTimeUs, bool needNotify)
68        : mSeekTimeUs(seekTimeUs),
69          mNeedNotify(needNotify) {
70    }
71
72    virtual void execute(NuPlayer *player) {
73        player->performSeek(mSeekTimeUs, mNeedNotify);
74    }
75
76private:
77    int64_t mSeekTimeUs;
78    bool mNeedNotify;
79
80    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
81};
82
83struct NuPlayer::SetSurfaceAction : public Action {
84    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
85        : mWrapper(wrapper) {
86    }
87
88    virtual void execute(NuPlayer *player) {
89        player->performSetSurface(mWrapper);
90    }
91
92private:
93    sp<NativeWindowWrapper> mWrapper;
94
95    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
96};
97
98struct NuPlayer::FlushDecoderAction : public Action {
99    FlushDecoderAction(FlushCommand audio, FlushCommand video)
100        : mAudio(audio),
101          mVideo(video) {
102    }
103
104    virtual void execute(NuPlayer *player) {
105        player->performDecoderFlush(mAudio, mVideo);
106    }
107
108private:
109    FlushCommand mAudio;
110    FlushCommand mVideo;
111
112    DISALLOW_EVIL_CONSTRUCTORS(FlushDecoderAction);
113};
114
115struct NuPlayer::PostMessageAction : public Action {
116    PostMessageAction(const sp<AMessage> &msg)
117        : mMessage(msg) {
118    }
119
120    virtual void execute(NuPlayer *) {
121        mMessage->post();
122    }
123
124private:
125    sp<AMessage> mMessage;
126
127    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
128};
129
130// Use this if there's no state necessary to save in order to execute
131// the action.
132struct NuPlayer::SimpleAction : public Action {
133    typedef void (NuPlayer::*ActionFunc)();
134
135    SimpleAction(ActionFunc func)
136        : mFunc(func) {
137    }
138
139    virtual void execute(NuPlayer *player) {
140        (player->*mFunc)();
141    }
142
143private:
144    ActionFunc mFunc;
145
146    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
147};
148
149////////////////////////////////////////////////////////////////////////////////
150
151NuPlayer::NuPlayer()
152    : mUIDValid(false),
153      mSourceFlags(0),
154      mVideoIsAVC(false),
155      mOffloadAudio(false),
156      mAudioDecoderGeneration(0),
157      mVideoDecoderGeneration(0),
158      mRendererGeneration(0),
159      mAudioEOS(false),
160      mVideoEOS(false),
161      mScanSourcesPending(false),
162      mScanSourcesGeneration(0),
163      mPollDurationGeneration(0),
164      mTimedTextGeneration(0),
165      mTimeDiscontinuityPending(false),
166      mFlushingAudio(NONE),
167      mFlushingVideo(NONE),
168      mNumFramesTotal(0ll),
169      mNumFramesDropped(0ll),
170      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
171      mStarted(false) {
172    clearFlushComplete();
173}
174
175NuPlayer::~NuPlayer() {
176}
177
178void NuPlayer::setUID(uid_t uid) {
179    mUIDValid = true;
180    mUID = uid;
181}
182
183void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
184    mDriver = driver;
185}
186
187void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
188    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
189
190    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
191
192    msg->setObject("source", new StreamingSource(notify, source));
193    msg->post();
194}
195
196static bool IsHTTPLiveURL(const char *url) {
197    if (!strncasecmp("http://", url, 7)
198            || !strncasecmp("https://", url, 8)
199            || !strncasecmp("file://", url, 7)) {
200        size_t len = strlen(url);
201        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
202            return true;
203        }
204
205        if (strstr(url,"m3u8")) {
206            return true;
207        }
208    }
209
210    return false;
211}
212
213void NuPlayer::setDataSourceAsync(
214        const sp<IMediaHTTPService> &httpService,
215        const char *url,
216        const KeyedVector<String8, String8> *headers) {
217
218    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
219    size_t len = strlen(url);
220
221    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
222
223    sp<Source> source;
224    if (IsHTTPLiveURL(url)) {
225        source = new HTTPLiveSource(notify, httpService, url, headers);
226    } else if (!strncasecmp(url, "rtsp://", 7)) {
227        source = new RTSPSource(
228                notify, httpService, url, headers, mUIDValid, mUID);
229    } else if ((!strncasecmp(url, "http://", 7)
230                || !strncasecmp(url, "https://", 8))
231                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
232                    || strstr(url, ".sdp?"))) {
233        source = new RTSPSource(
234                notify, httpService, url, headers, mUIDValid, mUID, true);
235    } else {
236        sp<GenericSource> genericSource =
237                new GenericSource(notify, mUIDValid, mUID);
238        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
239        // The correct flags will be updated in Source::kWhatFlagsChanged
240        // handler when  GenericSource is prepared.
241
242        status_t err = genericSource->setDataSource(httpService, url, headers);
243
244        if (err == OK) {
245            source = genericSource;
246        } else {
247            ALOGE("Failed to set data source!");
248        }
249    }
250    msg->setObject("source", source);
251    msg->post();
252}
253
254void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
255    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
256
257    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
258
259    sp<GenericSource> source =
260            new GenericSource(notify, mUIDValid, mUID);
261
262    status_t err = source->setDataSource(fd, offset, length);
263
264    if (err != OK) {
265        ALOGE("Failed to set data source!");
266        source = NULL;
267    }
268
269    msg->setObject("source", source);
270    msg->post();
271}
272
273void NuPlayer::prepareAsync() {
274    (new AMessage(kWhatPrepare, id()))->post();
275}
276
277void NuPlayer::setVideoSurfaceTextureAsync(
278        const sp<IGraphicBufferProducer> &bufferProducer) {
279    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id());
280
281    if (bufferProducer == NULL) {
282        msg->setObject("native-window", NULL);
283    } else {
284        msg->setObject(
285                "native-window",
286                new NativeWindowWrapper(
287                    new Surface(bufferProducer, true /* controlledByApp */)));
288    }
289
290    msg->post();
291}
292
293void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
294    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id());
295    msg->setObject("sink", sink);
296    msg->post();
297}
298
299void NuPlayer::start() {
300    (new AMessage(kWhatStart, id()))->post();
301}
302
303void NuPlayer::pause() {
304    (new AMessage(kWhatPause, id()))->post();
305}
306
307void NuPlayer::resetAsync() {
308    if (mSource != NULL) {
309        // During a reset, the data source might be unresponsive already, we need to
310        // disconnect explicitly so that reads exit promptly.
311        // We can't queue the disconnect request to the looper, as it might be
312        // queued behind a stuck read and never gets processed.
313        // Doing a disconnect outside the looper to allows the pending reads to exit
314        // (either successfully or with error).
315        mSource->disconnect();
316    }
317
318    (new AMessage(kWhatReset, id()))->post();
319}
320
321void NuPlayer::seekToAsync(int64_t seekTimeUs, bool needNotify) {
322    sp<AMessage> msg = new AMessage(kWhatSeek, id());
323    msg->setInt64("seekTimeUs", seekTimeUs);
324    msg->setInt32("needNotify", needNotify);
325    msg->post();
326}
327
328
329void NuPlayer::writeTrackInfo(
330        Parcel* reply, const sp<AMessage> format) const {
331    int32_t trackType;
332    CHECK(format->findInt32("type", &trackType));
333
334    AString lang;
335    CHECK(format->findString("language", &lang));
336
337    reply->writeInt32(2); // write something non-zero
338    reply->writeInt32(trackType);
339    reply->writeString16(String16(lang.c_str()));
340
341    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
342        AString mime;
343        CHECK(format->findString("mime", &mime));
344
345        int32_t isAuto, isDefault, isForced;
346        CHECK(format->findInt32("auto", &isAuto));
347        CHECK(format->findInt32("default", &isDefault));
348        CHECK(format->findInt32("forced", &isForced));
349
350        reply->writeString16(String16(mime.c_str()));
351        reply->writeInt32(isAuto);
352        reply->writeInt32(isDefault);
353        reply->writeInt32(isForced);
354    }
355}
356
357void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
358    switch (msg->what()) {
359        case kWhatSetDataSource:
360        {
361            ALOGV("kWhatSetDataSource");
362
363            CHECK(mSource == NULL);
364
365            status_t err = OK;
366            sp<RefBase> obj;
367            CHECK(msg->findObject("source", &obj));
368            if (obj != NULL) {
369                mSource = static_cast<Source *>(obj.get());
370            } else {
371                err = UNKNOWN_ERROR;
372            }
373
374            CHECK(mDriver != NULL);
375            sp<NuPlayerDriver> driver = mDriver.promote();
376            if (driver != NULL) {
377                driver->notifySetDataSourceCompleted(err);
378            }
379            break;
380        }
381
382        case kWhatPrepare:
383        {
384            mSource->prepareAsync();
385            break;
386        }
387
388        case kWhatGetTrackInfo:
389        {
390            uint32_t replyID;
391            CHECK(msg->senderAwaitsResponse(&replyID));
392
393            Parcel* reply;
394            CHECK(msg->findPointer("reply", (void**)&reply));
395
396            size_t inbandTracks = 0;
397            if (mSource != NULL) {
398                inbandTracks = mSource->getTrackCount();
399            }
400
401            size_t ccTracks = 0;
402            if (mCCDecoder != NULL) {
403                ccTracks = mCCDecoder->getTrackCount();
404            }
405
406            // total track count
407            reply->writeInt32(inbandTracks + ccTracks);
408
409            // write inband tracks
410            for (size_t i = 0; i < inbandTracks; ++i) {
411                writeTrackInfo(reply, mSource->getTrackInfo(i));
412            }
413
414            // write CC track
415            for (size_t i = 0; i < ccTracks; ++i) {
416                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
417            }
418
419            sp<AMessage> response = new AMessage;
420            response->postReply(replyID);
421            break;
422        }
423
424        case kWhatGetSelectedTrack:
425        {
426            status_t err = INVALID_OPERATION;
427            if (mSource != NULL) {
428                err = OK;
429
430                int32_t type32;
431                CHECK(msg->findInt32("type", (int32_t*)&type32));
432                media_track_type type = (media_track_type)type32;
433                ssize_t selectedTrack = mSource->getSelectedTrack(type);
434
435                Parcel* reply;
436                CHECK(msg->findPointer("reply", (void**)&reply));
437                reply->writeInt32(selectedTrack);
438            }
439
440            sp<AMessage> response = new AMessage;
441            response->setInt32("err", err);
442
443            uint32_t replyID;
444            CHECK(msg->senderAwaitsResponse(&replyID));
445            response->postReply(replyID);
446            break;
447        }
448
449        case kWhatSelectTrack:
450        {
451            uint32_t replyID;
452            CHECK(msg->senderAwaitsResponse(&replyID));
453
454            size_t trackIndex;
455            int32_t select;
456            CHECK(msg->findSize("trackIndex", &trackIndex));
457            CHECK(msg->findInt32("select", &select));
458
459            status_t err = INVALID_OPERATION;
460
461            size_t inbandTracks = 0;
462            if (mSource != NULL) {
463                inbandTracks = mSource->getTrackCount();
464            }
465            size_t ccTracks = 0;
466            if (mCCDecoder != NULL) {
467                ccTracks = mCCDecoder->getTrackCount();
468            }
469
470            if (trackIndex < inbandTracks) {
471                err = mSource->selectTrack(trackIndex, select);
472
473                if (!select && err == OK) {
474                    int32_t type;
475                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
476                    if (info != NULL
477                            && info->findInt32("type", &type)
478                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
479                        ++mTimedTextGeneration;
480                    }
481                }
482            } else {
483                trackIndex -= inbandTracks;
484
485                if (trackIndex < ccTracks) {
486                    err = mCCDecoder->selectTrack(trackIndex, select);
487                }
488            }
489
490            sp<AMessage> response = new AMessage;
491            response->setInt32("err", err);
492
493            response->postReply(replyID);
494            break;
495        }
496
497        case kWhatPollDuration:
498        {
499            int32_t generation;
500            CHECK(msg->findInt32("generation", &generation));
501
502            if (generation != mPollDurationGeneration) {
503                // stale
504                break;
505            }
506
507            int64_t durationUs;
508            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
509                sp<NuPlayerDriver> driver = mDriver.promote();
510                if (driver != NULL) {
511                    driver->notifyDuration(durationUs);
512                }
513            }
514
515            msg->post(1000000ll);  // poll again in a second.
516            break;
517        }
518
519        case kWhatSetVideoNativeWindow:
520        {
521            ALOGV("kWhatSetVideoNativeWindow");
522
523            sp<RefBase> obj;
524            CHECK(msg->findObject("native-window", &obj));
525
526            if (mSource->getFormat(false /* audio */) == NULL) {
527                performSetSurface(static_cast<NativeWindowWrapper *>(obj.get()));
528                break;
529            }
530
531            mDeferredActions.push_back(
532                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
533                                           FLUSH_CMD_SHUTDOWN /* video */));
534
535            mDeferredActions.push_back(
536                    new SetSurfaceAction(
537                        static_cast<NativeWindowWrapper *>(obj.get())));
538
539            if (obj != NULL) {
540                if (mStarted) {
541                    // Issue a seek to refresh the video screen only if started otherwise
542                    // the extractor may not yet be started and will assert.
543                    // If the video decoder is not set (perhaps audio only in this case)
544                    // do not perform a seek as it is not needed.
545                    int64_t currentPositionUs = 0;
546                    if (getCurrentPosition(&currentPositionUs) == OK) {
547                        mDeferredActions.push_back(
548                                new SeekAction(currentPositionUs, false /* needNotify */));
549                    }
550                }
551
552                // If there is a new surface texture, instantiate decoders
553                // again if possible.
554                mDeferredActions.push_back(
555                        new SimpleAction(&NuPlayer::performScanSources));
556            }
557
558            processDeferredActions();
559            break;
560        }
561
562        case kWhatSetAudioSink:
563        {
564            ALOGV("kWhatSetAudioSink");
565
566            sp<RefBase> obj;
567            CHECK(msg->findObject("sink", &obj));
568
569            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
570            break;
571        }
572
573        case kWhatStart:
574        {
575            ALOGV("kWhatStart");
576            if (mStarted) {
577                onResume();
578            } else {
579                onStart();
580            }
581            break;
582        }
583
584        case kWhatScanSources:
585        {
586            int32_t generation;
587            CHECK(msg->findInt32("generation", &generation));
588            if (generation != mScanSourcesGeneration) {
589                // Drop obsolete msg.
590                break;
591            }
592
593            mScanSourcesPending = false;
594
595            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
596                 mAudioDecoder != NULL, mVideoDecoder != NULL);
597
598            bool mHadAnySourcesBefore =
599                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
600
601            // initialize video before audio because successful initialization of
602            // video may change deep buffer mode of audio.
603            if (mNativeWindow != NULL) {
604                instantiateDecoder(false, &mVideoDecoder);
605            }
606
607            // Don't try to re-open audio sink if there's an existing decoder.
608            if (mAudioSink != NULL && mAudioDecoder == NULL) {
609                sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
610                sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
611                audio_stream_type_t streamType = mAudioSink->getAudioStreamType();
612                bool canOffload = canOffloadStream(audioMeta, (videoFormat != NULL),
613                         true /* is_streaming */, streamType);
614                if (canOffload) {
615                    if (!mOffloadAudio) {
616                        mRenderer->signalEnableOffloadAudio();
617                    }
618                    // open audio sink early under offload mode.
619                    sp<AMessage> format = mSource->getFormat(true /*audio*/);
620                    openAudioSink(format, true /*offloadOnly*/);
621                }
622                instantiateDecoder(true, &mAudioDecoder);
623            }
624
625            if (!mHadAnySourcesBefore
626                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
627                // This is the first time we've found anything playable.
628
629                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
630                    schedulePollDuration();
631                }
632            }
633
634            status_t err;
635            if ((err = mSource->feedMoreTSData()) != OK) {
636                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
637                    // We're not currently decoding anything (no audio or
638                    // video tracks found) and we just ran out of input data.
639
640                    if (err == ERROR_END_OF_STREAM) {
641                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
642                    } else {
643                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
644                    }
645                }
646                break;
647            }
648
649            if ((mAudioDecoder == NULL && mAudioSink != NULL)
650                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
651                msg->post(100000ll);
652                mScanSourcesPending = true;
653            }
654            break;
655        }
656
657        case kWhatVideoNotify:
658        case kWhatAudioNotify:
659        {
660            bool audio = msg->what() == kWhatAudioNotify;
661
662            int32_t currentDecoderGeneration =
663                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
664            int32_t requesterGeneration = currentDecoderGeneration - 1;
665            CHECK(msg->findInt32("generation", &requesterGeneration));
666
667            if (requesterGeneration != currentDecoderGeneration) {
668                ALOGV("got message from old %s decoder, generation(%d:%d)",
669                        audio ? "audio" : "video", requesterGeneration,
670                        currentDecoderGeneration);
671                sp<AMessage> reply;
672                if (!(msg->findMessage("reply", &reply))) {
673                    return;
674                }
675
676                reply->setInt32("err", INFO_DISCONTINUITY);
677                reply->post();
678                return;
679            }
680
681            int32_t what;
682            CHECK(msg->findInt32("what", &what));
683
684            if (what == Decoder::kWhatFillThisBuffer) {
685                status_t err = feedDecoderInputData(
686                        audio, msg);
687
688                if (err == -EWOULDBLOCK) {
689                    if (mSource->feedMoreTSData() == OK) {
690                        msg->post(10 * 1000ll);
691                    }
692                }
693            } else if (what == Decoder::kWhatEOS) {
694                int32_t err;
695                CHECK(msg->findInt32("err", &err));
696
697                if (err == ERROR_END_OF_STREAM) {
698                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
699                } else {
700                    ALOGV("got %s decoder EOS w/ error %d",
701                         audio ? "audio" : "video",
702                         err);
703                }
704
705                mRenderer->queueEOS(audio, err);
706            } else if (what == Decoder::kWhatFlushCompleted) {
707                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
708
709                handleFlushComplete(audio, true /* isDecoder */);
710                finishFlushIfPossible();
711            } else if (what == Decoder::kWhatVideoSizeChanged) {
712                sp<AMessage> format;
713                CHECK(msg->findMessage("format", &format));
714
715                sp<AMessage> inputFormat =
716                        mSource->getFormat(false /* audio */);
717
718                updateVideoSize(inputFormat, format);
719            } else if (what == Decoder::kWhatShutdownCompleted) {
720                ALOGV("%s shutdown completed", audio ? "audio" : "video");
721                if (audio) {
722                    mAudioDecoder.clear();
723                    ++mAudioDecoderGeneration;
724
725                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
726                    mFlushingAudio = SHUT_DOWN;
727                } else {
728                    mVideoDecoder.clear();
729                    ++mVideoDecoderGeneration;
730
731                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
732                    mFlushingVideo = SHUT_DOWN;
733                }
734
735                finishFlushIfPossible();
736            } else if (what == Decoder::kWhatError) {
737                status_t err;
738                if (!msg->findInt32("err", &err) || err == OK) {
739                    err = UNKNOWN_ERROR;
740                }
741
742                // Decoder errors can be due to Source (e.g. from streaming),
743                // or from decoding corrupted bitstreams, or from other decoder
744                // MediaCodec operations (e.g. from an ongoing reset or seek).
745                //
746                // We try to gracefully shut down the affected decoder if possible,
747                // rather than trying to force the shutdown with something
748                // similar to performReset(). This method can lead to a hang
749                // if MediaCodec functions block after an error, but they should
750                // typically return INVALID_OPERATION instead of blocking.
751
752                FlushStatus *flushing = audio ? &mFlushingAudio : &mFlushingVideo;
753                ALOGE("received error(%#x) from %s decoder, flushing(%d), now shutting down",
754                        err, audio ? "audio" : "video", *flushing);
755
756                switch (*flushing) {
757                    case NONE:
758                        mDeferredActions.push_back(
759                                new FlushDecoderAction(
760                                    audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
761                                    audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN));
762                        processDeferredActions();
763                        break;
764                    case FLUSHING_DECODER:
765                        *flushing = FLUSHING_DECODER_SHUTDOWN; // initiate shutdown after flush.
766                        break; // Wait for flush to complete.
767                    case FLUSHING_DECODER_SHUTDOWN:
768                        break; // Wait for flush to complete.
769                    case SHUTTING_DOWN_DECODER:
770                        break; // Wait for shutdown to complete.
771                    case FLUSHED:
772                        // Widevine source reads must stop before releasing the video decoder.
773                        if (!audio && mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
774                            mSource->stop();
775                        }
776                        getDecoder(audio)->initiateShutdown(); // In the middle of a seek.
777                        *flushing = SHUTTING_DOWN_DECODER;     // Shut down.
778                        break;
779                    case SHUT_DOWN:
780                        finishFlushIfPossible();  // Should not occur.
781                        break;                    // Finish anyways.
782                }
783                notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
784            } else if (what == Decoder::kWhatRenderBufferTime) {
785                renderBuffer(audio, msg);
786            } else {
787                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
788                      what,
789                      what >> 24,
790                      (what >> 16) & 0xff,
791                      (what >> 8) & 0xff,
792                      what & 0xff);
793            }
794
795            break;
796        }
797
798        case kWhatRendererNotify:
799        {
800            int32_t requesterGeneration = mRendererGeneration - 1;
801            CHECK(msg->findInt32("generation", &requesterGeneration));
802            if (requesterGeneration != mRendererGeneration) {
803                ALOGV("got message from old renderer, generation(%d:%d)",
804                        requesterGeneration, mRendererGeneration);
805                return;
806            }
807
808            int32_t what;
809            CHECK(msg->findInt32("what", &what));
810
811            if (what == Renderer::kWhatEOS) {
812                int32_t audio;
813                CHECK(msg->findInt32("audio", &audio));
814
815                int32_t finalResult;
816                CHECK(msg->findInt32("finalResult", &finalResult));
817
818                if (audio) {
819                    mAudioEOS = true;
820                } else {
821                    mVideoEOS = true;
822                }
823
824                if (finalResult == ERROR_END_OF_STREAM) {
825                    ALOGV("reached %s EOS", audio ? "audio" : "video");
826                } else {
827                    ALOGE("%s track encountered an error (%d)",
828                         audio ? "audio" : "video", finalResult);
829
830                    notifyListener(
831                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
832                }
833
834                if ((mAudioEOS || mAudioDecoder == NULL)
835                        && (mVideoEOS || mVideoDecoder == NULL)) {
836                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
837                }
838            } else if (what == Renderer::kWhatFlushComplete) {
839                int32_t audio;
840                CHECK(msg->findInt32("audio", &audio));
841
842                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
843                handleFlushComplete(audio, false /* isDecoder */);
844                finishFlushIfPossible();
845            } else if (what == Renderer::kWhatVideoRenderingStart) {
846                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
847            } else if (what == Renderer::kWhatMediaRenderingStart) {
848                ALOGV("media rendering started");
849                notifyListener(MEDIA_STARTED, 0, 0);
850            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
851                ALOGV("Tear down audio offload, fall back to s/w path if due to error.");
852                int64_t positionUs;
853                CHECK(msg->findInt64("positionUs", &positionUs));
854                int32_t reason;
855                CHECK(msg->findInt32("reason", &reason));
856                closeAudioSink();
857                mAudioDecoder.clear();
858                ++mAudioDecoderGeneration;
859                mRenderer->flush(true /* audio */);
860                if (mVideoDecoder != NULL) {
861                    mRenderer->flush(false /* audio */);
862                }
863
864                performSeek(positionUs, false /* needNotify */);
865                if (reason == Renderer::kDueToError) {
866                    mRenderer->signalDisableOffloadAudio();
867                    mOffloadAudio = false;
868                    instantiateDecoder(true /* audio */, &mAudioDecoder);
869                }
870            }
871            break;
872        }
873
874        case kWhatMoreDataQueued:
875        {
876            break;
877        }
878
879        case kWhatReset:
880        {
881            ALOGV("kWhatReset");
882
883            mDeferredActions.push_back(
884                    new FlushDecoderAction(
885                        FLUSH_CMD_SHUTDOWN /* audio */,
886                        FLUSH_CMD_SHUTDOWN /* video */));
887
888            mDeferredActions.push_back(
889                    new SimpleAction(&NuPlayer::performReset));
890
891            processDeferredActions();
892            break;
893        }
894
895        case kWhatSeek:
896        {
897            int64_t seekTimeUs;
898            int32_t needNotify;
899            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
900            CHECK(msg->findInt32("needNotify", &needNotify));
901
902            ALOGV("kWhatSeek seekTimeUs=%lld us, needNotify=%d",
903                    seekTimeUs, needNotify);
904
905            mDeferredActions.push_back(
906                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
907                                           FLUSH_CMD_FLUSH /* video */));
908
909            mDeferredActions.push_back(
910                    new SeekAction(seekTimeUs, needNotify));
911
912            processDeferredActions();
913            break;
914        }
915
916        case kWhatPause:
917        {
918            if (mSource != NULL) {
919                mSource->pause();
920            } else {
921                ALOGW("pause called when source is gone or not set");
922            }
923            if (mRenderer != NULL) {
924                mRenderer->pause();
925            } else {
926                ALOGW("pause called when renderer is gone or not set");
927            }
928            break;
929        }
930
931        case kWhatSourceNotify:
932        {
933            onSourceNotify(msg);
934            break;
935        }
936
937        case kWhatClosedCaptionNotify:
938        {
939            onClosedCaptionNotify(msg);
940            break;
941        }
942
943        default:
944            TRESPASS();
945            break;
946    }
947}
948
949void NuPlayer::onResume() {
950    if (mSource != NULL) {
951        mSource->resume();
952    } else {
953        ALOGW("resume called when source is gone or not set");
954    }
955    // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if
956    // needed.
957    if (audioDecoderStillNeeded() && mAudioDecoder == NULL) {
958        instantiateDecoder(true /* audio */, &mAudioDecoder);
959    }
960    if (mRenderer != NULL) {
961        mRenderer->resume();
962    } else {
963        ALOGW("resume called when renderer is gone or not set");
964    }
965}
966
967void NuPlayer::onStart() {
968    mVideoIsAVC = false;
969    mOffloadAudio = false;
970    mAudioEOS = false;
971    mVideoEOS = false;
972    mNumFramesTotal = 0;
973    mNumFramesDropped = 0;
974    mStarted = true;
975
976    /* instantiate decoders now for secure playback */
977    if (mSourceFlags & Source::FLAG_SECURE) {
978        if (mNativeWindow != NULL) {
979            instantiateDecoder(false, &mVideoDecoder);
980        }
981
982        if (mAudioSink != NULL) {
983            instantiateDecoder(true, &mAudioDecoder);
984        }
985    }
986
987    mSource->start();
988
989    uint32_t flags = 0;
990
991    if (mSource->isRealTime()) {
992        flags |= Renderer::FLAG_REAL_TIME;
993    }
994
995    sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
996    audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
997    if (mAudioSink != NULL) {
998        streamType = mAudioSink->getAudioStreamType();
999    }
1000
1001    sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
1002
1003    mOffloadAudio =
1004        canOffloadStream(audioMeta, (videoFormat != NULL),
1005                         true /* is_streaming */, streamType);
1006    if (mOffloadAudio) {
1007        flags |= Renderer::FLAG_OFFLOAD_AUDIO;
1008    }
1009
1010    sp<AMessage> notify = new AMessage(kWhatRendererNotify, id());
1011    ++mRendererGeneration;
1012    notify->setInt32("generation", mRendererGeneration);
1013    mRenderer = new Renderer(mAudioSink, notify, flags);
1014
1015    mRendererLooper = new ALooper;
1016    mRendererLooper->setName("NuPlayerRenderer");
1017    mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
1018    mRendererLooper->registerHandler(mRenderer);
1019
1020    sp<MetaData> meta = getFileMeta();
1021    int32_t rate;
1022    if (meta != NULL
1023            && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) {
1024        mRenderer->setVideoFrameRate(rate);
1025    }
1026
1027    if (mVideoDecoder != NULL) {
1028        mVideoDecoder->setRenderer(mRenderer);
1029    }
1030    if (mAudioDecoder != NULL) {
1031        mAudioDecoder->setRenderer(mRenderer);
1032    }
1033
1034    postScanSources();
1035}
1036
1037bool NuPlayer::audioDecoderStillNeeded() {
1038    // Audio decoder is no longer needed if it's in shut/shutting down status.
1039    return ((mFlushingAudio != SHUT_DOWN) && (mFlushingAudio != SHUTTING_DOWN_DECODER));
1040}
1041
1042void NuPlayer::handleFlushComplete(bool audio, bool isDecoder) {
1043    // We wait for both the decoder flush and the renderer flush to complete
1044    // before entering either the FLUSHED or the SHUTTING_DOWN_DECODER state.
1045
1046    mFlushComplete[audio][isDecoder] = true;
1047    if (!mFlushComplete[audio][!isDecoder]) {
1048        return;
1049    }
1050
1051    FlushStatus *state = audio ? &mFlushingAudio : &mFlushingVideo;
1052    switch (*state) {
1053        case FLUSHING_DECODER:
1054        {
1055            *state = FLUSHED;
1056            break;
1057        }
1058
1059        case FLUSHING_DECODER_SHUTDOWN:
1060        {
1061            *state = SHUTTING_DOWN_DECODER;
1062
1063            ALOGV("initiating %s decoder shutdown", audio ? "audio" : "video");
1064            if (!audio) {
1065                // Widevine source reads must stop before releasing the video decoder.
1066                if (mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
1067                    mSource->stop();
1068                }
1069            }
1070            getDecoder(audio)->initiateShutdown();
1071            break;
1072        }
1073
1074        default:
1075            // decoder flush completes only occur in a flushing state.
1076            LOG_ALWAYS_FATAL_IF(isDecoder, "decoder flush in invalid state %d", *state);
1077            break;
1078    }
1079}
1080
1081void NuPlayer::finishFlushIfPossible() {
1082    if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED
1083            && mFlushingAudio != SHUT_DOWN) {
1084        return;
1085    }
1086
1087    if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED
1088            && mFlushingVideo != SHUT_DOWN) {
1089        return;
1090    }
1091
1092    ALOGV("both audio and video are flushed now.");
1093
1094    mPendingAudioAccessUnit.clear();
1095    mAggregateBuffer.clear();
1096
1097    if (mTimeDiscontinuityPending) {
1098        mRenderer->signalTimeDiscontinuity();
1099        mTimeDiscontinuityPending = false;
1100    }
1101
1102    if (mAudioDecoder != NULL && mFlushingAudio == FLUSHED) {
1103        mAudioDecoder->signalResume();
1104    }
1105
1106    if (mVideoDecoder != NULL && mFlushingVideo == FLUSHED) {
1107        mVideoDecoder->signalResume();
1108    }
1109
1110    mFlushingAudio = NONE;
1111    mFlushingVideo = NONE;
1112
1113    clearFlushComplete();
1114
1115    processDeferredActions();
1116}
1117
1118void NuPlayer::postScanSources() {
1119    if (mScanSourcesPending) {
1120        return;
1121    }
1122
1123    sp<AMessage> msg = new AMessage(kWhatScanSources, id());
1124    msg->setInt32("generation", mScanSourcesGeneration);
1125    msg->post();
1126
1127    mScanSourcesPending = true;
1128}
1129
1130void NuPlayer::openAudioSink(const sp<AMessage> &format, bool offloadOnly) {
1131    uint32_t flags;
1132    int64_t durationUs;
1133    bool hasVideo = (mVideoDecoder != NULL);
1134    // FIXME: we should handle the case where the video decoder
1135    // is created after we receive the format change indication.
1136    // Current code will just make that we select deep buffer
1137    // with video which should not be a problem as it should
1138    // not prevent from keeping A/V sync.
1139    if (!hasVideo &&
1140            mSource->getDuration(&durationUs) == OK &&
1141            durationUs
1142                > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
1143        flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1144    } else {
1145        flags = AUDIO_OUTPUT_FLAG_NONE;
1146    }
1147
1148    mOffloadAudio = mRenderer->openAudioSink(
1149            format, offloadOnly, hasVideo, flags);
1150
1151    if (mOffloadAudio) {
1152        sp<MetaData> audioMeta =
1153                mSource->getFormatMeta(true /* audio */);
1154        sendMetaDataToHal(mAudioSink, audioMeta);
1155    }
1156}
1157
1158void NuPlayer::closeAudioSink() {
1159    mRenderer->closeAudioSink();
1160}
1161
1162status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
1163    if (*decoder != NULL) {
1164        return OK;
1165    }
1166
1167    sp<AMessage> format = mSource->getFormat(audio);
1168
1169    if (format == NULL) {
1170        return -EWOULDBLOCK;
1171    }
1172
1173    if (!audio) {
1174        AString mime;
1175        CHECK(format->findString("mime", &mime));
1176        mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
1177
1178        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
1179        mCCDecoder = new CCDecoder(ccNotify);
1180
1181        if (mSourceFlags & Source::FLAG_SECURE) {
1182            format->setInt32("secure", true);
1183        }
1184    }
1185
1186    if (audio) {
1187        sp<AMessage> notify = new AMessage(kWhatAudioNotify, id());
1188        ++mAudioDecoderGeneration;
1189        notify->setInt32("generation", mAudioDecoderGeneration);
1190
1191        if (mOffloadAudio) {
1192            *decoder = new DecoderPassThrough(notify, mSource, mRenderer);
1193        } else {
1194            *decoder = new Decoder(notify, mSource, mRenderer);
1195        }
1196    } else {
1197        sp<AMessage> notify = new AMessage(kWhatVideoNotify, id());
1198        ++mVideoDecoderGeneration;
1199        notify->setInt32("generation", mVideoDecoderGeneration);
1200
1201        *decoder = new Decoder(notify, mSource, mRenderer, mNativeWindow);
1202    }
1203    (*decoder)->init();
1204    (*decoder)->configure(format);
1205
1206    // allocate buffers to decrypt widevine source buffers
1207    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1208        Vector<sp<ABuffer> > inputBufs;
1209        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1210
1211        Vector<MediaBuffer *> mediaBufs;
1212        for (size_t i = 0; i < inputBufs.size(); i++) {
1213            const sp<ABuffer> &buffer = inputBufs[i];
1214            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1215            mediaBufs.push(mbuf);
1216        }
1217
1218        status_t err = mSource->setBuffers(audio, mediaBufs);
1219        if (err != OK) {
1220            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1221                mediaBufs[i]->release();
1222            }
1223            mediaBufs.clear();
1224            ALOGE("Secure source didn't support secure mediaBufs.");
1225            return err;
1226        }
1227    }
1228    return OK;
1229}
1230
1231status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
1232    sp<AMessage> reply;
1233    CHECK(msg->findMessage("reply", &reply));
1234
1235    if ((audio && mFlushingAudio != NONE)
1236            || (!audio && mFlushingVideo != NONE)
1237            || mSource == NULL) {
1238        reply->setInt32("err", INFO_DISCONTINUITY);
1239        reply->post();
1240        return OK;
1241    }
1242
1243    sp<ABuffer> accessUnit;
1244
1245    // Aggregate smaller buffers into a larger buffer.
1246    // The goal is to reduce power consumption.
1247    // Note this will not work if the decoder requires one frame per buffer.
1248    bool doBufferAggregation = (audio && mOffloadAudio);
1249    bool needMoreData = false;
1250
1251    bool dropAccessUnit;
1252    do {
1253        status_t err;
1254        // Did we save an accessUnit earlier because of a discontinuity?
1255        if (audio && (mPendingAudioAccessUnit != NULL)) {
1256            accessUnit = mPendingAudioAccessUnit;
1257            mPendingAudioAccessUnit.clear();
1258            err = mPendingAudioErr;
1259            ALOGV("feedDecoderInputData() use mPendingAudioAccessUnit");
1260        } else {
1261            err = mSource->dequeueAccessUnit(audio, &accessUnit);
1262        }
1263
1264        if (err == -EWOULDBLOCK) {
1265            return err;
1266        } else if (err != OK) {
1267            if (err == INFO_DISCONTINUITY) {
1268                if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1269                    // We already have some data so save this for later.
1270                    mPendingAudioErr = err;
1271                    mPendingAudioAccessUnit = accessUnit;
1272                    accessUnit.clear();
1273                    ALOGD("feedDecoderInputData() save discontinuity for later");
1274                    break;
1275                }
1276                int32_t type;
1277                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
1278
1279                bool formatChange =
1280                    (audio &&
1281                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
1282                    || (!audio &&
1283                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
1284
1285                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
1286
1287                ALOGI("%s discontinuity (formatChange=%d, time=%d)",
1288                     audio ? "audio" : "video", formatChange, timeChange);
1289
1290                mTimeDiscontinuityPending =
1291                    mTimeDiscontinuityPending || timeChange;
1292
1293                bool seamlessFormatChange = false;
1294                sp<AMessage> newFormat = mSource->getFormat(audio);
1295                if (formatChange) {
1296                    seamlessFormatChange =
1297                        getDecoder(audio)->supportsSeamlessFormatChange(newFormat);
1298                    // treat seamless format change separately
1299                    formatChange = !seamlessFormatChange;
1300                }
1301                bool shutdownOrFlush = formatChange || timeChange;
1302
1303                // We want to queue up scan-sources only once per discontinuity.
1304                // We control this by doing it only if neither audio nor video are
1305                // flushing or shutting down.  (After handling 1st discontinuity, one
1306                // of the flushing states will not be NONE.)
1307                // No need to scan sources if this discontinuity does not result
1308                // in a flush or shutdown, as the flushing state will stay NONE.
1309                if (mFlushingAudio == NONE && mFlushingVideo == NONE &&
1310                        shutdownOrFlush) {
1311                    // And we'll resume scanning sources once we're done
1312                    // flushing.
1313                    mDeferredActions.push_front(
1314                            new SimpleAction(
1315                                &NuPlayer::performScanSources));
1316                }
1317
1318                if (formatChange /* not seamless */) {
1319                    // must change decoder
1320                    flushDecoder(audio, /* needShutdown = */ true);
1321                } else if (timeChange) {
1322                    // need to flush
1323                    flushDecoder(audio, /* needShutdown = */ false, newFormat);
1324                    err = OK;
1325                } else if (seamlessFormatChange) {
1326                    // reuse existing decoder and don't flush
1327                    updateDecoderFormatWithoutFlush(audio, newFormat);
1328                    err = OK;
1329                } else {
1330                    // This stream is unaffected by the discontinuity
1331                    return -EWOULDBLOCK;
1332                }
1333            }
1334
1335            reply->setInt32("err", err);
1336            reply->post();
1337            return OK;
1338        }
1339
1340        if (!audio) {
1341            ++mNumFramesTotal;
1342        }
1343
1344        dropAccessUnit = false;
1345        if (!audio
1346                && !(mSourceFlags & Source::FLAG_SECURE)
1347                && mRenderer->getVideoLateByUs() > 100000ll
1348                && mVideoIsAVC
1349                && !IsAVCReferenceFrame(accessUnit)) {
1350            dropAccessUnit = true;
1351            ++mNumFramesDropped;
1352        }
1353
1354        size_t smallSize = accessUnit->size();
1355        needMoreData = false;
1356        if (doBufferAggregation && (mAggregateBuffer == NULL)
1357                // Don't bother if only room for a few small buffers.
1358                && (smallSize < (kAggregateBufferSizeBytes / 3))) {
1359            // Create a larger buffer for combining smaller buffers from the extractor.
1360            mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes);
1361            mAggregateBuffer->setRange(0, 0); // start empty
1362        }
1363
1364        if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1365            int64_t timeUs;
1366            int64_t dummy;
1367            bool smallTimestampValid = accessUnit->meta()->findInt64("timeUs", &timeUs);
1368            bool bigTimestampValid = mAggregateBuffer->meta()->findInt64("timeUs", &dummy);
1369            // Will the smaller buffer fit?
1370            size_t bigSize = mAggregateBuffer->size();
1371            size_t roomLeft = mAggregateBuffer->capacity() - bigSize;
1372            // Should we save this small buffer for the next big buffer?
1373            // If the first small buffer did not have a timestamp then save
1374            // any buffer that does have a timestamp until the next big buffer.
1375            if ((smallSize > roomLeft)
1376                || (!bigTimestampValid && (bigSize > 0) && smallTimestampValid)) {
1377                mPendingAudioErr = err;
1378                mPendingAudioAccessUnit = accessUnit;
1379                accessUnit.clear();
1380            } else {
1381                // Grab time from first small buffer if available.
1382                if ((bigSize == 0) && smallTimestampValid) {
1383                    mAggregateBuffer->meta()->setInt64("timeUs", timeUs);
1384                }
1385                // Append small buffer to the bigger buffer.
1386                memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize);
1387                bigSize += smallSize;
1388                mAggregateBuffer->setRange(0, bigSize);
1389
1390                // Keep looping until we run out of room in the mAggregateBuffer.
1391                needMoreData = true;
1392
1393                ALOGV("feedDecoderInputData() smallSize = %zu, bigSize = %zu, capacity = %zu",
1394                        smallSize, bigSize, mAggregateBuffer->capacity());
1395            }
1396        }
1397    } while (dropAccessUnit || needMoreData);
1398
1399    // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
1400
1401#if 0
1402    int64_t mediaTimeUs;
1403    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
1404    ALOGV("feeding %s input buffer at media time %.2f secs",
1405         audio ? "audio" : "video",
1406         mediaTimeUs / 1E6);
1407#endif
1408
1409    if (!audio) {
1410        mCCDecoder->decode(accessUnit);
1411    }
1412
1413    if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1414        ALOGV("feedDecoderInputData() reply with aggregated buffer, %zu",
1415                mAggregateBuffer->size());
1416        reply->setBuffer("buffer", mAggregateBuffer);
1417        mAggregateBuffer.clear();
1418    } else {
1419        reply->setBuffer("buffer", accessUnit);
1420    }
1421
1422    reply->post();
1423
1424    return OK;
1425}
1426
1427void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
1428    // ALOGV("renderBuffer %s", audio ? "audio" : "video");
1429
1430    if ((audio && mFlushingAudio != NONE)
1431            || (!audio && mFlushingVideo != NONE)) {
1432        // We're currently attempting to flush the decoder, in order
1433        // to complete this, the decoder wants all its buffers back,
1434        // so we don't want any output buffers it sent us (from before
1435        // we initiated the flush) to be stuck in the renderer's queue.
1436
1437        ALOGV("we're still flushing the %s decoder, sending its output buffer"
1438             " right back.", audio ? "audio" : "video");
1439
1440        return;
1441    }
1442
1443    int64_t mediaTimeUs;
1444    CHECK(msg->findInt64("timeUs", &mediaTimeUs));
1445
1446    if (!audio && mCCDecoder->isSelected()) {
1447        mCCDecoder->display(mediaTimeUs);
1448    }
1449}
1450
1451void NuPlayer::updateVideoSize(
1452        const sp<AMessage> &inputFormat,
1453        const sp<AMessage> &outputFormat) {
1454    if (inputFormat == NULL) {
1455        ALOGW("Unknown video size, reporting 0x0!");
1456        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1457        return;
1458    }
1459
1460    int32_t displayWidth, displayHeight;
1461    int32_t cropLeft, cropTop, cropRight, cropBottom;
1462
1463    if (outputFormat != NULL) {
1464        int32_t width, height;
1465        CHECK(outputFormat->findInt32("width", &width));
1466        CHECK(outputFormat->findInt32("height", &height));
1467
1468        int32_t cropLeft, cropTop, cropRight, cropBottom;
1469        CHECK(outputFormat->findRect(
1470                    "crop",
1471                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1472
1473        displayWidth = cropRight - cropLeft + 1;
1474        displayHeight = cropBottom - cropTop + 1;
1475
1476        ALOGV("Video output format changed to %d x %d "
1477             "(crop: %d x %d @ (%d, %d))",
1478             width, height,
1479             displayWidth,
1480             displayHeight,
1481             cropLeft, cropTop);
1482    } else {
1483        CHECK(inputFormat->findInt32("width", &displayWidth));
1484        CHECK(inputFormat->findInt32("height", &displayHeight));
1485
1486        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1487    }
1488
1489    // Take into account sample aspect ratio if necessary:
1490    int32_t sarWidth, sarHeight;
1491    if (inputFormat->findInt32("sar-width", &sarWidth)
1492            && inputFormat->findInt32("sar-height", &sarHeight)) {
1493        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1494
1495        displayWidth = (displayWidth * sarWidth) / sarHeight;
1496
1497        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1498    }
1499
1500    int32_t rotationDegrees;
1501    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1502        rotationDegrees = 0;
1503    }
1504
1505    if (rotationDegrees == 90 || rotationDegrees == 270) {
1506        int32_t tmp = displayWidth;
1507        displayWidth = displayHeight;
1508        displayHeight = tmp;
1509    }
1510
1511    notifyListener(
1512            MEDIA_SET_VIDEO_SIZE,
1513            displayWidth,
1514            displayHeight);
1515}
1516
1517void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1518    if (mDriver == NULL) {
1519        return;
1520    }
1521
1522    sp<NuPlayerDriver> driver = mDriver.promote();
1523
1524    if (driver == NULL) {
1525        return;
1526    }
1527
1528    driver->notifyListener(msg, ext1, ext2, in);
1529}
1530
1531void NuPlayer::flushDecoder(
1532        bool audio, bool needShutdown, const sp<AMessage> &newFormat) {
1533    ALOGV("[%s] flushDecoder needShutdown=%d",
1534          audio ? "audio" : "video", needShutdown);
1535
1536    const sp<Decoder> &decoder = getDecoder(audio);
1537    if (decoder == NULL) {
1538        ALOGI("flushDecoder %s without decoder present",
1539             audio ? "audio" : "video");
1540        return;
1541    }
1542
1543    // Make sure we don't continue to scan sources until we finish flushing.
1544    ++mScanSourcesGeneration;
1545    mScanSourcesPending = false;
1546
1547    decoder->signalFlush(newFormat);
1548
1549    FlushStatus newStatus =
1550        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1551
1552    mFlushComplete[audio][false /* isDecoder */] = false;
1553    mFlushComplete[audio][true /* isDecoder */] = false;
1554    if (audio) {
1555        ALOGE_IF(mFlushingAudio != NONE,
1556                "audio flushDecoder() is called in state %d", mFlushingAudio);
1557        mFlushingAudio = newStatus;
1558    } else {
1559        ALOGE_IF(mFlushingVideo != NONE,
1560                "video flushDecoder() is called in state %d", mFlushingVideo);
1561        mFlushingVideo = newStatus;
1562
1563        if (mCCDecoder != NULL) {
1564            mCCDecoder->flush();
1565        }
1566    }
1567}
1568
1569void NuPlayer::updateDecoderFormatWithoutFlush(
1570        bool audio, const sp<AMessage> &format) {
1571    ALOGV("[%s] updateDecoderFormatWithoutFlush", audio ? "audio" : "video");
1572
1573    const sp<Decoder> &decoder = getDecoder(audio);
1574    if (decoder == NULL) {
1575        ALOGI("updateDecoderFormatWithoutFlush %s without decoder present",
1576             audio ? "audio" : "video");
1577        return;
1578    }
1579
1580    decoder->signalUpdateFormat(format);
1581}
1582
1583void NuPlayer::queueDecoderShutdown(
1584        bool audio, bool video, const sp<AMessage> &reply) {
1585    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1586
1587    mDeferredActions.push_back(
1588            new FlushDecoderAction(
1589                audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
1590                video ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE));
1591
1592    mDeferredActions.push_back(
1593            new SimpleAction(&NuPlayer::performScanSources));
1594
1595    mDeferredActions.push_back(new PostMessageAction(reply));
1596
1597    processDeferredActions();
1598}
1599
1600status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1601    mVideoScalingMode = mode;
1602    if (mNativeWindow != NULL) {
1603        status_t ret = native_window_set_scaling_mode(
1604                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1605        if (ret != OK) {
1606            ALOGE("Failed to set scaling mode (%d): %s",
1607                -ret, strerror(-ret));
1608            return ret;
1609        }
1610    }
1611    return OK;
1612}
1613
1614status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1615    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id());
1616    msg->setPointer("reply", reply);
1617
1618    sp<AMessage> response;
1619    status_t err = msg->postAndAwaitResponse(&response);
1620    return err;
1621}
1622
1623status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1624    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id());
1625    msg->setPointer("reply", reply);
1626    msg->setInt32("type", type);
1627
1628    sp<AMessage> response;
1629    status_t err = msg->postAndAwaitResponse(&response);
1630    if (err == OK && response != NULL) {
1631        CHECK(response->findInt32("err", &err));
1632    }
1633    return err;
1634}
1635
1636status_t NuPlayer::selectTrack(size_t trackIndex, bool select) {
1637    sp<AMessage> msg = new AMessage(kWhatSelectTrack, id());
1638    msg->setSize("trackIndex", trackIndex);
1639    msg->setInt32("select", select);
1640
1641    sp<AMessage> response;
1642    status_t err = msg->postAndAwaitResponse(&response);
1643
1644    if (err != OK) {
1645        return err;
1646    }
1647
1648    if (!response->findInt32("err", &err)) {
1649        err = OK;
1650    }
1651
1652    return err;
1653}
1654
1655status_t NuPlayer::getCurrentPosition(int64_t *mediaUs) {
1656    sp<Renderer> renderer = mRenderer;
1657    if (renderer == NULL) {
1658        return NO_INIT;
1659    }
1660
1661    return renderer->getCurrentPosition(mediaUs);
1662}
1663
1664void NuPlayer::getStats(int64_t *numFramesTotal, int64_t *numFramesDropped) {
1665    *numFramesTotal = mNumFramesTotal;
1666    *numFramesDropped = mNumFramesDropped;
1667}
1668
1669sp<MetaData> NuPlayer::getFileMeta() {
1670    return mSource->getFileFormatMeta();
1671}
1672
1673void NuPlayer::schedulePollDuration() {
1674    sp<AMessage> msg = new AMessage(kWhatPollDuration, id());
1675    msg->setInt32("generation", mPollDurationGeneration);
1676    msg->post();
1677}
1678
1679void NuPlayer::cancelPollDuration() {
1680    ++mPollDurationGeneration;
1681}
1682
1683void NuPlayer::processDeferredActions() {
1684    while (!mDeferredActions.empty()) {
1685        // We won't execute any deferred actions until we're no longer in
1686        // an intermediate state, i.e. one more more decoders are currently
1687        // flushing or shutting down.
1688
1689        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1690            // We're currently flushing, postpone the reset until that's
1691            // completed.
1692
1693            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1694                  mFlushingAudio, mFlushingVideo);
1695
1696            break;
1697        }
1698
1699        sp<Action> action = *mDeferredActions.begin();
1700        mDeferredActions.erase(mDeferredActions.begin());
1701
1702        action->execute(this);
1703    }
1704}
1705
1706void NuPlayer::performSeek(int64_t seekTimeUs, bool needNotify) {
1707    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs), needNotify(%d)",
1708          seekTimeUs,
1709          seekTimeUs / 1E6,
1710          needNotify);
1711
1712    if (mSource == NULL) {
1713        // This happens when reset occurs right before the loop mode
1714        // asynchronously seeks to the start of the stream.
1715        LOG_ALWAYS_FATAL_IF(mAudioDecoder != NULL || mVideoDecoder != NULL,
1716                "mSource is NULL and decoders not NULL audio(%p) video(%p)",
1717                mAudioDecoder.get(), mVideoDecoder.get());
1718        return;
1719    }
1720    mSource->seekTo(seekTimeUs);
1721    ++mTimedTextGeneration;
1722
1723    if (mDriver != NULL) {
1724        sp<NuPlayerDriver> driver = mDriver.promote();
1725        if (driver != NULL) {
1726            if (needNotify) {
1727                driver->notifySeekComplete();
1728            }
1729        }
1730    }
1731
1732    // everything's flushed, continue playback.
1733}
1734
1735void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) {
1736    ALOGV("performDecoderFlush audio=%d, video=%d", audio, video);
1737
1738    if ((audio == FLUSH_CMD_NONE || mAudioDecoder == NULL)
1739            && (video == FLUSH_CMD_NONE || mVideoDecoder == NULL)) {
1740        return;
1741    }
1742
1743    mTimeDiscontinuityPending = true;
1744
1745    if (audio != FLUSH_CMD_NONE && mAudioDecoder != NULL) {
1746        flushDecoder(true /* audio */, (audio == FLUSH_CMD_SHUTDOWN));
1747    }
1748
1749    if (video != FLUSH_CMD_NONE && mVideoDecoder != NULL) {
1750        flushDecoder(false /* audio */, (video == FLUSH_CMD_SHUTDOWN));
1751    }
1752}
1753
1754void NuPlayer::performReset() {
1755    ALOGV("performReset");
1756
1757    CHECK(mAudioDecoder == NULL);
1758    CHECK(mVideoDecoder == NULL);
1759
1760    cancelPollDuration();
1761
1762    ++mScanSourcesGeneration;
1763    mScanSourcesPending = false;
1764
1765    if (mRendererLooper != NULL) {
1766        if (mRenderer != NULL) {
1767            mRendererLooper->unregisterHandler(mRenderer->id());
1768        }
1769        mRendererLooper->stop();
1770        mRendererLooper.clear();
1771    }
1772    mRenderer.clear();
1773    ++mRendererGeneration;
1774
1775    if (mSource != NULL) {
1776        mSource->stop();
1777
1778        mSource.clear();
1779    }
1780
1781    if (mDriver != NULL) {
1782        sp<NuPlayerDriver> driver = mDriver.promote();
1783        if (driver != NULL) {
1784            driver->notifyResetComplete();
1785        }
1786    }
1787
1788    mStarted = false;
1789}
1790
1791void NuPlayer::performScanSources() {
1792    ALOGV("performScanSources");
1793
1794    if (!mStarted) {
1795        return;
1796    }
1797
1798    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1799        postScanSources();
1800    }
1801}
1802
1803void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1804    ALOGV("performSetSurface");
1805
1806    mNativeWindow = wrapper;
1807
1808    // XXX - ignore error from setVideoScalingMode for now
1809    setVideoScalingMode(mVideoScalingMode);
1810
1811    if (mDriver != NULL) {
1812        sp<NuPlayerDriver> driver = mDriver.promote();
1813        if (driver != NULL) {
1814            driver->notifySetSurfaceComplete();
1815        }
1816    }
1817}
1818
1819void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1820    int32_t what;
1821    CHECK(msg->findInt32("what", &what));
1822
1823    switch (what) {
1824        case Source::kWhatPrepared:
1825        {
1826            if (mSource == NULL) {
1827                // This is a stale notification from a source that was
1828                // asynchronously preparing when the client called reset().
1829                // We handled the reset, the source is gone.
1830                break;
1831            }
1832
1833            int32_t err;
1834            CHECK(msg->findInt32("err", &err));
1835
1836            sp<NuPlayerDriver> driver = mDriver.promote();
1837            if (driver != NULL) {
1838                // notify duration first, so that it's definitely set when
1839                // the app received the "prepare complete" callback.
1840                int64_t durationUs;
1841                if (mSource->getDuration(&durationUs) == OK) {
1842                    driver->notifyDuration(durationUs);
1843                }
1844                driver->notifyPrepareCompleted(err);
1845            }
1846
1847            break;
1848        }
1849
1850        case Source::kWhatFlagsChanged:
1851        {
1852            uint32_t flags;
1853            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1854
1855            sp<NuPlayerDriver> driver = mDriver.promote();
1856            if (driver != NULL) {
1857                driver->notifyFlagsChanged(flags);
1858            }
1859
1860            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1861                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
1862                cancelPollDuration();
1863            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1864                    && (flags & Source::FLAG_DYNAMIC_DURATION)
1865                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
1866                schedulePollDuration();
1867            }
1868
1869            mSourceFlags = flags;
1870            break;
1871        }
1872
1873        case Source::kWhatVideoSizeChanged:
1874        {
1875            sp<AMessage> format;
1876            CHECK(msg->findMessage("format", &format));
1877
1878            updateVideoSize(format);
1879            break;
1880        }
1881
1882        case Source::kWhatBufferingUpdate:
1883        {
1884            int32_t percentage;
1885            CHECK(msg->findInt32("percentage", &percentage));
1886
1887            notifyListener(MEDIA_BUFFERING_UPDATE, percentage, 0);
1888            break;
1889        }
1890
1891        case Source::kWhatBufferingStart:
1892        {
1893            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
1894            break;
1895        }
1896
1897        case Source::kWhatBufferingEnd:
1898        {
1899            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
1900            break;
1901        }
1902
1903        case Source::kWhatSubtitleData:
1904        {
1905            sp<ABuffer> buffer;
1906            CHECK(msg->findBuffer("buffer", &buffer));
1907
1908            sendSubtitleData(buffer, 0 /* baseIndex */);
1909            break;
1910        }
1911
1912        case Source::kWhatTimedTextData:
1913        {
1914            int32_t generation;
1915            if (msg->findInt32("generation", &generation)
1916                    && generation != mTimedTextGeneration) {
1917                break;
1918            }
1919
1920            sp<ABuffer> buffer;
1921            CHECK(msg->findBuffer("buffer", &buffer));
1922
1923            sp<NuPlayerDriver> driver = mDriver.promote();
1924            if (driver == NULL) {
1925                break;
1926            }
1927
1928            int posMs;
1929            int64_t timeUs, posUs;
1930            driver->getCurrentPosition(&posMs);
1931            posUs = posMs * 1000;
1932            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1933
1934            if (posUs < timeUs) {
1935                if (!msg->findInt32("generation", &generation)) {
1936                    msg->setInt32("generation", mTimedTextGeneration);
1937                }
1938                msg->post(timeUs - posUs);
1939            } else {
1940                sendTimedTextData(buffer);
1941            }
1942            break;
1943        }
1944
1945        case Source::kWhatQueueDecoderShutdown:
1946        {
1947            int32_t audio, video;
1948            CHECK(msg->findInt32("audio", &audio));
1949            CHECK(msg->findInt32("video", &video));
1950
1951            sp<AMessage> reply;
1952            CHECK(msg->findMessage("reply", &reply));
1953
1954            queueDecoderShutdown(audio, video, reply);
1955            break;
1956        }
1957
1958        case Source::kWhatDrmNoLicense:
1959        {
1960            notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, ERROR_DRM_NO_LICENSE);
1961            break;
1962        }
1963
1964        default:
1965            TRESPASS();
1966    }
1967}
1968
1969void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
1970    int32_t what;
1971    CHECK(msg->findInt32("what", &what));
1972
1973    switch (what) {
1974        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
1975        {
1976            sp<ABuffer> buffer;
1977            CHECK(msg->findBuffer("buffer", &buffer));
1978
1979            size_t inbandTracks = 0;
1980            if (mSource != NULL) {
1981                inbandTracks = mSource->getTrackCount();
1982            }
1983
1984            sendSubtitleData(buffer, inbandTracks);
1985            break;
1986        }
1987
1988        case NuPlayer::CCDecoder::kWhatTrackAdded:
1989        {
1990            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
1991
1992            break;
1993        }
1994
1995        default:
1996            TRESPASS();
1997    }
1998
1999
2000}
2001
2002void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
2003    int32_t trackIndex;
2004    int64_t timeUs, durationUs;
2005    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
2006    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2007    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
2008
2009    Parcel in;
2010    in.writeInt32(trackIndex + baseIndex);
2011    in.writeInt64(timeUs);
2012    in.writeInt64(durationUs);
2013    in.writeInt32(buffer->size());
2014    in.writeInt32(buffer->size());
2015    in.write(buffer->data(), buffer->size());
2016
2017    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
2018}
2019
2020void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
2021    const void *data;
2022    size_t size = 0;
2023    int64_t timeUs;
2024    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
2025
2026    AString mime;
2027    CHECK(buffer->meta()->findString("mime", &mime));
2028    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
2029
2030    data = buffer->data();
2031    size = buffer->size();
2032
2033    Parcel parcel;
2034    if (size > 0) {
2035        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2036        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
2037        TextDescriptions::getParcelOfDescriptions(
2038                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
2039    }
2040
2041    if ((parcel.dataSize() > 0)) {
2042        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
2043    } else {  // send an empty timed text
2044        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
2045    }
2046}
2047////////////////////////////////////////////////////////////////////////////////
2048
2049sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
2050    sp<MetaData> meta = getFormatMeta(audio);
2051
2052    if (meta == NULL) {
2053        return NULL;
2054    }
2055
2056    sp<AMessage> msg = new AMessage;
2057
2058    if(convertMetaDataToMessage(meta, &msg) == OK) {
2059        return msg;
2060    }
2061    return NULL;
2062}
2063
2064void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
2065    sp<AMessage> notify = dupNotify();
2066    notify->setInt32("what", kWhatFlagsChanged);
2067    notify->setInt32("flags", flags);
2068    notify->post();
2069}
2070
2071void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
2072    sp<AMessage> notify = dupNotify();
2073    notify->setInt32("what", kWhatVideoSizeChanged);
2074    notify->setMessage("format", format);
2075    notify->post();
2076}
2077
2078void NuPlayer::Source::notifyPrepared(status_t err) {
2079    sp<AMessage> notify = dupNotify();
2080    notify->setInt32("what", kWhatPrepared);
2081    notify->setInt32("err", err);
2082    notify->post();
2083}
2084
2085void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
2086    TRESPASS();
2087}
2088
2089}  // namespace android
2090