NuPlayer.cpp revision c6cfd70f24a11b946859485ce398a189c301a4e2 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerDecoder.h"
25#include "NuPlayerDecoderPassThrough.h"
26#include "NuPlayerDriver.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29#include "RTSPSource.h"
30#include "StreamingSource.h"
31#include "GenericSource.h"
32#include "TextDescriptions.h"
33
34#include "ATSParser.h"
35
36#include <media/stagefright/foundation/hexdump.h>
37#include <media/stagefright/foundation/ABuffer.h>
38#include <media/stagefright/foundation/ADebug.h>
39#include <media/stagefright/foundation/AMessage.h>
40#include <media/stagefright/MediaBuffer.h>
41#include <media/stagefright/MediaDefs.h>
42#include <media/stagefright/MediaErrors.h>
43#include <media/stagefright/MetaData.h>
44#include <gui/IGraphicBufferProducer.h>
45
46#include "avc_utils.h"
47
48#include "ESDS.h"
49#include <media/stagefright/Utils.h>
50
51namespace android {
52
53// TODO optimize buffer size for power consumption
54// The offload read buffer size is 32 KB but 24 KB uses less power.
55const size_t NuPlayer::kAggregateBufferSizeBytes = 24 * 1024;
56
57struct NuPlayer::Action : public RefBase {
58    Action() {}
59
60    virtual void execute(NuPlayer *player) = 0;
61
62private:
63    DISALLOW_EVIL_CONSTRUCTORS(Action);
64};
65
66struct NuPlayer::SeekAction : public Action {
67    SeekAction(int64_t seekTimeUs, bool needNotify)
68        : mSeekTimeUs(seekTimeUs),
69          mNeedNotify(needNotify) {
70    }
71
72    virtual void execute(NuPlayer *player) {
73        player->performSeek(mSeekTimeUs, mNeedNotify);
74    }
75
76private:
77    int64_t mSeekTimeUs;
78    bool mNeedNotify;
79
80    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
81};
82
83struct NuPlayer::SetSurfaceAction : public Action {
84    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
85        : mWrapper(wrapper) {
86    }
87
88    virtual void execute(NuPlayer *player) {
89        player->performSetSurface(mWrapper);
90    }
91
92private:
93    sp<NativeWindowWrapper> mWrapper;
94
95    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
96};
97
98struct NuPlayer::FlushDecoderAction : public Action {
99    FlushDecoderAction(FlushCommand audio, FlushCommand video)
100        : mAudio(audio),
101          mVideo(video) {
102    }
103
104    virtual void execute(NuPlayer *player) {
105        player->performDecoderFlush(mAudio, mVideo);
106    }
107
108private:
109    FlushCommand mAudio;
110    FlushCommand mVideo;
111
112    DISALLOW_EVIL_CONSTRUCTORS(FlushDecoderAction);
113};
114
115struct NuPlayer::PostMessageAction : public Action {
116    PostMessageAction(const sp<AMessage> &msg)
117        : mMessage(msg) {
118    }
119
120    virtual void execute(NuPlayer *) {
121        mMessage->post();
122    }
123
124private:
125    sp<AMessage> mMessage;
126
127    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
128};
129
130// Use this if there's no state necessary to save in order to execute
131// the action.
132struct NuPlayer::SimpleAction : public Action {
133    typedef void (NuPlayer::*ActionFunc)();
134
135    SimpleAction(ActionFunc func)
136        : mFunc(func) {
137    }
138
139    virtual void execute(NuPlayer *player) {
140        (player->*mFunc)();
141    }
142
143private:
144    ActionFunc mFunc;
145
146    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
147};
148
149////////////////////////////////////////////////////////////////////////////////
150
151NuPlayer::NuPlayer()
152    : mUIDValid(false),
153      mSourceFlags(0),
154      mVideoIsAVC(false),
155      mOffloadAudio(false),
156      mAudioDecoderGeneration(0),
157      mVideoDecoderGeneration(0),
158      mRendererGeneration(0),
159      mAudioEOS(false),
160      mVideoEOS(false),
161      mScanSourcesPending(false),
162      mScanSourcesGeneration(0),
163      mPollDurationGeneration(0),
164      mTimedTextGeneration(0),
165      mTimeDiscontinuityPending(false),
166      mFlushingAudio(NONE),
167      mFlushingVideo(NONE),
168      mNumFramesTotal(0ll),
169      mNumFramesDropped(0ll),
170      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
171      mStarted(false) {
172    clearFlushComplete();
173}
174
175NuPlayer::~NuPlayer() {
176}
177
178void NuPlayer::setUID(uid_t uid) {
179    mUIDValid = true;
180    mUID = uid;
181}
182
183void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
184    mDriver = driver;
185}
186
187void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
188    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
189
190    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
191
192    msg->setObject("source", new StreamingSource(notify, source));
193    msg->post();
194}
195
196static bool IsHTTPLiveURL(const char *url) {
197    if (!strncasecmp("http://", url, 7)
198            || !strncasecmp("https://", url, 8)
199            || !strncasecmp("file://", url, 7)) {
200        size_t len = strlen(url);
201        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
202            return true;
203        }
204
205        if (strstr(url,"m3u8")) {
206            return true;
207        }
208    }
209
210    return false;
211}
212
213void NuPlayer::setDataSourceAsync(
214        const sp<IMediaHTTPService> &httpService,
215        const char *url,
216        const KeyedVector<String8, String8> *headers) {
217
218    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
219    size_t len = strlen(url);
220
221    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
222
223    sp<Source> source;
224    if (IsHTTPLiveURL(url)) {
225        source = new HTTPLiveSource(notify, httpService, url, headers);
226    } else if (!strncasecmp(url, "rtsp://", 7)) {
227        source = new RTSPSource(
228                notify, httpService, url, headers, mUIDValid, mUID);
229    } else if ((!strncasecmp(url, "http://", 7)
230                || !strncasecmp(url, "https://", 8))
231                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
232                    || strstr(url, ".sdp?"))) {
233        source = new RTSPSource(
234                notify, httpService, url, headers, mUIDValid, mUID, true);
235    } else {
236        sp<GenericSource> genericSource =
237                new GenericSource(notify, mUIDValid, mUID);
238        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
239        // The correct flags will be updated in Source::kWhatFlagsChanged
240        // handler when  GenericSource is prepared.
241
242        status_t err = genericSource->setDataSource(httpService, url, headers);
243
244        if (err == OK) {
245            source = genericSource;
246        } else {
247            ALOGE("Failed to set data source!");
248        }
249    }
250    msg->setObject("source", source);
251    msg->post();
252}
253
254void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
255    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
256
257    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
258
259    sp<GenericSource> source =
260            new GenericSource(notify, mUIDValid, mUID);
261
262    status_t err = source->setDataSource(fd, offset, length);
263
264    if (err != OK) {
265        ALOGE("Failed to set data source!");
266        source = NULL;
267    }
268
269    msg->setObject("source", source);
270    msg->post();
271}
272
273void NuPlayer::prepareAsync() {
274    (new AMessage(kWhatPrepare, id()))->post();
275}
276
277void NuPlayer::setVideoSurfaceTextureAsync(
278        const sp<IGraphicBufferProducer> &bufferProducer) {
279    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id());
280
281    if (bufferProducer == NULL) {
282        msg->setObject("native-window", NULL);
283    } else {
284        msg->setObject(
285                "native-window",
286                new NativeWindowWrapper(
287                    new Surface(bufferProducer, true /* controlledByApp */)));
288    }
289
290    msg->post();
291}
292
293void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
294    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id());
295    msg->setObject("sink", sink);
296    msg->post();
297}
298
299void NuPlayer::start() {
300    (new AMessage(kWhatStart, id()))->post();
301}
302
303void NuPlayer::pause() {
304    (new AMessage(kWhatPause, id()))->post();
305}
306
307void NuPlayer::resetAsync() {
308    if (mSource != NULL) {
309        // During a reset, the data source might be unresponsive already, we need to
310        // disconnect explicitly so that reads exit promptly.
311        // We can't queue the disconnect request to the looper, as it might be
312        // queued behind a stuck read and never gets processed.
313        // Doing a disconnect outside the looper to allows the pending reads to exit
314        // (either successfully or with error).
315        mSource->disconnect();
316    }
317
318    (new AMessage(kWhatReset, id()))->post();
319}
320
321void NuPlayer::seekToAsync(int64_t seekTimeUs, bool needNotify) {
322    sp<AMessage> msg = new AMessage(kWhatSeek, id());
323    msg->setInt64("seekTimeUs", seekTimeUs);
324    msg->setInt32("needNotify", needNotify);
325    msg->post();
326}
327
328
329void NuPlayer::writeTrackInfo(
330        Parcel* reply, const sp<AMessage> format) const {
331    int32_t trackType;
332    CHECK(format->findInt32("type", &trackType));
333
334    AString lang;
335    CHECK(format->findString("language", &lang));
336
337    reply->writeInt32(2); // write something non-zero
338    reply->writeInt32(trackType);
339    reply->writeString16(String16(lang.c_str()));
340
341    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
342        AString mime;
343        CHECK(format->findString("mime", &mime));
344
345        int32_t isAuto, isDefault, isForced;
346        CHECK(format->findInt32("auto", &isAuto));
347        CHECK(format->findInt32("default", &isDefault));
348        CHECK(format->findInt32("forced", &isForced));
349
350        reply->writeString16(String16(mime.c_str()));
351        reply->writeInt32(isAuto);
352        reply->writeInt32(isDefault);
353        reply->writeInt32(isForced);
354    }
355}
356
357void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
358    switch (msg->what()) {
359        case kWhatSetDataSource:
360        {
361            ALOGV("kWhatSetDataSource");
362
363            CHECK(mSource == NULL);
364
365            status_t err = OK;
366            sp<RefBase> obj;
367            CHECK(msg->findObject("source", &obj));
368            if (obj != NULL) {
369                mSource = static_cast<Source *>(obj.get());
370            } else {
371                err = UNKNOWN_ERROR;
372            }
373
374            CHECK(mDriver != NULL);
375            sp<NuPlayerDriver> driver = mDriver.promote();
376            if (driver != NULL) {
377                driver->notifySetDataSourceCompleted(err);
378            }
379            break;
380        }
381
382        case kWhatPrepare:
383        {
384            mSource->prepareAsync();
385            break;
386        }
387
388        case kWhatGetTrackInfo:
389        {
390            uint32_t replyID;
391            CHECK(msg->senderAwaitsResponse(&replyID));
392
393            Parcel* reply;
394            CHECK(msg->findPointer("reply", (void**)&reply));
395
396            size_t inbandTracks = 0;
397            if (mSource != NULL) {
398                inbandTracks = mSource->getTrackCount();
399            }
400
401            size_t ccTracks = 0;
402            if (mCCDecoder != NULL) {
403                ccTracks = mCCDecoder->getTrackCount();
404            }
405
406            // total track count
407            reply->writeInt32(inbandTracks + ccTracks);
408
409            // write inband tracks
410            for (size_t i = 0; i < inbandTracks; ++i) {
411                writeTrackInfo(reply, mSource->getTrackInfo(i));
412            }
413
414            // write CC track
415            for (size_t i = 0; i < ccTracks; ++i) {
416                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
417            }
418
419            sp<AMessage> response = new AMessage;
420            response->postReply(replyID);
421            break;
422        }
423
424        case kWhatGetSelectedTrack:
425        {
426            status_t err = INVALID_OPERATION;
427            if (mSource != NULL) {
428                err = OK;
429
430                int32_t type32;
431                CHECK(msg->findInt32("type", (int32_t*)&type32));
432                media_track_type type = (media_track_type)type32;
433                ssize_t selectedTrack = mSource->getSelectedTrack(type);
434
435                Parcel* reply;
436                CHECK(msg->findPointer("reply", (void**)&reply));
437                reply->writeInt32(selectedTrack);
438            }
439
440            sp<AMessage> response = new AMessage;
441            response->setInt32("err", err);
442
443            uint32_t replyID;
444            CHECK(msg->senderAwaitsResponse(&replyID));
445            response->postReply(replyID);
446            break;
447        }
448
449        case kWhatSelectTrack:
450        {
451            uint32_t replyID;
452            CHECK(msg->senderAwaitsResponse(&replyID));
453
454            size_t trackIndex;
455            int32_t select;
456            CHECK(msg->findSize("trackIndex", &trackIndex));
457            CHECK(msg->findInt32("select", &select));
458
459            status_t err = INVALID_OPERATION;
460
461            size_t inbandTracks = 0;
462            if (mSource != NULL) {
463                inbandTracks = mSource->getTrackCount();
464            }
465            size_t ccTracks = 0;
466            if (mCCDecoder != NULL) {
467                ccTracks = mCCDecoder->getTrackCount();
468            }
469
470            if (trackIndex < inbandTracks) {
471                err = mSource->selectTrack(trackIndex, select);
472
473                if (!select && err == OK) {
474                    int32_t type;
475                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
476                    if (info != NULL
477                            && info->findInt32("type", &type)
478                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
479                        ++mTimedTextGeneration;
480                    }
481                }
482            } else {
483                trackIndex -= inbandTracks;
484
485                if (trackIndex < ccTracks) {
486                    err = mCCDecoder->selectTrack(trackIndex, select);
487                }
488            }
489
490            sp<AMessage> response = new AMessage;
491            response->setInt32("err", err);
492
493            response->postReply(replyID);
494            break;
495        }
496
497        case kWhatPollDuration:
498        {
499            int32_t generation;
500            CHECK(msg->findInt32("generation", &generation));
501
502            if (generation != mPollDurationGeneration) {
503                // stale
504                break;
505            }
506
507            int64_t durationUs;
508            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
509                sp<NuPlayerDriver> driver = mDriver.promote();
510                if (driver != NULL) {
511                    driver->notifyDuration(durationUs);
512                }
513            }
514
515            msg->post(1000000ll);  // poll again in a second.
516            break;
517        }
518
519        case kWhatSetVideoNativeWindow:
520        {
521            ALOGV("kWhatSetVideoNativeWindow");
522
523            sp<RefBase> obj;
524            CHECK(msg->findObject("native-window", &obj));
525
526            if (mSource->getFormat(false /* audio */) == NULL) {
527                performSetSurface(static_cast<NativeWindowWrapper *>(obj.get()));
528                break;
529            }
530
531            mDeferredActions.push_back(
532                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
533                                           FLUSH_CMD_SHUTDOWN /* video */));
534
535            mDeferredActions.push_back(
536                    new SetSurfaceAction(
537                        static_cast<NativeWindowWrapper *>(obj.get())));
538
539            if (obj != NULL) {
540                if (mStarted) {
541                    // Issue a seek to refresh the video screen only if started otherwise
542                    // the extractor may not yet be started and will assert.
543                    // If the video decoder is not set (perhaps audio only in this case)
544                    // do not perform a seek as it is not needed.
545                    int64_t currentPositionUs = 0;
546                    if (getCurrentPosition(&currentPositionUs) == OK) {
547                        mDeferredActions.push_back(
548                                new SeekAction(currentPositionUs, false /* needNotify */));
549                    }
550                }
551
552                // If there is a new surface texture, instantiate decoders
553                // again if possible.
554                mDeferredActions.push_back(
555                        new SimpleAction(&NuPlayer::performScanSources));
556            }
557
558            processDeferredActions();
559            break;
560        }
561
562        case kWhatSetAudioSink:
563        {
564            ALOGV("kWhatSetAudioSink");
565
566            sp<RefBase> obj;
567            CHECK(msg->findObject("sink", &obj));
568
569            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
570            break;
571        }
572
573        case kWhatStart:
574        {
575            ALOGV("kWhatStart");
576            if (mStarted) {
577                onResume();
578            } else {
579                onStart();
580            }
581            break;
582        }
583
584        case kWhatScanSources:
585        {
586            int32_t generation;
587            CHECK(msg->findInt32("generation", &generation));
588            if (generation != mScanSourcesGeneration) {
589                // Drop obsolete msg.
590                break;
591            }
592
593            mScanSourcesPending = false;
594
595            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
596                 mAudioDecoder != NULL, mVideoDecoder != NULL);
597
598            bool mHadAnySourcesBefore =
599                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
600
601            // initialize video before audio because successful initialization of
602            // video may change deep buffer mode of audio.
603            if (mNativeWindow != NULL) {
604                instantiateDecoder(false, &mVideoDecoder);
605            }
606
607            if (mAudioSink != NULL) {
608                if (mOffloadAudio) {
609                    // open audio sink early under offload mode.
610                    sp<AMessage> format = mSource->getFormat(true /*audio*/);
611                    openAudioSink(format, true /*offloadOnly*/);
612                }
613                instantiateDecoder(true, &mAudioDecoder);
614            }
615
616            if (!mHadAnySourcesBefore
617                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
618                // This is the first time we've found anything playable.
619
620                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
621                    schedulePollDuration();
622                }
623            }
624
625            status_t err;
626            if ((err = mSource->feedMoreTSData()) != OK) {
627                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
628                    // We're not currently decoding anything (no audio or
629                    // video tracks found) and we just ran out of input data.
630
631                    if (err == ERROR_END_OF_STREAM) {
632                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
633                    } else {
634                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
635                    }
636                }
637                break;
638            }
639
640            if ((mAudioDecoder == NULL && mAudioSink != NULL)
641                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
642                msg->post(100000ll);
643                mScanSourcesPending = true;
644            }
645            break;
646        }
647
648        case kWhatVideoNotify:
649        case kWhatAudioNotify:
650        {
651            bool audio = msg->what() == kWhatAudioNotify;
652
653            int32_t currentDecoderGeneration =
654                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
655            int32_t requesterGeneration = currentDecoderGeneration - 1;
656            CHECK(msg->findInt32("generation", &requesterGeneration));
657
658            if (requesterGeneration != currentDecoderGeneration) {
659                ALOGV("got message from old %s decoder, generation(%d:%d)",
660                        audio ? "audio" : "video", requesterGeneration,
661                        currentDecoderGeneration);
662                sp<AMessage> reply;
663                if (!(msg->findMessage("reply", &reply))) {
664                    return;
665                }
666
667                reply->setInt32("err", INFO_DISCONTINUITY);
668                reply->post();
669                return;
670            }
671
672            int32_t what;
673            CHECK(msg->findInt32("what", &what));
674
675            if (what == Decoder::kWhatFillThisBuffer) {
676                status_t err = feedDecoderInputData(
677                        audio, msg);
678
679                if (err == -EWOULDBLOCK) {
680                    if (mSource->feedMoreTSData() == OK) {
681                        msg->post(10 * 1000ll);
682                    }
683                }
684            } else if (what == Decoder::kWhatEOS) {
685                int32_t err;
686                CHECK(msg->findInt32("err", &err));
687
688                if (err == ERROR_END_OF_STREAM) {
689                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
690                } else {
691                    ALOGV("got %s decoder EOS w/ error %d",
692                         audio ? "audio" : "video",
693                         err);
694                }
695
696                mRenderer->queueEOS(audio, err);
697            } else if (what == Decoder::kWhatFlushCompleted) {
698                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
699
700                handleFlushComplete(audio, true /* isDecoder */);
701                finishFlushIfPossible();
702            } else if (what == Decoder::kWhatVideoSizeChanged) {
703                sp<AMessage> format;
704                CHECK(msg->findMessage("format", &format));
705
706                sp<AMessage> inputFormat =
707                        mSource->getFormat(false /* audio */);
708
709                updateVideoSize(inputFormat, format);
710            } else if (what == Decoder::kWhatShutdownCompleted) {
711                ALOGV("%s shutdown completed", audio ? "audio" : "video");
712                if (audio) {
713                    mAudioDecoder.clear();
714                    ++mAudioDecoderGeneration;
715
716                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
717                    mFlushingAudio = SHUT_DOWN;
718                } else {
719                    mVideoDecoder.clear();
720                    ++mVideoDecoderGeneration;
721
722                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
723                    mFlushingVideo = SHUT_DOWN;
724                }
725
726                finishFlushIfPossible();
727            } else if (what == Decoder::kWhatError) {
728                status_t err;
729                if (!msg->findInt32("err", &err) || err == OK) {
730                    err = UNKNOWN_ERROR;
731                }
732
733                // Decoder errors can be due to Source (e.g. from streaming),
734                // or from decoding corrupted bitstreams, or from other decoder
735                // MediaCodec operations (e.g. from an ongoing reset or seek).
736                //
737                // We try to gracefully shut down the affected decoder if possible,
738                // rather than trying to force the shutdown with something
739                // similar to performReset(). This method can lead to a hang
740                // if MediaCodec functions block after an error, but they should
741                // typically return INVALID_OPERATION instead of blocking.
742
743                FlushStatus *flushing = audio ? &mFlushingAudio : &mFlushingVideo;
744                ALOGE("received error(%#x) from %s decoder, flushing(%d), now shutting down",
745                        err, audio ? "audio" : "video", *flushing);
746
747                switch (*flushing) {
748                    case NONE:
749                        mDeferredActions.push_back(
750                                new FlushDecoderAction(
751                                    audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
752                                    audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN));
753                        processDeferredActions();
754                        break;
755                    case FLUSHING_DECODER:
756                        *flushing = FLUSHING_DECODER_SHUTDOWN; // initiate shutdown after flush.
757                        break; // Wait for flush to complete.
758                    case FLUSHING_DECODER_SHUTDOWN:
759                        break; // Wait for flush to complete.
760                    case SHUTTING_DOWN_DECODER:
761                        break; // Wait for shutdown to complete.
762                    case FLUSHED:
763                        // Widevine source reads must stop before releasing the video decoder.
764                        if (!audio && mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
765                            mSource->stop();
766                        }
767                        getDecoder(audio)->initiateShutdown(); // In the middle of a seek.
768                        *flushing = SHUTTING_DOWN_DECODER;     // Shut down.
769                        break;
770                    case SHUT_DOWN:
771                        finishFlushIfPossible();  // Should not occur.
772                        break;                    // Finish anyways.
773                }
774                notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
775            } else if (what == Decoder::kWhatRenderBufferTime) {
776                renderBuffer(audio, msg);
777            } else {
778                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
779                      what,
780                      what >> 24,
781                      (what >> 16) & 0xff,
782                      (what >> 8) & 0xff,
783                      what & 0xff);
784            }
785
786            break;
787        }
788
789        case kWhatRendererNotify:
790        {
791            int32_t requesterGeneration = mRendererGeneration - 1;
792            CHECK(msg->findInt32("generation", &requesterGeneration));
793            if (requesterGeneration != mRendererGeneration) {
794                ALOGV("got message from old renderer, generation(%d:%d)",
795                        requesterGeneration, mRendererGeneration);
796                return;
797            }
798
799            int32_t what;
800            CHECK(msg->findInt32("what", &what));
801
802            if (what == Renderer::kWhatEOS) {
803                int32_t audio;
804                CHECK(msg->findInt32("audio", &audio));
805
806                int32_t finalResult;
807                CHECK(msg->findInt32("finalResult", &finalResult));
808
809                if (audio) {
810                    mAudioEOS = true;
811                } else {
812                    mVideoEOS = true;
813                }
814
815                if (finalResult == ERROR_END_OF_STREAM) {
816                    ALOGV("reached %s EOS", audio ? "audio" : "video");
817                } else {
818                    ALOGE("%s track encountered an error (%d)",
819                         audio ? "audio" : "video", finalResult);
820
821                    notifyListener(
822                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
823                }
824
825                if ((mAudioEOS || mAudioDecoder == NULL)
826                        && (mVideoEOS || mVideoDecoder == NULL)) {
827                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
828                }
829            } else if (what == Renderer::kWhatFlushComplete) {
830                int32_t audio;
831                CHECK(msg->findInt32("audio", &audio));
832
833                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
834                handleFlushComplete(audio, false /* isDecoder */);
835                finishFlushIfPossible();
836            } else if (what == Renderer::kWhatVideoRenderingStart) {
837                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
838            } else if (what == Renderer::kWhatMediaRenderingStart) {
839                ALOGV("media rendering started");
840                notifyListener(MEDIA_STARTED, 0, 0);
841            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
842                ALOGV("Tear down audio offload, fall back to s/w path");
843                int64_t positionUs;
844                CHECK(msg->findInt64("positionUs", &positionUs));
845                int32_t reason;
846                CHECK(msg->findInt32("reason", &reason));
847                closeAudioSink();
848                mAudioDecoder.clear();
849                ++mAudioDecoderGeneration;
850                mRenderer->flush(true /* audio */);
851                if (mVideoDecoder != NULL) {
852                    mRenderer->flush(false /* audio */);
853                }
854                mRenderer->signalDisableOffloadAudio();
855                mOffloadAudio = false;
856
857                performSeek(positionUs, false /* needNotify */);
858                if (reason == Renderer::kDueToError) {
859                    instantiateDecoder(true /* audio */, &mAudioDecoder);
860                }
861            }
862            break;
863        }
864
865        case kWhatMoreDataQueued:
866        {
867            break;
868        }
869
870        case kWhatReset:
871        {
872            ALOGV("kWhatReset");
873
874            mDeferredActions.push_back(
875                    new FlushDecoderAction(
876                        FLUSH_CMD_SHUTDOWN /* audio */,
877                        FLUSH_CMD_SHUTDOWN /* video */));
878
879            mDeferredActions.push_back(
880                    new SimpleAction(&NuPlayer::performReset));
881
882            processDeferredActions();
883            break;
884        }
885
886        case kWhatSeek:
887        {
888            int64_t seekTimeUs;
889            int32_t needNotify;
890            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
891            CHECK(msg->findInt32("needNotify", &needNotify));
892
893            ALOGV("kWhatSeek seekTimeUs=%lld us, needNotify=%d",
894                    seekTimeUs, needNotify);
895
896            mDeferredActions.push_back(
897                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
898                                           FLUSH_CMD_FLUSH /* video */));
899
900            mDeferredActions.push_back(
901                    new SeekAction(seekTimeUs, needNotify));
902
903            processDeferredActions();
904            break;
905        }
906
907        case kWhatPause:
908        {
909            if (mSource != NULL) {
910                mSource->pause();
911            } else {
912                ALOGW("pause called when source is gone or not set");
913            }
914            if (mRenderer != NULL) {
915                mRenderer->pause();
916            } else {
917                ALOGW("pause called when renderer is gone or not set");
918            }
919            break;
920        }
921
922        case kWhatSourceNotify:
923        {
924            onSourceNotify(msg);
925            break;
926        }
927
928        case kWhatClosedCaptionNotify:
929        {
930            onClosedCaptionNotify(msg);
931            break;
932        }
933
934        default:
935            TRESPASS();
936            break;
937    }
938}
939
940void NuPlayer::onResume() {
941    if (mSource != NULL) {
942        mSource->resume();
943    } else {
944        ALOGW("resume called when source is gone or not set");
945    }
946    // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if
947    // needed.
948    if (audioDecoderStillNeeded() && mAudioDecoder == NULL) {
949        instantiateDecoder(true /* audio */, &mAudioDecoder);
950    }
951    if (mRenderer != NULL) {
952        mRenderer->resume();
953    } else {
954        ALOGW("resume called when renderer is gone or not set");
955    }
956}
957
958void NuPlayer::onStart() {
959    mVideoIsAVC = false;
960    mOffloadAudio = false;
961    mAudioEOS = false;
962    mVideoEOS = false;
963    mNumFramesTotal = 0;
964    mNumFramesDropped = 0;
965    mStarted = true;
966
967    /* instantiate decoders now for secure playback */
968    if (mSourceFlags & Source::FLAG_SECURE) {
969        if (mNativeWindow != NULL) {
970            instantiateDecoder(false, &mVideoDecoder);
971        }
972
973        if (mAudioSink != NULL) {
974            instantiateDecoder(true, &mAudioDecoder);
975        }
976    }
977
978    mSource->start();
979
980    uint32_t flags = 0;
981
982    if (mSource->isRealTime()) {
983        flags |= Renderer::FLAG_REAL_TIME;
984    }
985
986    sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
987    audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
988    if (mAudioSink != NULL) {
989        streamType = mAudioSink->getAudioStreamType();
990    }
991
992    sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
993
994    mOffloadAudio =
995        canOffloadStream(audioMeta, (videoFormat != NULL),
996                         true /* is_streaming */, streamType);
997    if (mOffloadAudio) {
998        flags |= Renderer::FLAG_OFFLOAD_AUDIO;
999    }
1000
1001    sp<AMessage> notify = new AMessage(kWhatRendererNotify, id());
1002    ++mRendererGeneration;
1003    notify->setInt32("generation", mRendererGeneration);
1004    mRenderer = new Renderer(mAudioSink, notify, flags);
1005
1006    mRendererLooper = new ALooper;
1007    mRendererLooper->setName("NuPlayerRenderer");
1008    mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
1009    mRendererLooper->registerHandler(mRenderer);
1010
1011    sp<MetaData> meta = getFileMeta();
1012    int32_t rate;
1013    if (meta != NULL
1014            && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) {
1015        mRenderer->setVideoFrameRate(rate);
1016    }
1017
1018    if (mVideoDecoder != NULL) {
1019        mVideoDecoder->setRenderer(mRenderer);
1020    }
1021    if (mAudioDecoder != NULL) {
1022        mAudioDecoder->setRenderer(mRenderer);
1023    }
1024
1025    postScanSources();
1026}
1027
1028bool NuPlayer::audioDecoderStillNeeded() {
1029    // Audio decoder is no longer needed if it's in shut/shutting down status.
1030    return ((mFlushingAudio != SHUT_DOWN) && (mFlushingAudio != SHUTTING_DOWN_DECODER));
1031}
1032
1033void NuPlayer::handleFlushComplete(bool audio, bool isDecoder) {
1034    // We wait for both the decoder flush and the renderer flush to complete
1035    // before entering either the FLUSHED or the SHUTTING_DOWN_DECODER state.
1036
1037    mFlushComplete[audio][isDecoder] = true;
1038    if (!mFlushComplete[audio][!isDecoder]) {
1039        return;
1040    }
1041
1042    FlushStatus *state = audio ? &mFlushingAudio : &mFlushingVideo;
1043    switch (*state) {
1044        case FLUSHING_DECODER:
1045        {
1046            *state = FLUSHED;
1047            break;
1048        }
1049
1050        case FLUSHING_DECODER_SHUTDOWN:
1051        {
1052            *state = SHUTTING_DOWN_DECODER;
1053
1054            ALOGV("initiating %s decoder shutdown", audio ? "audio" : "video");
1055            if (!audio) {
1056                // Widevine source reads must stop before releasing the video decoder.
1057                if (mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
1058                    mSource->stop();
1059                }
1060            }
1061            getDecoder(audio)->initiateShutdown();
1062            break;
1063        }
1064
1065        default:
1066            // decoder flush completes only occur in a flushing state.
1067            LOG_ALWAYS_FATAL_IF(isDecoder, "decoder flush in invalid state %d", *state);
1068            break;
1069    }
1070}
1071
1072void NuPlayer::finishFlushIfPossible() {
1073    if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED
1074            && mFlushingAudio != SHUT_DOWN) {
1075        return;
1076    }
1077
1078    if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED
1079            && mFlushingVideo != SHUT_DOWN) {
1080        return;
1081    }
1082
1083    ALOGV("both audio and video are flushed now.");
1084
1085    mPendingAudioAccessUnit.clear();
1086    mAggregateBuffer.clear();
1087
1088    if (mTimeDiscontinuityPending) {
1089        mRenderer->signalTimeDiscontinuity();
1090        mTimeDiscontinuityPending = false;
1091    }
1092
1093    if (mAudioDecoder != NULL && mFlushingAudio == FLUSHED) {
1094        mAudioDecoder->signalResume();
1095    }
1096
1097    if (mVideoDecoder != NULL && mFlushingVideo == FLUSHED) {
1098        mVideoDecoder->signalResume();
1099    }
1100
1101    mFlushingAudio = NONE;
1102    mFlushingVideo = NONE;
1103
1104    clearFlushComplete();
1105
1106    processDeferredActions();
1107}
1108
1109void NuPlayer::postScanSources() {
1110    if (mScanSourcesPending) {
1111        return;
1112    }
1113
1114    sp<AMessage> msg = new AMessage(kWhatScanSources, id());
1115    msg->setInt32("generation", mScanSourcesGeneration);
1116    msg->post();
1117
1118    mScanSourcesPending = true;
1119}
1120
1121void NuPlayer::openAudioSink(const sp<AMessage> &format, bool offloadOnly) {
1122    uint32_t flags;
1123    int64_t durationUs;
1124    bool hasVideo = (mVideoDecoder != NULL);
1125    // FIXME: we should handle the case where the video decoder
1126    // is created after we receive the format change indication.
1127    // Current code will just make that we select deep buffer
1128    // with video which should not be a problem as it should
1129    // not prevent from keeping A/V sync.
1130    if (!hasVideo &&
1131            mSource->getDuration(&durationUs) == OK &&
1132            durationUs
1133                > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
1134        flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1135    } else {
1136        flags = AUDIO_OUTPUT_FLAG_NONE;
1137    }
1138
1139    mOffloadAudio = mRenderer->openAudioSink(
1140            format, offloadOnly, hasVideo, flags);
1141
1142    if (mOffloadAudio) {
1143        sp<MetaData> audioMeta =
1144                mSource->getFormatMeta(true /* audio */);
1145        sendMetaDataToHal(mAudioSink, audioMeta);
1146    }
1147}
1148
1149void NuPlayer::closeAudioSink() {
1150    mRenderer->closeAudioSink();
1151}
1152
1153status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
1154    if (*decoder != NULL) {
1155        return OK;
1156    }
1157
1158    sp<AMessage> format = mSource->getFormat(audio);
1159
1160    if (format == NULL) {
1161        return -EWOULDBLOCK;
1162    }
1163
1164    if (!audio) {
1165        AString mime;
1166        CHECK(format->findString("mime", &mime));
1167        mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
1168
1169        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
1170        mCCDecoder = new CCDecoder(ccNotify);
1171
1172        if (mSourceFlags & Source::FLAG_SECURE) {
1173            format->setInt32("secure", true);
1174        }
1175    }
1176
1177    if (audio) {
1178        sp<AMessage> notify = new AMessage(kWhatAudioNotify, id());
1179        ++mAudioDecoderGeneration;
1180        notify->setInt32("generation", mAudioDecoderGeneration);
1181
1182        if (mOffloadAudio) {
1183            *decoder = new DecoderPassThrough(notify, mSource, mRenderer);
1184        } else {
1185            *decoder = new Decoder(notify, mSource, mRenderer);
1186        }
1187    } else {
1188        sp<AMessage> notify = new AMessage(kWhatVideoNotify, id());
1189        ++mVideoDecoderGeneration;
1190        notify->setInt32("generation", mVideoDecoderGeneration);
1191
1192        *decoder = new Decoder(notify, mSource, mRenderer, mNativeWindow);
1193    }
1194    (*decoder)->init();
1195    (*decoder)->configure(format);
1196
1197    // allocate buffers to decrypt widevine source buffers
1198    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1199        Vector<sp<ABuffer> > inputBufs;
1200        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1201
1202        Vector<MediaBuffer *> mediaBufs;
1203        for (size_t i = 0; i < inputBufs.size(); i++) {
1204            const sp<ABuffer> &buffer = inputBufs[i];
1205            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1206            mediaBufs.push(mbuf);
1207        }
1208
1209        status_t err = mSource->setBuffers(audio, mediaBufs);
1210        if (err != OK) {
1211            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1212                mediaBufs[i]->release();
1213            }
1214            mediaBufs.clear();
1215            ALOGE("Secure source didn't support secure mediaBufs.");
1216            return err;
1217        }
1218    }
1219    return OK;
1220}
1221
1222status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
1223    sp<AMessage> reply;
1224    CHECK(msg->findMessage("reply", &reply));
1225
1226    if ((audio && mFlushingAudio != NONE)
1227            || (!audio && mFlushingVideo != NONE)
1228            || mSource == NULL) {
1229        reply->setInt32("err", INFO_DISCONTINUITY);
1230        reply->post();
1231        return OK;
1232    }
1233
1234    sp<ABuffer> accessUnit;
1235
1236    // Aggregate smaller buffers into a larger buffer.
1237    // The goal is to reduce power consumption.
1238    // Note this will not work if the decoder requires one frame per buffer.
1239    bool doBufferAggregation = (audio && mOffloadAudio);
1240    bool needMoreData = false;
1241
1242    bool dropAccessUnit;
1243    do {
1244        status_t err;
1245        // Did we save an accessUnit earlier because of a discontinuity?
1246        if (audio && (mPendingAudioAccessUnit != NULL)) {
1247            accessUnit = mPendingAudioAccessUnit;
1248            mPendingAudioAccessUnit.clear();
1249            err = mPendingAudioErr;
1250            ALOGV("feedDecoderInputData() use mPendingAudioAccessUnit");
1251        } else {
1252            err = mSource->dequeueAccessUnit(audio, &accessUnit);
1253        }
1254
1255        if (err == -EWOULDBLOCK) {
1256            return err;
1257        } else if (err != OK) {
1258            if (err == INFO_DISCONTINUITY) {
1259                if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1260                    // We already have some data so save this for later.
1261                    mPendingAudioErr = err;
1262                    mPendingAudioAccessUnit = accessUnit;
1263                    accessUnit.clear();
1264                    ALOGD("feedDecoderInputData() save discontinuity for later");
1265                    break;
1266                }
1267                int32_t type;
1268                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
1269
1270                bool formatChange =
1271                    (audio &&
1272                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
1273                    || (!audio &&
1274                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
1275
1276                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
1277
1278                ALOGI("%s discontinuity (formatChange=%d, time=%d)",
1279                     audio ? "audio" : "video", formatChange, timeChange);
1280
1281                mTimeDiscontinuityPending =
1282                    mTimeDiscontinuityPending || timeChange;
1283
1284                bool seamlessFormatChange = false;
1285                sp<AMessage> newFormat = mSource->getFormat(audio);
1286                if (formatChange) {
1287                    seamlessFormatChange =
1288                        getDecoder(audio)->supportsSeamlessFormatChange(newFormat);
1289                    // treat seamless format change separately
1290                    formatChange = !seamlessFormatChange;
1291                }
1292                bool shutdownOrFlush = formatChange || timeChange;
1293
1294                // We want to queue up scan-sources only once per discontinuity.
1295                // We control this by doing it only if neither audio nor video are
1296                // flushing or shutting down.  (After handling 1st discontinuity, one
1297                // of the flushing states will not be NONE.)
1298                // No need to scan sources if this discontinuity does not result
1299                // in a flush or shutdown, as the flushing state will stay NONE.
1300                if (mFlushingAudio == NONE && mFlushingVideo == NONE &&
1301                        shutdownOrFlush) {
1302                    // And we'll resume scanning sources once we're done
1303                    // flushing.
1304                    mDeferredActions.push_front(
1305                            new SimpleAction(
1306                                &NuPlayer::performScanSources));
1307                }
1308
1309                if (formatChange /* not seamless */) {
1310                    // must change decoder
1311                    flushDecoder(audio, /* needShutdown = */ true);
1312                } else if (timeChange) {
1313                    // need to flush
1314                    flushDecoder(audio, /* needShutdown = */ false, newFormat);
1315                    err = OK;
1316                } else if (seamlessFormatChange) {
1317                    // reuse existing decoder and don't flush
1318                    updateDecoderFormatWithoutFlush(audio, newFormat);
1319                    err = OK;
1320                } else {
1321                    // This stream is unaffected by the discontinuity
1322                    return -EWOULDBLOCK;
1323                }
1324            }
1325
1326            reply->setInt32("err", err);
1327            reply->post();
1328            return OK;
1329        }
1330
1331        if (!audio) {
1332            ++mNumFramesTotal;
1333        }
1334
1335        dropAccessUnit = false;
1336        if (!audio
1337                && !(mSourceFlags & Source::FLAG_SECURE)
1338                && mRenderer->getVideoLateByUs() > 100000ll
1339                && mVideoIsAVC
1340                && !IsAVCReferenceFrame(accessUnit)) {
1341            dropAccessUnit = true;
1342            ++mNumFramesDropped;
1343        }
1344
1345        size_t smallSize = accessUnit->size();
1346        needMoreData = false;
1347        if (doBufferAggregation && (mAggregateBuffer == NULL)
1348                // Don't bother if only room for a few small buffers.
1349                && (smallSize < (kAggregateBufferSizeBytes / 3))) {
1350            // Create a larger buffer for combining smaller buffers from the extractor.
1351            mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes);
1352            mAggregateBuffer->setRange(0, 0); // start empty
1353        }
1354
1355        if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1356            int64_t timeUs;
1357            int64_t dummy;
1358            bool smallTimestampValid = accessUnit->meta()->findInt64("timeUs", &timeUs);
1359            bool bigTimestampValid = mAggregateBuffer->meta()->findInt64("timeUs", &dummy);
1360            // Will the smaller buffer fit?
1361            size_t bigSize = mAggregateBuffer->size();
1362            size_t roomLeft = mAggregateBuffer->capacity() - bigSize;
1363            // Should we save this small buffer for the next big buffer?
1364            // If the first small buffer did not have a timestamp then save
1365            // any buffer that does have a timestamp until the next big buffer.
1366            if ((smallSize > roomLeft)
1367                || (!bigTimestampValid && (bigSize > 0) && smallTimestampValid)) {
1368                mPendingAudioErr = err;
1369                mPendingAudioAccessUnit = accessUnit;
1370                accessUnit.clear();
1371            } else {
1372                // Grab time from first small buffer if available.
1373                if ((bigSize == 0) && smallTimestampValid) {
1374                    mAggregateBuffer->meta()->setInt64("timeUs", timeUs);
1375                }
1376                // Append small buffer to the bigger buffer.
1377                memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize);
1378                bigSize += smallSize;
1379                mAggregateBuffer->setRange(0, bigSize);
1380
1381                // Keep looping until we run out of room in the mAggregateBuffer.
1382                needMoreData = true;
1383
1384                ALOGV("feedDecoderInputData() smallSize = %zu, bigSize = %zu, capacity = %zu",
1385                        smallSize, bigSize, mAggregateBuffer->capacity());
1386            }
1387        }
1388    } while (dropAccessUnit || needMoreData);
1389
1390    // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
1391
1392#if 0
1393    int64_t mediaTimeUs;
1394    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
1395    ALOGV("feeding %s input buffer at media time %.2f secs",
1396         audio ? "audio" : "video",
1397         mediaTimeUs / 1E6);
1398#endif
1399
1400    if (!audio) {
1401        mCCDecoder->decode(accessUnit);
1402    }
1403
1404    if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1405        ALOGV("feedDecoderInputData() reply with aggregated buffer, %zu",
1406                mAggregateBuffer->size());
1407        reply->setBuffer("buffer", mAggregateBuffer);
1408        mAggregateBuffer.clear();
1409    } else {
1410        reply->setBuffer("buffer", accessUnit);
1411    }
1412
1413    reply->post();
1414
1415    return OK;
1416}
1417
1418void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
1419    // ALOGV("renderBuffer %s", audio ? "audio" : "video");
1420
1421    if ((audio && mFlushingAudio != NONE)
1422            || (!audio && mFlushingVideo != NONE)) {
1423        // We're currently attempting to flush the decoder, in order
1424        // to complete this, the decoder wants all its buffers back,
1425        // so we don't want any output buffers it sent us (from before
1426        // we initiated the flush) to be stuck in the renderer's queue.
1427
1428        ALOGV("we're still flushing the %s decoder, sending its output buffer"
1429             " right back.", audio ? "audio" : "video");
1430
1431        return;
1432    }
1433
1434    int64_t mediaTimeUs;
1435    CHECK(msg->findInt64("timeUs", &mediaTimeUs));
1436
1437    if (!audio && mCCDecoder->isSelected()) {
1438        mCCDecoder->display(mediaTimeUs);
1439    }
1440}
1441
1442void NuPlayer::updateVideoSize(
1443        const sp<AMessage> &inputFormat,
1444        const sp<AMessage> &outputFormat) {
1445    if (inputFormat == NULL) {
1446        ALOGW("Unknown video size, reporting 0x0!");
1447        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1448        return;
1449    }
1450
1451    int32_t displayWidth, displayHeight;
1452    int32_t cropLeft, cropTop, cropRight, cropBottom;
1453
1454    if (outputFormat != NULL) {
1455        int32_t width, height;
1456        CHECK(outputFormat->findInt32("width", &width));
1457        CHECK(outputFormat->findInt32("height", &height));
1458
1459        int32_t cropLeft, cropTop, cropRight, cropBottom;
1460        CHECK(outputFormat->findRect(
1461                    "crop",
1462                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1463
1464        displayWidth = cropRight - cropLeft + 1;
1465        displayHeight = cropBottom - cropTop + 1;
1466
1467        ALOGV("Video output format changed to %d x %d "
1468             "(crop: %d x %d @ (%d, %d))",
1469             width, height,
1470             displayWidth,
1471             displayHeight,
1472             cropLeft, cropTop);
1473    } else {
1474        CHECK(inputFormat->findInt32("width", &displayWidth));
1475        CHECK(inputFormat->findInt32("height", &displayHeight));
1476
1477        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1478    }
1479
1480    // Take into account sample aspect ratio if necessary:
1481    int32_t sarWidth, sarHeight;
1482    if (inputFormat->findInt32("sar-width", &sarWidth)
1483            && inputFormat->findInt32("sar-height", &sarHeight)) {
1484        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1485
1486        displayWidth = (displayWidth * sarWidth) / sarHeight;
1487
1488        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1489    }
1490
1491    int32_t rotationDegrees;
1492    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1493        rotationDegrees = 0;
1494    }
1495
1496    if (rotationDegrees == 90 || rotationDegrees == 270) {
1497        int32_t tmp = displayWidth;
1498        displayWidth = displayHeight;
1499        displayHeight = tmp;
1500    }
1501
1502    notifyListener(
1503            MEDIA_SET_VIDEO_SIZE,
1504            displayWidth,
1505            displayHeight);
1506}
1507
1508void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1509    if (mDriver == NULL) {
1510        return;
1511    }
1512
1513    sp<NuPlayerDriver> driver = mDriver.promote();
1514
1515    if (driver == NULL) {
1516        return;
1517    }
1518
1519    driver->notifyListener(msg, ext1, ext2, in);
1520}
1521
1522void NuPlayer::flushDecoder(
1523        bool audio, bool needShutdown, const sp<AMessage> &newFormat) {
1524    ALOGV("[%s] flushDecoder needShutdown=%d",
1525          audio ? "audio" : "video", needShutdown);
1526
1527    const sp<Decoder> &decoder = getDecoder(audio);
1528    if (decoder == NULL) {
1529        ALOGI("flushDecoder %s without decoder present",
1530             audio ? "audio" : "video");
1531        return;
1532    }
1533
1534    // Make sure we don't continue to scan sources until we finish flushing.
1535    ++mScanSourcesGeneration;
1536    mScanSourcesPending = false;
1537
1538    decoder->signalFlush(newFormat);
1539
1540    FlushStatus newStatus =
1541        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1542
1543    mFlushComplete[audio][false /* isDecoder */] = false;
1544    mFlushComplete[audio][true /* isDecoder */] = false;
1545    if (audio) {
1546        ALOGE_IF(mFlushingAudio != NONE,
1547                "audio flushDecoder() is called in state %d", mFlushingAudio);
1548        mFlushingAudio = newStatus;
1549    } else {
1550        ALOGE_IF(mFlushingVideo != NONE,
1551                "video flushDecoder() is called in state %d", mFlushingVideo);
1552        mFlushingVideo = newStatus;
1553
1554        if (mCCDecoder != NULL) {
1555            mCCDecoder->flush();
1556        }
1557    }
1558}
1559
1560void NuPlayer::updateDecoderFormatWithoutFlush(
1561        bool audio, const sp<AMessage> &format) {
1562    ALOGV("[%s] updateDecoderFormatWithoutFlush", audio ? "audio" : "video");
1563
1564    const sp<Decoder> &decoder = getDecoder(audio);
1565    if (decoder == NULL) {
1566        ALOGI("updateDecoderFormatWithoutFlush %s without decoder present",
1567             audio ? "audio" : "video");
1568        return;
1569    }
1570
1571    decoder->signalUpdateFormat(format);
1572}
1573
1574void NuPlayer::queueDecoderShutdown(
1575        bool audio, bool video, const sp<AMessage> &reply) {
1576    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1577
1578    mDeferredActions.push_back(
1579            new FlushDecoderAction(
1580                audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
1581                video ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE));
1582
1583    mDeferredActions.push_back(
1584            new SimpleAction(&NuPlayer::performScanSources));
1585
1586    mDeferredActions.push_back(new PostMessageAction(reply));
1587
1588    processDeferredActions();
1589}
1590
1591status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1592    mVideoScalingMode = mode;
1593    if (mNativeWindow != NULL) {
1594        status_t ret = native_window_set_scaling_mode(
1595                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1596        if (ret != OK) {
1597            ALOGE("Failed to set scaling mode (%d): %s",
1598                -ret, strerror(-ret));
1599            return ret;
1600        }
1601    }
1602    return OK;
1603}
1604
1605status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1606    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id());
1607    msg->setPointer("reply", reply);
1608
1609    sp<AMessage> response;
1610    status_t err = msg->postAndAwaitResponse(&response);
1611    return err;
1612}
1613
1614status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1615    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id());
1616    msg->setPointer("reply", reply);
1617    msg->setInt32("type", type);
1618
1619    sp<AMessage> response;
1620    status_t err = msg->postAndAwaitResponse(&response);
1621    if (err == OK && response != NULL) {
1622        CHECK(response->findInt32("err", &err));
1623    }
1624    return err;
1625}
1626
1627status_t NuPlayer::selectTrack(size_t trackIndex, bool select) {
1628    sp<AMessage> msg = new AMessage(kWhatSelectTrack, id());
1629    msg->setSize("trackIndex", trackIndex);
1630    msg->setInt32("select", select);
1631
1632    sp<AMessage> response;
1633    status_t err = msg->postAndAwaitResponse(&response);
1634
1635    if (err != OK) {
1636        return err;
1637    }
1638
1639    if (!response->findInt32("err", &err)) {
1640        err = OK;
1641    }
1642
1643    return err;
1644}
1645
1646status_t NuPlayer::getCurrentPosition(int64_t *mediaUs) {
1647    sp<Renderer> renderer = mRenderer;
1648    if (renderer == NULL) {
1649        return NO_INIT;
1650    }
1651
1652    return renderer->getCurrentPosition(mediaUs);
1653}
1654
1655void NuPlayer::getStats(int64_t *numFramesTotal, int64_t *numFramesDropped) {
1656    *numFramesTotal = mNumFramesTotal;
1657    *numFramesDropped = mNumFramesDropped;
1658}
1659
1660sp<MetaData> NuPlayer::getFileMeta() {
1661    return mSource->getFileFormatMeta();
1662}
1663
1664void NuPlayer::schedulePollDuration() {
1665    sp<AMessage> msg = new AMessage(kWhatPollDuration, id());
1666    msg->setInt32("generation", mPollDurationGeneration);
1667    msg->post();
1668}
1669
1670void NuPlayer::cancelPollDuration() {
1671    ++mPollDurationGeneration;
1672}
1673
1674void NuPlayer::processDeferredActions() {
1675    while (!mDeferredActions.empty()) {
1676        // We won't execute any deferred actions until we're no longer in
1677        // an intermediate state, i.e. one more more decoders are currently
1678        // flushing or shutting down.
1679
1680        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1681            // We're currently flushing, postpone the reset until that's
1682            // completed.
1683
1684            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1685                  mFlushingAudio, mFlushingVideo);
1686
1687            break;
1688        }
1689
1690        sp<Action> action = *mDeferredActions.begin();
1691        mDeferredActions.erase(mDeferredActions.begin());
1692
1693        action->execute(this);
1694    }
1695}
1696
1697void NuPlayer::performSeek(int64_t seekTimeUs, bool needNotify) {
1698    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs), needNotify(%d)",
1699          seekTimeUs,
1700          seekTimeUs / 1E6,
1701          needNotify);
1702
1703    if (mSource == NULL) {
1704        // This happens when reset occurs right before the loop mode
1705        // asynchronously seeks to the start of the stream.
1706        LOG_ALWAYS_FATAL_IF(mAudioDecoder != NULL || mVideoDecoder != NULL,
1707                "mSource is NULL and decoders not NULL audio(%p) video(%p)",
1708                mAudioDecoder.get(), mVideoDecoder.get());
1709        return;
1710    }
1711    mSource->seekTo(seekTimeUs);
1712    ++mTimedTextGeneration;
1713
1714    if (mDriver != NULL) {
1715        sp<NuPlayerDriver> driver = mDriver.promote();
1716        if (driver != NULL) {
1717            if (needNotify) {
1718                driver->notifySeekComplete();
1719            }
1720        }
1721    }
1722
1723    // everything's flushed, continue playback.
1724}
1725
1726void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) {
1727    ALOGV("performDecoderFlush audio=%d, video=%d", audio, video);
1728
1729    if ((audio == FLUSH_CMD_NONE || mAudioDecoder == NULL)
1730            && (video == FLUSH_CMD_NONE || mVideoDecoder == NULL)) {
1731        return;
1732    }
1733
1734    mTimeDiscontinuityPending = true;
1735
1736    if (audio != FLUSH_CMD_NONE && mAudioDecoder != NULL) {
1737        flushDecoder(true /* audio */, (audio == FLUSH_CMD_SHUTDOWN));
1738    }
1739
1740    if (video != FLUSH_CMD_NONE && mVideoDecoder != NULL) {
1741        flushDecoder(false /* audio */, (video == FLUSH_CMD_SHUTDOWN));
1742    }
1743}
1744
1745void NuPlayer::performReset() {
1746    ALOGV("performReset");
1747
1748    CHECK(mAudioDecoder == NULL);
1749    CHECK(mVideoDecoder == NULL);
1750
1751    cancelPollDuration();
1752
1753    ++mScanSourcesGeneration;
1754    mScanSourcesPending = false;
1755
1756    if (mRendererLooper != NULL) {
1757        if (mRenderer != NULL) {
1758            mRendererLooper->unregisterHandler(mRenderer->id());
1759        }
1760        mRendererLooper->stop();
1761        mRendererLooper.clear();
1762    }
1763    mRenderer.clear();
1764    ++mRendererGeneration;
1765
1766    if (mSource != NULL) {
1767        mSource->stop();
1768
1769        mSource.clear();
1770    }
1771
1772    if (mDriver != NULL) {
1773        sp<NuPlayerDriver> driver = mDriver.promote();
1774        if (driver != NULL) {
1775            driver->notifyResetComplete();
1776        }
1777    }
1778
1779    mStarted = false;
1780}
1781
1782void NuPlayer::performScanSources() {
1783    ALOGV("performScanSources");
1784
1785    if (!mStarted) {
1786        return;
1787    }
1788
1789    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1790        postScanSources();
1791    }
1792}
1793
1794void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1795    ALOGV("performSetSurface");
1796
1797    mNativeWindow = wrapper;
1798
1799    // XXX - ignore error from setVideoScalingMode for now
1800    setVideoScalingMode(mVideoScalingMode);
1801
1802    if (mDriver != NULL) {
1803        sp<NuPlayerDriver> driver = mDriver.promote();
1804        if (driver != NULL) {
1805            driver->notifySetSurfaceComplete();
1806        }
1807    }
1808}
1809
1810void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1811    int32_t what;
1812    CHECK(msg->findInt32("what", &what));
1813
1814    switch (what) {
1815        case Source::kWhatPrepared:
1816        {
1817            if (mSource == NULL) {
1818                // This is a stale notification from a source that was
1819                // asynchronously preparing when the client called reset().
1820                // We handled the reset, the source is gone.
1821                break;
1822            }
1823
1824            int32_t err;
1825            CHECK(msg->findInt32("err", &err));
1826
1827            sp<NuPlayerDriver> driver = mDriver.promote();
1828            if (driver != NULL) {
1829                // notify duration first, so that it's definitely set when
1830                // the app received the "prepare complete" callback.
1831                int64_t durationUs;
1832                if (mSource->getDuration(&durationUs) == OK) {
1833                    driver->notifyDuration(durationUs);
1834                }
1835                driver->notifyPrepareCompleted(err);
1836            }
1837
1838            break;
1839        }
1840
1841        case Source::kWhatFlagsChanged:
1842        {
1843            uint32_t flags;
1844            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1845
1846            sp<NuPlayerDriver> driver = mDriver.promote();
1847            if (driver != NULL) {
1848                driver->notifyFlagsChanged(flags);
1849            }
1850
1851            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1852                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
1853                cancelPollDuration();
1854            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1855                    && (flags & Source::FLAG_DYNAMIC_DURATION)
1856                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
1857                schedulePollDuration();
1858            }
1859
1860            mSourceFlags = flags;
1861            break;
1862        }
1863
1864        case Source::kWhatVideoSizeChanged:
1865        {
1866            sp<AMessage> format;
1867            CHECK(msg->findMessage("format", &format));
1868
1869            updateVideoSize(format);
1870            break;
1871        }
1872
1873        case Source::kWhatBufferingUpdate:
1874        {
1875            int32_t percentage;
1876            CHECK(msg->findInt32("percentage", &percentage));
1877
1878            notifyListener(MEDIA_BUFFERING_UPDATE, percentage, 0);
1879            break;
1880        }
1881
1882        case Source::kWhatBufferingStart:
1883        {
1884            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
1885            break;
1886        }
1887
1888        case Source::kWhatBufferingEnd:
1889        {
1890            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
1891            break;
1892        }
1893
1894        case Source::kWhatSubtitleData:
1895        {
1896            sp<ABuffer> buffer;
1897            CHECK(msg->findBuffer("buffer", &buffer));
1898
1899            sendSubtitleData(buffer, 0 /* baseIndex */);
1900            break;
1901        }
1902
1903        case Source::kWhatTimedTextData:
1904        {
1905            int32_t generation;
1906            if (msg->findInt32("generation", &generation)
1907                    && generation != mTimedTextGeneration) {
1908                break;
1909            }
1910
1911            sp<ABuffer> buffer;
1912            CHECK(msg->findBuffer("buffer", &buffer));
1913
1914            sp<NuPlayerDriver> driver = mDriver.promote();
1915            if (driver == NULL) {
1916                break;
1917            }
1918
1919            int posMs;
1920            int64_t timeUs, posUs;
1921            driver->getCurrentPosition(&posMs);
1922            posUs = posMs * 1000;
1923            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1924
1925            if (posUs < timeUs) {
1926                if (!msg->findInt32("generation", &generation)) {
1927                    msg->setInt32("generation", mTimedTextGeneration);
1928                }
1929                msg->post(timeUs - posUs);
1930            } else {
1931                sendTimedTextData(buffer);
1932            }
1933            break;
1934        }
1935
1936        case Source::kWhatQueueDecoderShutdown:
1937        {
1938            int32_t audio, video;
1939            CHECK(msg->findInt32("audio", &audio));
1940            CHECK(msg->findInt32("video", &video));
1941
1942            sp<AMessage> reply;
1943            CHECK(msg->findMessage("reply", &reply));
1944
1945            queueDecoderShutdown(audio, video, reply);
1946            break;
1947        }
1948
1949        case Source::kWhatDrmNoLicense:
1950        {
1951            notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, ERROR_DRM_NO_LICENSE);
1952            break;
1953        }
1954
1955        default:
1956            TRESPASS();
1957    }
1958}
1959
1960void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
1961    int32_t what;
1962    CHECK(msg->findInt32("what", &what));
1963
1964    switch (what) {
1965        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
1966        {
1967            sp<ABuffer> buffer;
1968            CHECK(msg->findBuffer("buffer", &buffer));
1969
1970            size_t inbandTracks = 0;
1971            if (mSource != NULL) {
1972                inbandTracks = mSource->getTrackCount();
1973            }
1974
1975            sendSubtitleData(buffer, inbandTracks);
1976            break;
1977        }
1978
1979        case NuPlayer::CCDecoder::kWhatTrackAdded:
1980        {
1981            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
1982
1983            break;
1984        }
1985
1986        default:
1987            TRESPASS();
1988    }
1989
1990
1991}
1992
1993void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
1994    int32_t trackIndex;
1995    int64_t timeUs, durationUs;
1996    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
1997    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1998    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
1999
2000    Parcel in;
2001    in.writeInt32(trackIndex + baseIndex);
2002    in.writeInt64(timeUs);
2003    in.writeInt64(durationUs);
2004    in.writeInt32(buffer->size());
2005    in.writeInt32(buffer->size());
2006    in.write(buffer->data(), buffer->size());
2007
2008    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
2009}
2010
2011void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
2012    const void *data;
2013    size_t size = 0;
2014    int64_t timeUs;
2015    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
2016
2017    AString mime;
2018    CHECK(buffer->meta()->findString("mime", &mime));
2019    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
2020
2021    data = buffer->data();
2022    size = buffer->size();
2023
2024    Parcel parcel;
2025    if (size > 0) {
2026        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2027        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
2028        TextDescriptions::getParcelOfDescriptions(
2029                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
2030    }
2031
2032    if ((parcel.dataSize() > 0)) {
2033        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
2034    } else {  // send an empty timed text
2035        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
2036    }
2037}
2038////////////////////////////////////////////////////////////////////////////////
2039
2040sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
2041    sp<MetaData> meta = getFormatMeta(audio);
2042
2043    if (meta == NULL) {
2044        return NULL;
2045    }
2046
2047    sp<AMessage> msg = new AMessage;
2048
2049    if(convertMetaDataToMessage(meta, &msg) == OK) {
2050        return msg;
2051    }
2052    return NULL;
2053}
2054
2055void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
2056    sp<AMessage> notify = dupNotify();
2057    notify->setInt32("what", kWhatFlagsChanged);
2058    notify->setInt32("flags", flags);
2059    notify->post();
2060}
2061
2062void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
2063    sp<AMessage> notify = dupNotify();
2064    notify->setInt32("what", kWhatVideoSizeChanged);
2065    notify->setMessage("format", format);
2066    notify->post();
2067}
2068
2069void NuPlayer::Source::notifyPrepared(status_t err) {
2070    sp<AMessage> notify = dupNotify();
2071    notify->setInt32("what", kWhatPrepared);
2072    notify->setInt32("err", err);
2073    notify->post();
2074}
2075
2076void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
2077    TRESPASS();
2078}
2079
2080}  // namespace android
2081