NuPlayer.cpp revision 6c41e622512a99581afcbdbd6ad284237923c942 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerDecoder.h"
25#include "NuPlayerDecoderPassThrough.h"
26#include "NuPlayerDriver.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29#include "RTSPSource.h"
30#include "StreamingSource.h"
31#include "GenericSource.h"
32#include "TextDescriptions.h"
33
34#include "ATSParser.h"
35
36#include <media/stagefright/foundation/hexdump.h>
37#include <media/stagefright/foundation/ABuffer.h>
38#include <media/stagefright/foundation/ADebug.h>
39#include <media/stagefright/foundation/AMessage.h>
40#include <media/stagefright/MediaBuffer.h>
41#include <media/stagefright/MediaDefs.h>
42#include <media/stagefright/MediaErrors.h>
43#include <media/stagefright/MetaData.h>
44#include <gui/IGraphicBufferProducer.h>
45
46#include "avc_utils.h"
47
48#include "ESDS.h"
49#include <media/stagefright/Utils.h>
50
51namespace android {
52
53// TODO optimize buffer size for power consumption
54// The offload read buffer size is 32 KB but 24 KB uses less power.
55const size_t NuPlayer::kAggregateBufferSizeBytes = 24 * 1024;
56
57struct NuPlayer::Action : public RefBase {
58    Action() {}
59
60    virtual void execute(NuPlayer *player) = 0;
61
62private:
63    DISALLOW_EVIL_CONSTRUCTORS(Action);
64};
65
66struct NuPlayer::SeekAction : public Action {
67    SeekAction(int64_t seekTimeUs, bool needNotify)
68        : mSeekTimeUs(seekTimeUs),
69          mNeedNotify(needNotify) {
70    }
71
72    virtual void execute(NuPlayer *player) {
73        player->performSeek(mSeekTimeUs, mNeedNotify);
74    }
75
76private:
77    int64_t mSeekTimeUs;
78    bool mNeedNotify;
79
80    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
81};
82
83struct NuPlayer::SetSurfaceAction : public Action {
84    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
85        : mWrapper(wrapper) {
86    }
87
88    virtual void execute(NuPlayer *player) {
89        player->performSetSurface(mWrapper);
90    }
91
92private:
93    sp<NativeWindowWrapper> mWrapper;
94
95    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
96};
97
98struct NuPlayer::FlushDecoderAction : public Action {
99    FlushDecoderAction(FlushCommand audio, FlushCommand video)
100        : mAudio(audio),
101          mVideo(video) {
102    }
103
104    virtual void execute(NuPlayer *player) {
105        player->performDecoderFlush(mAudio, mVideo);
106    }
107
108private:
109    FlushCommand mAudio;
110    FlushCommand mVideo;
111
112    DISALLOW_EVIL_CONSTRUCTORS(FlushDecoderAction);
113};
114
115struct NuPlayer::PostMessageAction : public Action {
116    PostMessageAction(const sp<AMessage> &msg)
117        : mMessage(msg) {
118    }
119
120    virtual void execute(NuPlayer *) {
121        mMessage->post();
122    }
123
124private:
125    sp<AMessage> mMessage;
126
127    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
128};
129
130// Use this if there's no state necessary to save in order to execute
131// the action.
132struct NuPlayer::SimpleAction : public Action {
133    typedef void (NuPlayer::*ActionFunc)();
134
135    SimpleAction(ActionFunc func)
136        : mFunc(func) {
137    }
138
139    virtual void execute(NuPlayer *player) {
140        (player->*mFunc)();
141    }
142
143private:
144    ActionFunc mFunc;
145
146    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
147};
148
149////////////////////////////////////////////////////////////////////////////////
150
151NuPlayer::NuPlayer()
152    : mUIDValid(false),
153      mSourceFlags(0),
154      mVideoIsAVC(false),
155      mOffloadAudio(false),
156      mAudioDecoderGeneration(0),
157      mVideoDecoderGeneration(0),
158      mRendererGeneration(0),
159      mAudioEOS(false),
160      mVideoEOS(false),
161      mScanSourcesPending(false),
162      mScanSourcesGeneration(0),
163      mPollDurationGeneration(0),
164      mTimedTextGeneration(0),
165      mTimeDiscontinuityPending(false),
166      mFlushingAudio(NONE),
167      mFlushingVideo(NONE),
168      mNumFramesTotal(0ll),
169      mNumFramesDropped(0ll),
170      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
171      mStarted(false) {
172    clearFlushComplete();
173}
174
175NuPlayer::~NuPlayer() {
176}
177
178void NuPlayer::setUID(uid_t uid) {
179    mUIDValid = true;
180    mUID = uid;
181}
182
183void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
184    mDriver = driver;
185}
186
187void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
188    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
189
190    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
191
192    msg->setObject("source", new StreamingSource(notify, source));
193    msg->post();
194}
195
196static bool IsHTTPLiveURL(const char *url) {
197    if (!strncasecmp("http://", url, 7)
198            || !strncasecmp("https://", url, 8)
199            || !strncasecmp("file://", url, 7)) {
200        size_t len = strlen(url);
201        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
202            return true;
203        }
204
205        if (strstr(url,"m3u8")) {
206            return true;
207        }
208    }
209
210    return false;
211}
212
213void NuPlayer::setDataSourceAsync(
214        const sp<IMediaHTTPService> &httpService,
215        const char *url,
216        const KeyedVector<String8, String8> *headers) {
217
218    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
219    size_t len = strlen(url);
220
221    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
222
223    sp<Source> source;
224    if (IsHTTPLiveURL(url)) {
225        source = new HTTPLiveSource(notify, httpService, url, headers);
226    } else if (!strncasecmp(url, "rtsp://", 7)) {
227        source = new RTSPSource(
228                notify, httpService, url, headers, mUIDValid, mUID);
229    } else if ((!strncasecmp(url, "http://", 7)
230                || !strncasecmp(url, "https://", 8))
231                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
232                    || strstr(url, ".sdp?"))) {
233        source = new RTSPSource(
234                notify, httpService, url, headers, mUIDValid, mUID, true);
235    } else {
236        sp<GenericSource> genericSource =
237                new GenericSource(notify, mUIDValid, mUID);
238        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
239        // The correct flags will be updated in Source::kWhatFlagsChanged
240        // handler when  GenericSource is prepared.
241
242        status_t err = genericSource->setDataSource(httpService, url, headers);
243
244        if (err == OK) {
245            source = genericSource;
246        } else {
247            ALOGE("Failed to set data source!");
248        }
249    }
250    msg->setObject("source", source);
251    msg->post();
252}
253
254void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
255    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
256
257    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
258
259    sp<GenericSource> source =
260            new GenericSource(notify, mUIDValid, mUID);
261
262    status_t err = source->setDataSource(fd, offset, length);
263
264    if (err != OK) {
265        ALOGE("Failed to set data source!");
266        source = NULL;
267    }
268
269    msg->setObject("source", source);
270    msg->post();
271}
272
273void NuPlayer::prepareAsync() {
274    (new AMessage(kWhatPrepare, id()))->post();
275}
276
277void NuPlayer::setVideoSurfaceTextureAsync(
278        const sp<IGraphicBufferProducer> &bufferProducer) {
279    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id());
280
281    if (bufferProducer == NULL) {
282        msg->setObject("native-window", NULL);
283    } else {
284        msg->setObject(
285                "native-window",
286                new NativeWindowWrapper(
287                    new Surface(bufferProducer, true /* controlledByApp */)));
288    }
289
290    msg->post();
291}
292
293void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
294    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id());
295    msg->setObject("sink", sink);
296    msg->post();
297}
298
299void NuPlayer::start() {
300    (new AMessage(kWhatStart, id()))->post();
301}
302
303void NuPlayer::pause() {
304    (new AMessage(kWhatPause, id()))->post();
305}
306
307void NuPlayer::resetAsync() {
308    if (mSource != NULL) {
309        // During a reset, the data source might be unresponsive already, we need to
310        // disconnect explicitly so that reads exit promptly.
311        // We can't queue the disconnect request to the looper, as it might be
312        // queued behind a stuck read and never gets processed.
313        // Doing a disconnect outside the looper to allows the pending reads to exit
314        // (either successfully or with error).
315        mSource->disconnect();
316    }
317
318    (new AMessage(kWhatReset, id()))->post();
319}
320
321void NuPlayer::seekToAsync(int64_t seekTimeUs, bool needNotify) {
322    sp<AMessage> msg = new AMessage(kWhatSeek, id());
323    msg->setInt64("seekTimeUs", seekTimeUs);
324    msg->setInt32("needNotify", needNotify);
325    msg->post();
326}
327
328
329void NuPlayer::writeTrackInfo(
330        Parcel* reply, const sp<AMessage> format) const {
331    int32_t trackType;
332    CHECK(format->findInt32("type", &trackType));
333
334    AString lang;
335    CHECK(format->findString("language", &lang));
336
337    reply->writeInt32(2); // write something non-zero
338    reply->writeInt32(trackType);
339    reply->writeString16(String16(lang.c_str()));
340
341    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
342        AString mime;
343        CHECK(format->findString("mime", &mime));
344
345        int32_t isAuto, isDefault, isForced;
346        CHECK(format->findInt32("auto", &isAuto));
347        CHECK(format->findInt32("default", &isDefault));
348        CHECK(format->findInt32("forced", &isForced));
349
350        reply->writeString16(String16(mime.c_str()));
351        reply->writeInt32(isAuto);
352        reply->writeInt32(isDefault);
353        reply->writeInt32(isForced);
354    }
355}
356
357void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
358    switch (msg->what()) {
359        case kWhatSetDataSource:
360        {
361            ALOGV("kWhatSetDataSource");
362
363            CHECK(mSource == NULL);
364
365            status_t err = OK;
366            sp<RefBase> obj;
367            CHECK(msg->findObject("source", &obj));
368            if (obj != NULL) {
369                mSource = static_cast<Source *>(obj.get());
370            } else {
371                err = UNKNOWN_ERROR;
372            }
373
374            CHECK(mDriver != NULL);
375            sp<NuPlayerDriver> driver = mDriver.promote();
376            if (driver != NULL) {
377                driver->notifySetDataSourceCompleted(err);
378            }
379            break;
380        }
381
382        case kWhatPrepare:
383        {
384            mSource->prepareAsync();
385            break;
386        }
387
388        case kWhatGetTrackInfo:
389        {
390            uint32_t replyID;
391            CHECK(msg->senderAwaitsResponse(&replyID));
392
393            Parcel* reply;
394            CHECK(msg->findPointer("reply", (void**)&reply));
395
396            size_t inbandTracks = 0;
397            if (mSource != NULL) {
398                inbandTracks = mSource->getTrackCount();
399            }
400
401            size_t ccTracks = 0;
402            if (mCCDecoder != NULL) {
403                ccTracks = mCCDecoder->getTrackCount();
404            }
405
406            // total track count
407            reply->writeInt32(inbandTracks + ccTracks);
408
409            // write inband tracks
410            for (size_t i = 0; i < inbandTracks; ++i) {
411                writeTrackInfo(reply, mSource->getTrackInfo(i));
412            }
413
414            // write CC track
415            for (size_t i = 0; i < ccTracks; ++i) {
416                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
417            }
418
419            sp<AMessage> response = new AMessage;
420            response->postReply(replyID);
421            break;
422        }
423
424        case kWhatGetSelectedTrack:
425        {
426            status_t err = INVALID_OPERATION;
427            if (mSource != NULL) {
428                err = OK;
429
430                int32_t type32;
431                CHECK(msg->findInt32("type", (int32_t*)&type32));
432                media_track_type type = (media_track_type)type32;
433                ssize_t selectedTrack = mSource->getSelectedTrack(type);
434
435                Parcel* reply;
436                CHECK(msg->findPointer("reply", (void**)&reply));
437                reply->writeInt32(selectedTrack);
438            }
439
440            sp<AMessage> response = new AMessage;
441            response->setInt32("err", err);
442
443            uint32_t replyID;
444            CHECK(msg->senderAwaitsResponse(&replyID));
445            response->postReply(replyID);
446            break;
447        }
448
449        case kWhatSelectTrack:
450        {
451            uint32_t replyID;
452            CHECK(msg->senderAwaitsResponse(&replyID));
453
454            size_t trackIndex;
455            int32_t select;
456            CHECK(msg->findSize("trackIndex", &trackIndex));
457            CHECK(msg->findInt32("select", &select));
458
459            status_t err = INVALID_OPERATION;
460
461            size_t inbandTracks = 0;
462            if (mSource != NULL) {
463                inbandTracks = mSource->getTrackCount();
464            }
465            size_t ccTracks = 0;
466            if (mCCDecoder != NULL) {
467                ccTracks = mCCDecoder->getTrackCount();
468            }
469
470            if (trackIndex < inbandTracks) {
471                err = mSource->selectTrack(trackIndex, select);
472
473                if (!select && err == OK) {
474                    int32_t type;
475                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
476                    if (info != NULL
477                            && info->findInt32("type", &type)
478                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
479                        ++mTimedTextGeneration;
480                    }
481                }
482            } else {
483                trackIndex -= inbandTracks;
484
485                if (trackIndex < ccTracks) {
486                    err = mCCDecoder->selectTrack(trackIndex, select);
487                }
488            }
489
490            sp<AMessage> response = new AMessage;
491            response->setInt32("err", err);
492
493            response->postReply(replyID);
494            break;
495        }
496
497        case kWhatPollDuration:
498        {
499            int32_t generation;
500            CHECK(msg->findInt32("generation", &generation));
501
502            if (generation != mPollDurationGeneration) {
503                // stale
504                break;
505            }
506
507            int64_t durationUs;
508            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
509                sp<NuPlayerDriver> driver = mDriver.promote();
510                if (driver != NULL) {
511                    driver->notifyDuration(durationUs);
512                }
513            }
514
515            msg->post(1000000ll);  // poll again in a second.
516            break;
517        }
518
519        case kWhatSetVideoNativeWindow:
520        {
521            ALOGV("kWhatSetVideoNativeWindow");
522
523            sp<RefBase> obj;
524            CHECK(msg->findObject("native-window", &obj));
525
526            if (mSource->getFormat(false /* audio */) == NULL) {
527                performSetSurface(static_cast<NativeWindowWrapper *>(obj.get()));
528                break;
529            }
530
531            mDeferredActions.push_back(
532                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
533                                           FLUSH_CMD_SHUTDOWN /* video */));
534
535            mDeferredActions.push_back(
536                    new SetSurfaceAction(
537                        static_cast<NativeWindowWrapper *>(obj.get())));
538
539            if (obj != NULL) {
540                if (mStarted) {
541                    // Issue a seek to refresh the video screen only if started otherwise
542                    // the extractor may not yet be started and will assert.
543                    // If the video decoder is not set (perhaps audio only in this case)
544                    // do not perform a seek as it is not needed.
545                    int64_t currentPositionUs = 0;
546                    if (getCurrentPosition(&currentPositionUs) == OK) {
547                        mDeferredActions.push_back(
548                                new SeekAction(currentPositionUs, false /* needNotify */));
549                    }
550                }
551
552                // If there is a new surface texture, instantiate decoders
553                // again if possible.
554                mDeferredActions.push_back(
555                        new SimpleAction(&NuPlayer::performScanSources));
556            }
557
558            processDeferredActions();
559            break;
560        }
561
562        case kWhatSetAudioSink:
563        {
564            ALOGV("kWhatSetAudioSink");
565
566            sp<RefBase> obj;
567            CHECK(msg->findObject("sink", &obj));
568
569            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
570            break;
571        }
572
573        case kWhatStart:
574        {
575            ALOGV("kWhatStart");
576            if (mStarted) {
577                onResume();
578            } else {
579                onStart();
580            }
581            break;
582        }
583
584        case kWhatScanSources:
585        {
586            int32_t generation;
587            CHECK(msg->findInt32("generation", &generation));
588            if (generation != mScanSourcesGeneration) {
589                // Drop obsolete msg.
590                break;
591            }
592
593            mScanSourcesPending = false;
594
595            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
596                 mAudioDecoder != NULL, mVideoDecoder != NULL);
597
598            bool mHadAnySourcesBefore =
599                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
600
601            // initialize video before audio because successful initialization of
602            // video may change deep buffer mode of audio.
603            if (mNativeWindow != NULL) {
604                instantiateDecoder(false, &mVideoDecoder);
605            }
606
607            if (mAudioSink != NULL) {
608                if (mOffloadAudio) {
609                    // open audio sink early under offload mode.
610                    sp<AMessage> format = mSource->getFormat(true /*audio*/);
611                    openAudioSink(format, true /*offloadOnly*/);
612                }
613                instantiateDecoder(true, &mAudioDecoder);
614            }
615
616            if (!mHadAnySourcesBefore
617                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
618                // This is the first time we've found anything playable.
619
620                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
621                    schedulePollDuration();
622                }
623            }
624
625            status_t err;
626            if ((err = mSource->feedMoreTSData()) != OK) {
627                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
628                    // We're not currently decoding anything (no audio or
629                    // video tracks found) and we just ran out of input data.
630
631                    if (err == ERROR_END_OF_STREAM) {
632                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
633                    } else {
634                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
635                    }
636                }
637                break;
638            }
639
640            if ((mAudioDecoder == NULL && mAudioSink != NULL)
641                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
642                msg->post(100000ll);
643                mScanSourcesPending = true;
644            }
645            break;
646        }
647
648        case kWhatVideoNotify:
649        case kWhatAudioNotify:
650        {
651            bool audio = msg->what() == kWhatAudioNotify;
652
653            int32_t currentDecoderGeneration =
654                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
655            int32_t requesterGeneration = currentDecoderGeneration - 1;
656            CHECK(msg->findInt32("generation", &requesterGeneration));
657
658            if (requesterGeneration != currentDecoderGeneration) {
659                ALOGV("got message from old %s decoder, generation(%d:%d)",
660                        audio ? "audio" : "video", requesterGeneration,
661                        currentDecoderGeneration);
662                sp<AMessage> reply;
663                if (!(msg->findMessage("reply", &reply))) {
664                    return;
665                }
666
667                reply->setInt32("err", INFO_DISCONTINUITY);
668                reply->post();
669                return;
670            }
671
672            int32_t what;
673            CHECK(msg->findInt32("what", &what));
674
675            if (what == Decoder::kWhatFillThisBuffer) {
676                status_t err = feedDecoderInputData(
677                        audio, msg);
678
679                if (err == -EWOULDBLOCK) {
680                    if (mSource->feedMoreTSData() == OK) {
681                        msg->post(10 * 1000ll);
682                    }
683                }
684            } else if (what == Decoder::kWhatEOS) {
685                int32_t err;
686                CHECK(msg->findInt32("err", &err));
687
688                if (err == ERROR_END_OF_STREAM) {
689                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
690                } else {
691                    ALOGV("got %s decoder EOS w/ error %d",
692                         audio ? "audio" : "video",
693                         err);
694                }
695
696                mRenderer->queueEOS(audio, err);
697            } else if (what == Decoder::kWhatFlushCompleted) {
698                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
699
700                handleFlushComplete(audio, true /* isDecoder */);
701                finishFlushIfPossible();
702            } else if (what == Decoder::kWhatVideoSizeChanged) {
703                sp<AMessage> format;
704                CHECK(msg->findMessage("format", &format));
705
706                sp<AMessage> inputFormat =
707                        mSource->getFormat(false /* audio */);
708
709                updateVideoSize(inputFormat, format);
710            } else if (what == Decoder::kWhatShutdownCompleted) {
711                ALOGV("%s shutdown completed", audio ? "audio" : "video");
712                if (audio) {
713                    mAudioDecoder.clear();
714                    ++mAudioDecoderGeneration;
715
716                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
717                    mFlushingAudio = SHUT_DOWN;
718                } else {
719                    mVideoDecoder.clear();
720                    ++mVideoDecoderGeneration;
721
722                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
723                    mFlushingVideo = SHUT_DOWN;
724                }
725
726                finishFlushIfPossible();
727            } else if (what == Decoder::kWhatError) {
728                status_t err;
729                if (!msg->findInt32("err", &err) || err == OK) {
730                    err = UNKNOWN_ERROR;
731                }
732
733                // Decoder errors can be due to Source (e.g. from streaming),
734                // or from decoding corrupted bitstreams, or from other decoder
735                // MediaCodec operations (e.g. from an ongoing reset or seek).
736                //
737                // We try to gracefully shut down the affected decoder if possible,
738                // rather than trying to force the shutdown with something
739                // similar to performReset(). This method can lead to a hang
740                // if MediaCodec functions block after an error, but they should
741                // typically return INVALID_OPERATION instead of blocking.
742
743                FlushStatus *flushing = audio ? &mFlushingAudio : &mFlushingVideo;
744                ALOGE("received error(%#x) from %s decoder, flushing(%d), now shutting down",
745                        err, audio ? "audio" : "video", *flushing);
746
747                switch (*flushing) {
748                    case NONE:
749                        mDeferredActions.push_back(
750                                new FlushDecoderAction(
751                                    audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
752                                    audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN));
753                        processDeferredActions();
754                        break;
755                    case FLUSHING_DECODER:
756                        *flushing = FLUSHING_DECODER_SHUTDOWN; // initiate shutdown after flush.
757                        break; // Wait for flush to complete.
758                    case FLUSHING_DECODER_SHUTDOWN:
759                        break; // Wait for flush to complete.
760                    case SHUTTING_DOWN_DECODER:
761                        break; // Wait for shutdown to complete.
762                    case FLUSHED:
763                        // Widevine source reads must stop before releasing the video decoder.
764                        if (!audio && mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
765                            mSource->stop();
766                        }
767                        getDecoder(audio)->initiateShutdown(); // In the middle of a seek.
768                        *flushing = SHUTTING_DOWN_DECODER;     // Shut down.
769                        break;
770                    case SHUT_DOWN:
771                        finishFlushIfPossible();  // Should not occur.
772                        break;                    // Finish anyways.
773                }
774                notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
775            } else if (what == Decoder::kWhatRenderBufferTime) {
776                renderBuffer(audio, msg);
777            } else {
778                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
779                      what,
780                      what >> 24,
781                      (what >> 16) & 0xff,
782                      (what >> 8) & 0xff,
783                      what & 0xff);
784            }
785
786            break;
787        }
788
789        case kWhatRendererNotify:
790        {
791            int32_t requesterGeneration = mRendererGeneration - 1;
792            CHECK(msg->findInt32("generation", &requesterGeneration));
793            if (requesterGeneration != mRendererGeneration) {
794                ALOGV("got message from old renderer, generation(%d:%d)",
795                        requesterGeneration, mRendererGeneration);
796                return;
797            }
798
799            int32_t what;
800            CHECK(msg->findInt32("what", &what));
801
802            if (what == Renderer::kWhatEOS) {
803                int32_t audio;
804                CHECK(msg->findInt32("audio", &audio));
805
806                int32_t finalResult;
807                CHECK(msg->findInt32("finalResult", &finalResult));
808
809                if (audio) {
810                    mAudioEOS = true;
811                } else {
812                    mVideoEOS = true;
813                }
814
815                if (finalResult == ERROR_END_OF_STREAM) {
816                    ALOGV("reached %s EOS", audio ? "audio" : "video");
817                } else {
818                    ALOGE("%s track encountered an error (%d)",
819                         audio ? "audio" : "video", finalResult);
820
821                    notifyListener(
822                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
823                }
824
825                if ((mAudioEOS || mAudioDecoder == NULL)
826                        && (mVideoEOS || mVideoDecoder == NULL)) {
827                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
828                }
829            } else if (what == Renderer::kWhatFlushComplete) {
830                int32_t audio;
831                CHECK(msg->findInt32("audio", &audio));
832
833                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
834                handleFlushComplete(audio, false /* isDecoder */);
835                finishFlushIfPossible();
836            } else if (what == Renderer::kWhatVideoRenderingStart) {
837                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
838            } else if (what == Renderer::kWhatMediaRenderingStart) {
839                ALOGV("media rendering started");
840                notifyListener(MEDIA_STARTED, 0, 0);
841            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
842                ALOGV("Tear down audio offload, fall back to s/w path");
843                int64_t positionUs;
844                CHECK(msg->findInt64("positionUs", &positionUs));
845                int32_t reason;
846                CHECK(msg->findInt32("reason", &reason));
847                closeAudioSink();
848                mAudioDecoder.clear();
849                ++mAudioDecoderGeneration;
850                mRenderer->flush(true /* audio */);
851                if (mVideoDecoder != NULL) {
852                    mRenderer->flush(false /* audio */);
853                }
854                mRenderer->signalDisableOffloadAudio();
855                mOffloadAudio = false;
856
857                performSeek(positionUs, false /* needNotify */);
858                if (reason == Renderer::kDueToError) {
859                    instantiateDecoder(true /* audio */, &mAudioDecoder);
860                }
861            }
862            break;
863        }
864
865        case kWhatMoreDataQueued:
866        {
867            break;
868        }
869
870        case kWhatReset:
871        {
872            ALOGV("kWhatReset");
873
874            mDeferredActions.push_back(
875                    new FlushDecoderAction(
876                        FLUSH_CMD_SHUTDOWN /* audio */,
877                        FLUSH_CMD_SHUTDOWN /* video */));
878
879            mDeferredActions.push_back(
880                    new SimpleAction(&NuPlayer::performReset));
881
882            processDeferredActions();
883            break;
884        }
885
886        case kWhatSeek:
887        {
888            int64_t seekTimeUs;
889            int32_t needNotify;
890            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
891            CHECK(msg->findInt32("needNotify", &needNotify));
892
893            ALOGV("kWhatSeek seekTimeUs=%lld us, needNotify=%d",
894                    seekTimeUs, needNotify);
895
896            mDeferredActions.push_back(
897                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
898                                           FLUSH_CMD_FLUSH /* video */));
899
900            mDeferredActions.push_back(
901                    new SeekAction(seekTimeUs, needNotify));
902
903            processDeferredActions();
904            break;
905        }
906
907        case kWhatPause:
908        {
909            if (mSource != NULL) {
910                mSource->pause();
911            } else {
912                ALOGW("pause called when source is gone or not set");
913            }
914            if (mRenderer != NULL) {
915                mRenderer->pause();
916            } else {
917                ALOGW("pause called when renderer is gone or not set");
918            }
919            break;
920        }
921
922        case kWhatSourceNotify:
923        {
924            onSourceNotify(msg);
925            break;
926        }
927
928        case kWhatClosedCaptionNotify:
929        {
930            onClosedCaptionNotify(msg);
931            break;
932        }
933
934        default:
935            TRESPASS();
936            break;
937    }
938}
939
940void NuPlayer::onResume() {
941    if (mSource != NULL) {
942        mSource->resume();
943    } else {
944        ALOGW("resume called when source is gone or not set");
945    }
946    // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if
947    // needed.
948    if (audioDecoderStillNeeded() && mAudioDecoder == NULL) {
949        instantiateDecoder(true /* audio */, &mAudioDecoder);
950    }
951    if (mRenderer != NULL) {
952        mRenderer->resume();
953    } else {
954        ALOGW("resume called when renderer is gone or not set");
955    }
956}
957
958void NuPlayer::onStart() {
959    mVideoIsAVC = false;
960    mOffloadAudio = false;
961    mAudioEOS = false;
962    mVideoEOS = false;
963    mNumFramesTotal = 0;
964    mNumFramesDropped = 0;
965    mStarted = true;
966
967    /* instantiate decoders now for secure playback */
968    if (mSourceFlags & Source::FLAG_SECURE) {
969        if (mNativeWindow != NULL) {
970            instantiateDecoder(false, &mVideoDecoder);
971        }
972
973        if (mAudioSink != NULL) {
974            instantiateDecoder(true, &mAudioDecoder);
975        }
976    }
977
978    mSource->start();
979
980    uint32_t flags = 0;
981
982    if (mSource->isRealTime()) {
983        flags |= Renderer::FLAG_REAL_TIME;
984    }
985
986    sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
987    audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
988    if (mAudioSink != NULL) {
989        streamType = mAudioSink->getAudioStreamType();
990    }
991
992    sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
993
994    mOffloadAudio =
995        canOffloadStream(audioMeta, (videoFormat != NULL),
996                         true /* is_streaming */, streamType);
997    if (mOffloadAudio) {
998        flags |= Renderer::FLAG_OFFLOAD_AUDIO;
999    }
1000
1001    sp<AMessage> notify = new AMessage(kWhatRendererNotify, id());
1002    ++mRendererGeneration;
1003    notify->setInt32("generation", mRendererGeneration);
1004    mRenderer = new Renderer(mAudioSink, notify, flags);
1005
1006    mRendererLooper = new ALooper;
1007    mRendererLooper->setName("NuPlayerRenderer");
1008    mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
1009    mRendererLooper->registerHandler(mRenderer);
1010
1011    sp<MetaData> meta = getFileMeta();
1012    int32_t rate;
1013    if (meta != NULL
1014            && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) {
1015        mRenderer->setVideoFrameRate(rate);
1016    }
1017
1018    if (mVideoDecoder != NULL) {
1019        mVideoDecoder->setRenderer(mRenderer);
1020    }
1021    if (mAudioDecoder != NULL) {
1022        mAudioDecoder->setRenderer(mRenderer);
1023    }
1024
1025    postScanSources();
1026}
1027
1028bool NuPlayer::audioDecoderStillNeeded() {
1029    // Audio decoder is no longer needed if it's in shut/shutting down status.
1030    return ((mFlushingAudio != SHUT_DOWN) && (mFlushingAudio != SHUTTING_DOWN_DECODER));
1031}
1032
1033void NuPlayer::handleFlushComplete(bool audio, bool isDecoder) {
1034    // We wait for both the decoder flush and the renderer flush to complete
1035    // before entering either the FLUSHED or the SHUTTING_DOWN_DECODER state.
1036
1037    mFlushComplete[audio][isDecoder] = true;
1038    if (!mFlushComplete[audio][!isDecoder]) {
1039        return;
1040    }
1041
1042    FlushStatus *state = audio ? &mFlushingAudio : &mFlushingVideo;
1043    switch (*state) {
1044        case FLUSHING_DECODER:
1045        {
1046            *state = FLUSHED;
1047            break;
1048        }
1049
1050        case FLUSHING_DECODER_SHUTDOWN:
1051        {
1052            *state = SHUTTING_DOWN_DECODER;
1053
1054            ALOGV("initiating %s decoder shutdown", audio ? "audio" : "video");
1055            if (!audio) {
1056                // Widevine source reads must stop before releasing the video decoder.
1057                if (mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
1058                    mSource->stop();
1059                }
1060            }
1061            getDecoder(audio)->initiateShutdown();
1062            break;
1063        }
1064
1065        default:
1066            // decoder flush completes only occur in a flushing state.
1067            LOG_ALWAYS_FATAL_IF(isDecoder, "decoder flush in invalid state %d", *state);
1068            break;
1069    }
1070}
1071
1072void NuPlayer::finishFlushIfPossible() {
1073    if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED
1074            && mFlushingAudio != SHUT_DOWN) {
1075        return;
1076    }
1077
1078    if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED
1079            && mFlushingVideo != SHUT_DOWN) {
1080        return;
1081    }
1082
1083    ALOGV("both audio and video are flushed now.");
1084
1085    mPendingAudioAccessUnit.clear();
1086    mAggregateBuffer.clear();
1087
1088    if (mTimeDiscontinuityPending) {
1089        mRenderer->signalTimeDiscontinuity();
1090        mTimeDiscontinuityPending = false;
1091    }
1092
1093    if (mAudioDecoder != NULL && mFlushingAudio == FLUSHED) {
1094        mAudioDecoder->signalResume();
1095    }
1096
1097    if (mVideoDecoder != NULL && mFlushingVideo == FLUSHED) {
1098        mVideoDecoder->signalResume();
1099    }
1100
1101    mFlushingAudio = NONE;
1102    mFlushingVideo = NONE;
1103
1104    clearFlushComplete();
1105
1106    processDeferredActions();
1107}
1108
1109void NuPlayer::postScanSources() {
1110    if (mScanSourcesPending) {
1111        return;
1112    }
1113
1114    sp<AMessage> msg = new AMessage(kWhatScanSources, id());
1115    msg->setInt32("generation", mScanSourcesGeneration);
1116    msg->post();
1117
1118    mScanSourcesPending = true;
1119}
1120
1121void NuPlayer::openAudioSink(const sp<AMessage> &format, bool offloadOnly) {
1122    uint32_t flags;
1123    int64_t durationUs;
1124    bool hasVideo = (mVideoDecoder != NULL);
1125    // FIXME: we should handle the case where the video decoder
1126    // is created after we receive the format change indication.
1127    // Current code will just make that we select deep buffer
1128    // with video which should not be a problem as it should
1129    // not prevent from keeping A/V sync.
1130    if (!hasVideo &&
1131            mSource->getDuration(&durationUs) == OK &&
1132            durationUs
1133                > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
1134        flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1135    } else {
1136        flags = AUDIO_OUTPUT_FLAG_NONE;
1137    }
1138
1139    mOffloadAudio = mRenderer->openAudioSink(
1140            format, offloadOnly, hasVideo, flags);
1141
1142    if (mOffloadAudio) {
1143        sp<MetaData> audioMeta =
1144                mSource->getFormatMeta(true /* audio */);
1145        sendMetaDataToHal(mAudioSink, audioMeta);
1146    }
1147}
1148
1149void NuPlayer::closeAudioSink() {
1150    mRenderer->closeAudioSink();
1151}
1152
1153status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
1154    if (*decoder != NULL) {
1155        return OK;
1156    }
1157
1158    sp<AMessage> format = mSource->getFormat(audio);
1159
1160    if (format == NULL) {
1161        return -EWOULDBLOCK;
1162    }
1163
1164    if (!audio) {
1165        AString mime;
1166        CHECK(format->findString("mime", &mime));
1167        mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
1168
1169        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
1170        mCCDecoder = new CCDecoder(ccNotify);
1171
1172        if (mSourceFlags & Source::FLAG_SECURE) {
1173            format->setInt32("secure", true);
1174        }
1175    }
1176
1177    if (audio) {
1178        sp<AMessage> notify = new AMessage(kWhatAudioNotify, id());
1179        ++mAudioDecoderGeneration;
1180        notify->setInt32("generation", mAudioDecoderGeneration);
1181
1182        if (mOffloadAudio) {
1183            *decoder = new DecoderPassThrough(notify, mSource, mRenderer);
1184        } else {
1185            *decoder = new Decoder(notify, mSource, mRenderer);
1186        }
1187    } else {
1188        sp<AMessage> notify = new AMessage(kWhatVideoNotify, id());
1189        ++mVideoDecoderGeneration;
1190        notify->setInt32("generation", mVideoDecoderGeneration);
1191
1192        *decoder = new Decoder(notify, mSource, mRenderer, mNativeWindow);
1193    }
1194    (*decoder)->init();
1195    (*decoder)->configure(format);
1196
1197    // allocate buffers to decrypt widevine source buffers
1198    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1199        Vector<sp<ABuffer> > inputBufs;
1200        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1201
1202        Vector<MediaBuffer *> mediaBufs;
1203        for (size_t i = 0; i < inputBufs.size(); i++) {
1204            const sp<ABuffer> &buffer = inputBufs[i];
1205            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1206            mediaBufs.push(mbuf);
1207        }
1208
1209        status_t err = mSource->setBuffers(audio, mediaBufs);
1210        if (err != OK) {
1211            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1212                mediaBufs[i]->release();
1213            }
1214            mediaBufs.clear();
1215            ALOGE("Secure source didn't support secure mediaBufs.");
1216            return err;
1217        }
1218    }
1219    return OK;
1220}
1221
1222status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
1223    sp<AMessage> reply;
1224    CHECK(msg->findMessage("reply", &reply));
1225
1226    if ((audio && mFlushingAudio != NONE)
1227            || (!audio && mFlushingVideo != NONE)
1228            || mSource == NULL) {
1229        reply->setInt32("err", INFO_DISCONTINUITY);
1230        reply->post();
1231        return OK;
1232    }
1233
1234    sp<ABuffer> accessUnit;
1235
1236    // Aggregate smaller buffers into a larger buffer.
1237    // The goal is to reduce power consumption.
1238    // Note this will not work if the decoder requires one frame per buffer.
1239    bool doBufferAggregation = (audio && mOffloadAudio);
1240    bool needMoreData = false;
1241
1242    bool dropAccessUnit;
1243    do {
1244        status_t err;
1245        // Did we save an accessUnit earlier because of a discontinuity?
1246        if (audio && (mPendingAudioAccessUnit != NULL)) {
1247            accessUnit = mPendingAudioAccessUnit;
1248            mPendingAudioAccessUnit.clear();
1249            err = mPendingAudioErr;
1250            ALOGV("feedDecoderInputData() use mPendingAudioAccessUnit");
1251        } else {
1252            err = mSource->dequeueAccessUnit(audio, &accessUnit);
1253        }
1254
1255        if (err == -EWOULDBLOCK) {
1256            return err;
1257        } else if (err != OK) {
1258            if (err == INFO_DISCONTINUITY) {
1259                if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1260                    // We already have some data so save this for later.
1261                    mPendingAudioErr = err;
1262                    mPendingAudioAccessUnit = accessUnit;
1263                    accessUnit.clear();
1264                    ALOGD("feedDecoderInputData() save discontinuity for later");
1265                    break;
1266                }
1267                int32_t type;
1268                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
1269
1270                bool formatChange =
1271                    (audio &&
1272                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
1273                    || (!audio &&
1274                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
1275
1276                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
1277
1278                ALOGI("%s discontinuity (formatChange=%d, time=%d)",
1279                     audio ? "audio" : "video", formatChange, timeChange);
1280
1281                mTimeDiscontinuityPending =
1282                    mTimeDiscontinuityPending || timeChange;
1283
1284                bool seamlessFormatChange = false;
1285                sp<AMessage> newFormat = mSource->getFormat(audio);
1286                if (formatChange) {
1287                    seamlessFormatChange =
1288                        getDecoder(audio)->supportsSeamlessFormatChange(newFormat);
1289                    // treat seamless format change separately
1290                    formatChange = !seamlessFormatChange;
1291                }
1292                bool shutdownOrFlush = formatChange || timeChange;
1293
1294                // We want to queue up scan-sources only once per discontinuity.
1295                // We control this by doing it only if neither audio nor video are
1296                // flushing or shutting down.  (After handling 1st discontinuity, one
1297                // of the flushing states will not be NONE.)
1298                // No need to scan sources if this discontinuity does not result
1299                // in a flush or shutdown, as the flushing state will stay NONE.
1300                if (mFlushingAudio == NONE && mFlushingVideo == NONE &&
1301                        shutdownOrFlush) {
1302                    // And we'll resume scanning sources once we're done
1303                    // flushing.
1304                    mDeferredActions.push_front(
1305                            new SimpleAction(
1306                                &NuPlayer::performScanSources));
1307                }
1308
1309                if (formatChange /* not seamless */) {
1310                    // must change decoder
1311                    flushDecoder(audio, /* needShutdown = */ true);
1312                } else if (timeChange) {
1313                    // need to flush
1314                    flushDecoder(audio, /* needShutdown = */ false, newFormat);
1315                    err = OK;
1316                } else if (seamlessFormatChange) {
1317                    // reuse existing decoder and don't flush
1318                    updateDecoderFormatWithoutFlush(audio, newFormat);
1319                    err = OK;
1320                } else {
1321                    // This stream is unaffected by the discontinuity
1322                    return -EWOULDBLOCK;
1323                }
1324            } else if (err == ERROR_END_OF_STREAM
1325                    && doBufferAggregation && (mAggregateBuffer != NULL)) {
1326                // send out the last bit of aggregated data
1327                reply->setBuffer("buffer", mAggregateBuffer);
1328                mAggregateBuffer.clear();
1329                err = OK;
1330            }
1331
1332            reply->setInt32("err", err);
1333            reply->post();
1334            return OK;
1335        }
1336
1337        if (!audio) {
1338            ++mNumFramesTotal;
1339        }
1340
1341        dropAccessUnit = false;
1342        if (!audio
1343                && !(mSourceFlags & Source::FLAG_SECURE)
1344                && mRenderer->getVideoLateByUs() > 100000ll
1345                && mVideoIsAVC
1346                && !IsAVCReferenceFrame(accessUnit)) {
1347            dropAccessUnit = true;
1348            ++mNumFramesDropped;
1349        }
1350
1351        size_t smallSize = accessUnit->size();
1352        needMoreData = false;
1353        if (doBufferAggregation && (mAggregateBuffer == NULL)
1354                // Don't bother if only room for a few small buffers.
1355                && (smallSize < (kAggregateBufferSizeBytes / 3))) {
1356            // Create a larger buffer for combining smaller buffers from the extractor.
1357            mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes);
1358            mAggregateBuffer->setRange(0, 0); // start empty
1359        }
1360
1361        if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1362            int64_t timeUs;
1363            int64_t dummy;
1364            bool smallTimestampValid = accessUnit->meta()->findInt64("timeUs", &timeUs);
1365            bool bigTimestampValid = mAggregateBuffer->meta()->findInt64("timeUs", &dummy);
1366            // Will the smaller buffer fit?
1367            size_t bigSize = mAggregateBuffer->size();
1368            size_t roomLeft = mAggregateBuffer->capacity() - bigSize;
1369            // Should we save this small buffer for the next big buffer?
1370            // If the first small buffer did not have a timestamp then save
1371            // any buffer that does have a timestamp until the next big buffer.
1372            if ((smallSize > roomLeft)
1373                || (!bigTimestampValid && (bigSize > 0) && smallTimestampValid)) {
1374                mPendingAudioErr = err;
1375                mPendingAudioAccessUnit = accessUnit;
1376                accessUnit.clear();
1377            } else {
1378                // Grab time from first small buffer if available.
1379                if ((bigSize == 0) && smallTimestampValid) {
1380                    mAggregateBuffer->meta()->setInt64("timeUs", timeUs);
1381                }
1382                // Append small buffer to the bigger buffer.
1383                memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize);
1384                bigSize += smallSize;
1385                mAggregateBuffer->setRange(0, bigSize);
1386
1387                // Keep looping until we run out of room in the mAggregateBuffer.
1388                needMoreData = true;
1389
1390                ALOGV("feedDecoderInputData() smallSize = %zu, bigSize = %zu, capacity = %zu",
1391                        smallSize, bigSize, mAggregateBuffer->capacity());
1392            }
1393        }
1394    } while (dropAccessUnit || needMoreData);
1395
1396    // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
1397
1398#if 0
1399    int64_t mediaTimeUs;
1400    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
1401    ALOGV("feeding %s input buffer at media time %.2f secs",
1402         audio ? "audio" : "video",
1403         mediaTimeUs / 1E6);
1404#endif
1405
1406    if (!audio) {
1407        mCCDecoder->decode(accessUnit);
1408    }
1409
1410    if (doBufferAggregation && (mAggregateBuffer != NULL)) {
1411        ALOGV("feedDecoderInputData() reply with aggregated buffer, %zu",
1412                mAggregateBuffer->size());
1413        reply->setBuffer("buffer", mAggregateBuffer);
1414        mAggregateBuffer.clear();
1415    } else {
1416        reply->setBuffer("buffer", accessUnit);
1417    }
1418
1419    reply->post();
1420
1421    return OK;
1422}
1423
1424void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
1425    // ALOGV("renderBuffer %s", audio ? "audio" : "video");
1426
1427    if ((audio && mFlushingAudio != NONE)
1428            || (!audio && mFlushingVideo != NONE)) {
1429        // We're currently attempting to flush the decoder, in order
1430        // to complete this, the decoder wants all its buffers back,
1431        // so we don't want any output buffers it sent us (from before
1432        // we initiated the flush) to be stuck in the renderer's queue.
1433
1434        ALOGV("we're still flushing the %s decoder, sending its output buffer"
1435             " right back.", audio ? "audio" : "video");
1436
1437        return;
1438    }
1439
1440    int64_t mediaTimeUs;
1441    CHECK(msg->findInt64("timeUs", &mediaTimeUs));
1442
1443    if (!audio && mCCDecoder->isSelected()) {
1444        mCCDecoder->display(mediaTimeUs);
1445    }
1446}
1447
1448void NuPlayer::updateVideoSize(
1449        const sp<AMessage> &inputFormat,
1450        const sp<AMessage> &outputFormat) {
1451    if (inputFormat == NULL) {
1452        ALOGW("Unknown video size, reporting 0x0!");
1453        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1454        return;
1455    }
1456
1457    int32_t displayWidth, displayHeight;
1458    int32_t cropLeft, cropTop, cropRight, cropBottom;
1459
1460    if (outputFormat != NULL) {
1461        int32_t width, height;
1462        CHECK(outputFormat->findInt32("width", &width));
1463        CHECK(outputFormat->findInt32("height", &height));
1464
1465        int32_t cropLeft, cropTop, cropRight, cropBottom;
1466        CHECK(outputFormat->findRect(
1467                    "crop",
1468                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1469
1470        displayWidth = cropRight - cropLeft + 1;
1471        displayHeight = cropBottom - cropTop + 1;
1472
1473        ALOGV("Video output format changed to %d x %d "
1474             "(crop: %d x %d @ (%d, %d))",
1475             width, height,
1476             displayWidth,
1477             displayHeight,
1478             cropLeft, cropTop);
1479    } else {
1480        CHECK(inputFormat->findInt32("width", &displayWidth));
1481        CHECK(inputFormat->findInt32("height", &displayHeight));
1482
1483        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1484    }
1485
1486    // Take into account sample aspect ratio if necessary:
1487    int32_t sarWidth, sarHeight;
1488    if (inputFormat->findInt32("sar-width", &sarWidth)
1489            && inputFormat->findInt32("sar-height", &sarHeight)) {
1490        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1491
1492        displayWidth = (displayWidth * sarWidth) / sarHeight;
1493
1494        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1495    }
1496
1497    int32_t rotationDegrees;
1498    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1499        rotationDegrees = 0;
1500    }
1501
1502    if (rotationDegrees == 90 || rotationDegrees == 270) {
1503        int32_t tmp = displayWidth;
1504        displayWidth = displayHeight;
1505        displayHeight = tmp;
1506    }
1507
1508    notifyListener(
1509            MEDIA_SET_VIDEO_SIZE,
1510            displayWidth,
1511            displayHeight);
1512}
1513
1514void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1515    if (mDriver == NULL) {
1516        return;
1517    }
1518
1519    sp<NuPlayerDriver> driver = mDriver.promote();
1520
1521    if (driver == NULL) {
1522        return;
1523    }
1524
1525    driver->notifyListener(msg, ext1, ext2, in);
1526}
1527
1528void NuPlayer::flushDecoder(
1529        bool audio, bool needShutdown, const sp<AMessage> &newFormat) {
1530    ALOGV("[%s] flushDecoder needShutdown=%d",
1531          audio ? "audio" : "video", needShutdown);
1532
1533    const sp<Decoder> &decoder = getDecoder(audio);
1534    if (decoder == NULL) {
1535        ALOGI("flushDecoder %s without decoder present",
1536             audio ? "audio" : "video");
1537        return;
1538    }
1539
1540    // Make sure we don't continue to scan sources until we finish flushing.
1541    ++mScanSourcesGeneration;
1542    mScanSourcesPending = false;
1543
1544    decoder->signalFlush(newFormat);
1545
1546    FlushStatus newStatus =
1547        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1548
1549    mFlushComplete[audio][false /* isDecoder */] = false;
1550    mFlushComplete[audio][true /* isDecoder */] = false;
1551    if (audio) {
1552        ALOGE_IF(mFlushingAudio != NONE,
1553                "audio flushDecoder() is called in state %d", mFlushingAudio);
1554        mFlushingAudio = newStatus;
1555    } else {
1556        ALOGE_IF(mFlushingVideo != NONE,
1557                "video flushDecoder() is called in state %d", mFlushingVideo);
1558        mFlushingVideo = newStatus;
1559
1560        if (mCCDecoder != NULL) {
1561            mCCDecoder->flush();
1562        }
1563    }
1564}
1565
1566void NuPlayer::updateDecoderFormatWithoutFlush(
1567        bool audio, const sp<AMessage> &format) {
1568    ALOGV("[%s] updateDecoderFormatWithoutFlush", audio ? "audio" : "video");
1569
1570    const sp<Decoder> &decoder = getDecoder(audio);
1571    if (decoder == NULL) {
1572        ALOGI("updateDecoderFormatWithoutFlush %s without decoder present",
1573             audio ? "audio" : "video");
1574        return;
1575    }
1576
1577    decoder->signalUpdateFormat(format);
1578}
1579
1580void NuPlayer::queueDecoderShutdown(
1581        bool audio, bool video, const sp<AMessage> &reply) {
1582    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1583
1584    mDeferredActions.push_back(
1585            new FlushDecoderAction(
1586                audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
1587                video ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE));
1588
1589    mDeferredActions.push_back(
1590            new SimpleAction(&NuPlayer::performScanSources));
1591
1592    mDeferredActions.push_back(new PostMessageAction(reply));
1593
1594    processDeferredActions();
1595}
1596
1597status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1598    mVideoScalingMode = mode;
1599    if (mNativeWindow != NULL) {
1600        status_t ret = native_window_set_scaling_mode(
1601                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1602        if (ret != OK) {
1603            ALOGE("Failed to set scaling mode (%d): %s",
1604                -ret, strerror(-ret));
1605            return ret;
1606        }
1607    }
1608    return OK;
1609}
1610
1611status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1612    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id());
1613    msg->setPointer("reply", reply);
1614
1615    sp<AMessage> response;
1616    status_t err = msg->postAndAwaitResponse(&response);
1617    return err;
1618}
1619
1620status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1621    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id());
1622    msg->setPointer("reply", reply);
1623    msg->setInt32("type", type);
1624
1625    sp<AMessage> response;
1626    status_t err = msg->postAndAwaitResponse(&response);
1627    if (err == OK && response != NULL) {
1628        CHECK(response->findInt32("err", &err));
1629    }
1630    return err;
1631}
1632
1633status_t NuPlayer::selectTrack(size_t trackIndex, bool select) {
1634    sp<AMessage> msg = new AMessage(kWhatSelectTrack, id());
1635    msg->setSize("trackIndex", trackIndex);
1636    msg->setInt32("select", select);
1637
1638    sp<AMessage> response;
1639    status_t err = msg->postAndAwaitResponse(&response);
1640
1641    if (err != OK) {
1642        return err;
1643    }
1644
1645    if (!response->findInt32("err", &err)) {
1646        err = OK;
1647    }
1648
1649    return err;
1650}
1651
1652status_t NuPlayer::getCurrentPosition(int64_t *mediaUs) {
1653    sp<Renderer> renderer = mRenderer;
1654    if (renderer == NULL) {
1655        return NO_INIT;
1656    }
1657
1658    return renderer->getCurrentPosition(mediaUs);
1659}
1660
1661void NuPlayer::getStats(int64_t *numFramesTotal, int64_t *numFramesDropped) {
1662    *numFramesTotal = mNumFramesTotal;
1663    *numFramesDropped = mNumFramesDropped;
1664}
1665
1666sp<MetaData> NuPlayer::getFileMeta() {
1667    return mSource->getFileFormatMeta();
1668}
1669
1670void NuPlayer::schedulePollDuration() {
1671    sp<AMessage> msg = new AMessage(kWhatPollDuration, id());
1672    msg->setInt32("generation", mPollDurationGeneration);
1673    msg->post();
1674}
1675
1676void NuPlayer::cancelPollDuration() {
1677    ++mPollDurationGeneration;
1678}
1679
1680void NuPlayer::processDeferredActions() {
1681    while (!mDeferredActions.empty()) {
1682        // We won't execute any deferred actions until we're no longer in
1683        // an intermediate state, i.e. one more more decoders are currently
1684        // flushing or shutting down.
1685
1686        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1687            // We're currently flushing, postpone the reset until that's
1688            // completed.
1689
1690            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1691                  mFlushingAudio, mFlushingVideo);
1692
1693            break;
1694        }
1695
1696        sp<Action> action = *mDeferredActions.begin();
1697        mDeferredActions.erase(mDeferredActions.begin());
1698
1699        action->execute(this);
1700    }
1701}
1702
1703void NuPlayer::performSeek(int64_t seekTimeUs, bool needNotify) {
1704    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs), needNotify(%d)",
1705          seekTimeUs,
1706          seekTimeUs / 1E6,
1707          needNotify);
1708
1709    if (mSource == NULL) {
1710        // This happens when reset occurs right before the loop mode
1711        // asynchronously seeks to the start of the stream.
1712        LOG_ALWAYS_FATAL_IF(mAudioDecoder != NULL || mVideoDecoder != NULL,
1713                "mSource is NULL and decoders not NULL audio(%p) video(%p)",
1714                mAudioDecoder.get(), mVideoDecoder.get());
1715        return;
1716    }
1717    mSource->seekTo(seekTimeUs);
1718    ++mTimedTextGeneration;
1719
1720    if (mDriver != NULL) {
1721        sp<NuPlayerDriver> driver = mDriver.promote();
1722        if (driver != NULL) {
1723            if (needNotify) {
1724                driver->notifySeekComplete();
1725            }
1726        }
1727    }
1728
1729    // everything's flushed, continue playback.
1730}
1731
1732void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) {
1733    ALOGV("performDecoderFlush audio=%d, video=%d", audio, video);
1734
1735    if ((audio == FLUSH_CMD_NONE || mAudioDecoder == NULL)
1736            && (video == FLUSH_CMD_NONE || mVideoDecoder == NULL)) {
1737        return;
1738    }
1739
1740    mTimeDiscontinuityPending = true;
1741
1742    if (audio != FLUSH_CMD_NONE && mAudioDecoder != NULL) {
1743        flushDecoder(true /* audio */, (audio == FLUSH_CMD_SHUTDOWN));
1744    }
1745
1746    if (video != FLUSH_CMD_NONE && mVideoDecoder != NULL) {
1747        flushDecoder(false /* audio */, (video == FLUSH_CMD_SHUTDOWN));
1748    }
1749}
1750
1751void NuPlayer::performReset() {
1752    ALOGV("performReset");
1753
1754    CHECK(mAudioDecoder == NULL);
1755    CHECK(mVideoDecoder == NULL);
1756
1757    cancelPollDuration();
1758
1759    ++mScanSourcesGeneration;
1760    mScanSourcesPending = false;
1761
1762    if (mRendererLooper != NULL) {
1763        if (mRenderer != NULL) {
1764            mRendererLooper->unregisterHandler(mRenderer->id());
1765        }
1766        mRendererLooper->stop();
1767        mRendererLooper.clear();
1768    }
1769    mRenderer.clear();
1770    ++mRendererGeneration;
1771
1772    if (mSource != NULL) {
1773        mSource->stop();
1774
1775        mSource.clear();
1776    }
1777
1778    if (mDriver != NULL) {
1779        sp<NuPlayerDriver> driver = mDriver.promote();
1780        if (driver != NULL) {
1781            driver->notifyResetComplete();
1782        }
1783    }
1784
1785    mStarted = false;
1786}
1787
1788void NuPlayer::performScanSources() {
1789    ALOGV("performScanSources");
1790
1791    if (!mStarted) {
1792        return;
1793    }
1794
1795    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1796        postScanSources();
1797    }
1798}
1799
1800void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1801    ALOGV("performSetSurface");
1802
1803    mNativeWindow = wrapper;
1804
1805    // XXX - ignore error from setVideoScalingMode for now
1806    setVideoScalingMode(mVideoScalingMode);
1807
1808    if (mDriver != NULL) {
1809        sp<NuPlayerDriver> driver = mDriver.promote();
1810        if (driver != NULL) {
1811            driver->notifySetSurfaceComplete();
1812        }
1813    }
1814}
1815
1816void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1817    int32_t what;
1818    CHECK(msg->findInt32("what", &what));
1819
1820    switch (what) {
1821        case Source::kWhatPrepared:
1822        {
1823            if (mSource == NULL) {
1824                // This is a stale notification from a source that was
1825                // asynchronously preparing when the client called reset().
1826                // We handled the reset, the source is gone.
1827                break;
1828            }
1829
1830            int32_t err;
1831            CHECK(msg->findInt32("err", &err));
1832
1833            sp<NuPlayerDriver> driver = mDriver.promote();
1834            if (driver != NULL) {
1835                // notify duration first, so that it's definitely set when
1836                // the app received the "prepare complete" callback.
1837                int64_t durationUs;
1838                if (mSource->getDuration(&durationUs) == OK) {
1839                    driver->notifyDuration(durationUs);
1840                }
1841                driver->notifyPrepareCompleted(err);
1842            }
1843
1844            break;
1845        }
1846
1847        case Source::kWhatFlagsChanged:
1848        {
1849            uint32_t flags;
1850            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1851
1852            sp<NuPlayerDriver> driver = mDriver.promote();
1853            if (driver != NULL) {
1854                driver->notifyFlagsChanged(flags);
1855            }
1856
1857            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1858                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
1859                cancelPollDuration();
1860            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1861                    && (flags & Source::FLAG_DYNAMIC_DURATION)
1862                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
1863                schedulePollDuration();
1864            }
1865
1866            mSourceFlags = flags;
1867            break;
1868        }
1869
1870        case Source::kWhatVideoSizeChanged:
1871        {
1872            sp<AMessage> format;
1873            CHECK(msg->findMessage("format", &format));
1874
1875            updateVideoSize(format);
1876            break;
1877        }
1878
1879        case Source::kWhatBufferingUpdate:
1880        {
1881            int32_t percentage;
1882            CHECK(msg->findInt32("percentage", &percentage));
1883
1884            notifyListener(MEDIA_BUFFERING_UPDATE, percentage, 0);
1885            break;
1886        }
1887
1888        case Source::kWhatBufferingStart:
1889        {
1890            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
1891            break;
1892        }
1893
1894        case Source::kWhatBufferingEnd:
1895        {
1896            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
1897            break;
1898        }
1899
1900        case Source::kWhatSubtitleData:
1901        {
1902            sp<ABuffer> buffer;
1903            CHECK(msg->findBuffer("buffer", &buffer));
1904
1905            sendSubtitleData(buffer, 0 /* baseIndex */);
1906            break;
1907        }
1908
1909        case Source::kWhatTimedTextData:
1910        {
1911            int32_t generation;
1912            if (msg->findInt32("generation", &generation)
1913                    && generation != mTimedTextGeneration) {
1914                break;
1915            }
1916
1917            sp<ABuffer> buffer;
1918            CHECK(msg->findBuffer("buffer", &buffer));
1919
1920            sp<NuPlayerDriver> driver = mDriver.promote();
1921            if (driver == NULL) {
1922                break;
1923            }
1924
1925            int posMs;
1926            int64_t timeUs, posUs;
1927            driver->getCurrentPosition(&posMs);
1928            posUs = posMs * 1000;
1929            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1930
1931            if (posUs < timeUs) {
1932                if (!msg->findInt32("generation", &generation)) {
1933                    msg->setInt32("generation", mTimedTextGeneration);
1934                }
1935                msg->post(timeUs - posUs);
1936            } else {
1937                sendTimedTextData(buffer);
1938            }
1939            break;
1940        }
1941
1942        case Source::kWhatQueueDecoderShutdown:
1943        {
1944            int32_t audio, video;
1945            CHECK(msg->findInt32("audio", &audio));
1946            CHECK(msg->findInt32("video", &video));
1947
1948            sp<AMessage> reply;
1949            CHECK(msg->findMessage("reply", &reply));
1950
1951            queueDecoderShutdown(audio, video, reply);
1952            break;
1953        }
1954
1955        case Source::kWhatDrmNoLicense:
1956        {
1957            notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, ERROR_DRM_NO_LICENSE);
1958            break;
1959        }
1960
1961        default:
1962            TRESPASS();
1963    }
1964}
1965
1966void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
1967    int32_t what;
1968    CHECK(msg->findInt32("what", &what));
1969
1970    switch (what) {
1971        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
1972        {
1973            sp<ABuffer> buffer;
1974            CHECK(msg->findBuffer("buffer", &buffer));
1975
1976            size_t inbandTracks = 0;
1977            if (mSource != NULL) {
1978                inbandTracks = mSource->getTrackCount();
1979            }
1980
1981            sendSubtitleData(buffer, inbandTracks);
1982            break;
1983        }
1984
1985        case NuPlayer::CCDecoder::kWhatTrackAdded:
1986        {
1987            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
1988
1989            break;
1990        }
1991
1992        default:
1993            TRESPASS();
1994    }
1995
1996
1997}
1998
1999void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
2000    int32_t trackIndex;
2001    int64_t timeUs, durationUs;
2002    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
2003    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2004    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
2005
2006    Parcel in;
2007    in.writeInt32(trackIndex + baseIndex);
2008    in.writeInt64(timeUs);
2009    in.writeInt64(durationUs);
2010    in.writeInt32(buffer->size());
2011    in.writeInt32(buffer->size());
2012    in.write(buffer->data(), buffer->size());
2013
2014    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
2015}
2016
2017void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
2018    const void *data;
2019    size_t size = 0;
2020    int64_t timeUs;
2021    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
2022
2023    AString mime;
2024    CHECK(buffer->meta()->findString("mime", &mime));
2025    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
2026
2027    data = buffer->data();
2028    size = buffer->size();
2029
2030    Parcel parcel;
2031    if (size > 0) {
2032        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2033        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
2034        TextDescriptions::getParcelOfDescriptions(
2035                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
2036    }
2037
2038    if ((parcel.dataSize() > 0)) {
2039        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
2040    } else {  // send an empty timed text
2041        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
2042    }
2043}
2044////////////////////////////////////////////////////////////////////////////////
2045
2046sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
2047    sp<MetaData> meta = getFormatMeta(audio);
2048
2049    if (meta == NULL) {
2050        return NULL;
2051    }
2052
2053    sp<AMessage> msg = new AMessage;
2054
2055    if(convertMetaDataToMessage(meta, &msg) == OK) {
2056        return msg;
2057    }
2058    return NULL;
2059}
2060
2061void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
2062    sp<AMessage> notify = dupNotify();
2063    notify->setInt32("what", kWhatFlagsChanged);
2064    notify->setInt32("flags", flags);
2065    notify->post();
2066}
2067
2068void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
2069    sp<AMessage> notify = dupNotify();
2070    notify->setInt32("what", kWhatVideoSizeChanged);
2071    notify->setMessage("format", format);
2072    notify->post();
2073}
2074
2075void NuPlayer::Source::notifyPrepared(status_t err) {
2076    sp<AMessage> notify = dupNotify();
2077    notify->setInt32("what", kWhatPrepared);
2078    notify->setInt32("err", err);
2079    notify->post();
2080}
2081
2082void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
2083    TRESPASS();
2084}
2085
2086}  // namespace android
2087