NuPlayer.cpp revision d4d00616b6452e16889e6ca14b7fc5380ce0ed43 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19
20#include <inttypes.h>
21
22#include <utils/Log.h>
23
24#include "NuPlayer.h"
25
26#include "HTTPLiveSource.h"
27#include "NuPlayerCCDecoder.h"
28#include "NuPlayerDecoder.h"
29#include "NuPlayerDecoderBase.h"
30#include "NuPlayerDecoderPassThrough.h"
31#include "NuPlayerDriver.h"
32#include "NuPlayerRenderer.h"
33#include "NuPlayerSource.h"
34#include "RTSPSource.h"
35#include "StreamingSource.h"
36#include "GenericSource.h"
37#include "TextDescriptions.h"
38
39#include "ATSParser.h"
40
41#include <cutils/properties.h>
42
43#include <media/AudioResamplerPublic.h>
44#include <media/AVSyncSettings.h>
45#include <media/MediaCodecBuffer.h>
46
47#include <media/stagefright/foundation/hexdump.h>
48#include <media/stagefright/foundation/ABuffer.h>
49#include <media/stagefright/foundation/ADebug.h>
50#include <media/stagefright/foundation/AMessage.h>
51#include <media/stagefright/MediaBuffer.h>
52#include <media/stagefright/MediaDefs.h>
53#include <media/stagefright/MediaErrors.h>
54#include <media/stagefright/MetaData.h>
55
56#include <gui/IGraphicBufferProducer.h>
57#include <gui/Surface.h>
58
59#include "avc_utils.h"
60
61#include "ESDS.h"
62#include <media/stagefright/Utils.h>
63
64namespace android {
65
66struct NuPlayer::Action : public RefBase {
67    Action() {}
68
69    virtual void execute(NuPlayer *player) = 0;
70
71private:
72    DISALLOW_EVIL_CONSTRUCTORS(Action);
73};
74
75struct NuPlayer::SeekAction : public Action {
76    explicit SeekAction(int64_t seekTimeUs, MediaPlayerSeekMode mode)
77        : mSeekTimeUs(seekTimeUs),
78          mMode(mode) {
79    }
80
81    virtual void execute(NuPlayer *player) {
82        player->performSeek(mSeekTimeUs, mMode);
83    }
84
85private:
86    int64_t mSeekTimeUs;
87    MediaPlayerSeekMode mMode;
88
89    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
90};
91
92struct NuPlayer::ResumeDecoderAction : public Action {
93    explicit ResumeDecoderAction(bool needNotify)
94        : mNeedNotify(needNotify) {
95    }
96
97    virtual void execute(NuPlayer *player) {
98        player->performResumeDecoders(mNeedNotify);
99    }
100
101private:
102    bool mNeedNotify;
103
104    DISALLOW_EVIL_CONSTRUCTORS(ResumeDecoderAction);
105};
106
107struct NuPlayer::SetSurfaceAction : public Action {
108    explicit SetSurfaceAction(const sp<Surface> &surface)
109        : mSurface(surface) {
110    }
111
112    virtual void execute(NuPlayer *player) {
113        player->performSetSurface(mSurface);
114    }
115
116private:
117    sp<Surface> mSurface;
118
119    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
120};
121
122struct NuPlayer::FlushDecoderAction : public Action {
123    FlushDecoderAction(FlushCommand audio, FlushCommand video)
124        : mAudio(audio),
125          mVideo(video) {
126    }
127
128    virtual void execute(NuPlayer *player) {
129        player->performDecoderFlush(mAudio, mVideo);
130    }
131
132private:
133    FlushCommand mAudio;
134    FlushCommand mVideo;
135
136    DISALLOW_EVIL_CONSTRUCTORS(FlushDecoderAction);
137};
138
139struct NuPlayer::PostMessageAction : public Action {
140    explicit PostMessageAction(const sp<AMessage> &msg)
141        : mMessage(msg) {
142    }
143
144    virtual void execute(NuPlayer *) {
145        mMessage->post();
146    }
147
148private:
149    sp<AMessage> mMessage;
150
151    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
152};
153
154// Use this if there's no state necessary to save in order to execute
155// the action.
156struct NuPlayer::SimpleAction : public Action {
157    typedef void (NuPlayer::*ActionFunc)();
158
159    explicit SimpleAction(ActionFunc func)
160        : mFunc(func) {
161    }
162
163    virtual void execute(NuPlayer *player) {
164        (player->*mFunc)();
165    }
166
167private:
168    ActionFunc mFunc;
169
170    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
171};
172
173////////////////////////////////////////////////////////////////////////////////
174
175NuPlayer::NuPlayer(pid_t pid)
176    : mUIDValid(false),
177      mPID(pid),
178      mSourceFlags(0),
179      mOffloadAudio(false),
180      mAudioDecoderGeneration(0),
181      mVideoDecoderGeneration(0),
182      mRendererGeneration(0),
183      mPreviousSeekTimeUs(0),
184      mAudioEOS(false),
185      mVideoEOS(false),
186      mScanSourcesPending(false),
187      mScanSourcesGeneration(0),
188      mPollDurationGeneration(0),
189      mTimedTextGeneration(0),
190      mFlushingAudio(NONE),
191      mFlushingVideo(NONE),
192      mResumePending(false),
193      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
194      mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
195      mVideoFpsHint(-1.f),
196      mStarted(false),
197      mPrepared(false),
198      mResetting(false),
199      mSourceStarted(false),
200      mPaused(false),
201      mPausedByClient(true),
202      mPausedForBuffering(false) {
203    clearFlushComplete();
204}
205
206NuPlayer::~NuPlayer() {
207}
208
209void NuPlayer::setUID(uid_t uid) {
210    mUIDValid = true;
211    mUID = uid;
212}
213
214void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
215    mDriver = driver;
216}
217
218void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
219    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
220
221    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
222
223    msg->setObject("source", new StreamingSource(notify, source));
224    msg->post();
225}
226
227static bool IsHTTPLiveURL(const char *url) {
228    if (!strncasecmp("http://", url, 7)
229            || !strncasecmp("https://", url, 8)
230            || !strncasecmp("file://", url, 7)) {
231        size_t len = strlen(url);
232        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
233            return true;
234        }
235
236        if (strstr(url,"m3u8")) {
237            return true;
238        }
239    }
240
241    return false;
242}
243
244void NuPlayer::setDataSourceAsync(
245        const sp<IMediaHTTPService> &httpService,
246        const char *url,
247        const KeyedVector<String8, String8> *headers) {
248
249    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
250    size_t len = strlen(url);
251
252    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
253
254    sp<Source> source;
255    if (IsHTTPLiveURL(url)) {
256        source = new HTTPLiveSource(notify, httpService, url, headers);
257    } else if (!strncasecmp(url, "rtsp://", 7)) {
258        source = new RTSPSource(
259                notify, httpService, url, headers, mUIDValid, mUID);
260    } else if ((!strncasecmp(url, "http://", 7)
261                || !strncasecmp(url, "https://", 8))
262                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
263                    || strstr(url, ".sdp?"))) {
264        source = new RTSPSource(
265                notify, httpService, url, headers, mUIDValid, mUID, true);
266    } else {
267        sp<GenericSource> genericSource =
268                new GenericSource(notify, mUIDValid, mUID);
269
270        status_t err = genericSource->setDataSource(httpService, url, headers);
271
272        if (err == OK) {
273            source = genericSource;
274        } else {
275            ALOGE("Failed to set data source!");
276        }
277    }
278    msg->setObject("source", source);
279    msg->post();
280}
281
282void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
283    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
284
285    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
286
287    sp<GenericSource> source =
288            new GenericSource(notify, mUIDValid, mUID);
289
290    status_t err = source->setDataSource(fd, offset, length);
291
292    if (err != OK) {
293        ALOGE("Failed to set data source!");
294        source = NULL;
295    }
296
297    msg->setObject("source", source);
298    msg->post();
299}
300
301void NuPlayer::setDataSourceAsync(const sp<DataSource> &dataSource) {
302    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
303    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
304
305    sp<GenericSource> source = new GenericSource(notify, mUIDValid, mUID);
306    status_t err = source->setDataSource(dataSource);
307
308    if (err != OK) {
309        ALOGE("Failed to set data source!");
310        source = NULL;
311    }
312
313    msg->setObject("source", source);
314    msg->post();
315}
316
317void NuPlayer::prepareAsync() {
318    (new AMessage(kWhatPrepare, this))->post();
319}
320
321void NuPlayer::setVideoSurfaceTextureAsync(
322        const sp<IGraphicBufferProducer> &bufferProducer) {
323    sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
324
325    if (bufferProducer == NULL) {
326        msg->setObject("surface", NULL);
327    } else {
328        msg->setObject("surface", new Surface(bufferProducer, true /* controlledByApp */));
329    }
330
331    msg->post();
332}
333
334void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
335    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, this);
336    msg->setObject("sink", sink);
337    msg->post();
338}
339
340void NuPlayer::start() {
341    (new AMessage(kWhatStart, this))->post();
342}
343
344status_t NuPlayer::setPlaybackSettings(const AudioPlaybackRate &rate) {
345    // do some cursory validation of the settings here. audio modes are
346    // only validated when set on the audiosink.
347     if ((rate.mSpeed != 0.f && rate.mSpeed < AUDIO_TIMESTRETCH_SPEED_MIN)
348            || rate.mSpeed > AUDIO_TIMESTRETCH_SPEED_MAX
349            || rate.mPitch < AUDIO_TIMESTRETCH_SPEED_MIN
350            || rate.mPitch > AUDIO_TIMESTRETCH_SPEED_MAX) {
351        return BAD_VALUE;
352    }
353    sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
354    writeToAMessage(msg, rate);
355    sp<AMessage> response;
356    status_t err = msg->postAndAwaitResponse(&response);
357    if (err == OK && response != NULL) {
358        CHECK(response->findInt32("err", &err));
359    }
360    return err;
361}
362
363status_t NuPlayer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
364    sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
365    sp<AMessage> response;
366    status_t err = msg->postAndAwaitResponse(&response);
367    if (err == OK && response != NULL) {
368        CHECK(response->findInt32("err", &err));
369        if (err == OK) {
370            readFromAMessage(response, rate);
371        }
372    }
373    return err;
374}
375
376status_t NuPlayer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
377    sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
378    writeToAMessage(msg, sync, videoFpsHint);
379    sp<AMessage> response;
380    status_t err = msg->postAndAwaitResponse(&response);
381    if (err == OK && response != NULL) {
382        CHECK(response->findInt32("err", &err));
383    }
384    return err;
385}
386
387status_t NuPlayer::getSyncSettings(
388        AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
389    sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
390    sp<AMessage> response;
391    status_t err = msg->postAndAwaitResponse(&response);
392    if (err == OK && response != NULL) {
393        CHECK(response->findInt32("err", &err));
394        if (err == OK) {
395            readFromAMessage(response, sync, videoFps);
396        }
397    }
398    return err;
399}
400
401void NuPlayer::pause() {
402    (new AMessage(kWhatPause, this))->post();
403}
404
405void NuPlayer::resetAsync() {
406    sp<Source> source;
407    {
408        Mutex::Autolock autoLock(mSourceLock);
409        source = mSource;
410    }
411
412    if (source != NULL) {
413        // During a reset, the data source might be unresponsive already, we need to
414        // disconnect explicitly so that reads exit promptly.
415        // We can't queue the disconnect request to the looper, as it might be
416        // queued behind a stuck read and never gets processed.
417        // Doing a disconnect outside the looper to allows the pending reads to exit
418        // (either successfully or with error).
419        source->disconnect();
420    }
421
422    (new AMessage(kWhatReset, this))->post();
423}
424
425void NuPlayer::seekToAsync(int64_t seekTimeUs, MediaPlayerSeekMode mode, bool needNotify) {
426    sp<AMessage> msg = new AMessage(kWhatSeek, this);
427    msg->setInt64("seekTimeUs", seekTimeUs);
428    msg->setInt32("mode", mode);
429    msg->setInt32("needNotify", needNotify);
430    msg->post();
431}
432
433
434void NuPlayer::writeTrackInfo(
435        Parcel* reply, const sp<AMessage>& format) const {
436    if (format == NULL) {
437        ALOGE("NULL format");
438        return;
439    }
440    int32_t trackType;
441    if (!format->findInt32("type", &trackType)) {
442        ALOGE("no track type");
443        return;
444    }
445
446    AString mime;
447    if (!format->findString("mime", &mime)) {
448        // Java MediaPlayer only uses mimetype for subtitle and timedtext tracks.
449        // If we can't find the mimetype here it means that we wouldn't be needing
450        // the mimetype on the Java end. We still write a placeholder mime to keep the
451        // (de)serialization logic simple.
452        if (trackType == MEDIA_TRACK_TYPE_AUDIO) {
453            mime = "audio/";
454        } else if (trackType == MEDIA_TRACK_TYPE_VIDEO) {
455            mime = "video/";
456        } else {
457            ALOGE("unknown track type: %d", trackType);
458            return;
459        }
460    }
461
462    AString lang;
463    if (!format->findString("language", &lang)) {
464        ALOGE("no language");
465        return;
466    }
467
468    reply->writeInt32(2); // write something non-zero
469    reply->writeInt32(trackType);
470    reply->writeString16(String16(mime.c_str()));
471    reply->writeString16(String16(lang.c_str()));
472
473    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
474        int32_t isAuto, isDefault, isForced;
475        CHECK(format->findInt32("auto", &isAuto));
476        CHECK(format->findInt32("default", &isDefault));
477        CHECK(format->findInt32("forced", &isForced));
478
479        reply->writeInt32(isAuto);
480        reply->writeInt32(isDefault);
481        reply->writeInt32(isForced);
482    }
483}
484
485void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
486    switch (msg->what()) {
487        case kWhatSetDataSource:
488        {
489            ALOGV("kWhatSetDataSource");
490
491            CHECK(mSource == NULL);
492
493            status_t err = OK;
494            sp<RefBase> obj;
495            CHECK(msg->findObject("source", &obj));
496            if (obj != NULL) {
497                Mutex::Autolock autoLock(mSourceLock);
498                mSource = static_cast<Source *>(obj.get());
499            } else {
500                err = UNKNOWN_ERROR;
501            }
502
503            CHECK(mDriver != NULL);
504            sp<NuPlayerDriver> driver = mDriver.promote();
505            if (driver != NULL) {
506                driver->notifySetDataSourceCompleted(err);
507            }
508            break;
509        }
510
511        case kWhatPrepare:
512        {
513            mSource->prepareAsync();
514            break;
515        }
516
517        case kWhatGetTrackInfo:
518        {
519            sp<AReplyToken> replyID;
520            CHECK(msg->senderAwaitsResponse(&replyID));
521
522            Parcel* reply;
523            CHECK(msg->findPointer("reply", (void**)&reply));
524
525            size_t inbandTracks = 0;
526            if (mSource != NULL) {
527                inbandTracks = mSource->getTrackCount();
528            }
529
530            size_t ccTracks = 0;
531            if (mCCDecoder != NULL) {
532                ccTracks = mCCDecoder->getTrackCount();
533            }
534
535            // total track count
536            reply->writeInt32(inbandTracks + ccTracks);
537
538            // write inband tracks
539            for (size_t i = 0; i < inbandTracks; ++i) {
540                writeTrackInfo(reply, mSource->getTrackInfo(i));
541            }
542
543            // write CC track
544            for (size_t i = 0; i < ccTracks; ++i) {
545                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
546            }
547
548            sp<AMessage> response = new AMessage;
549            response->postReply(replyID);
550            break;
551        }
552
553        case kWhatGetSelectedTrack:
554        {
555            status_t err = INVALID_OPERATION;
556            if (mSource != NULL) {
557                err = OK;
558
559                int32_t type32;
560                CHECK(msg->findInt32("type", (int32_t*)&type32));
561                media_track_type type = (media_track_type)type32;
562                ssize_t selectedTrack = mSource->getSelectedTrack(type);
563
564                Parcel* reply;
565                CHECK(msg->findPointer("reply", (void**)&reply));
566                reply->writeInt32(selectedTrack);
567            }
568
569            sp<AMessage> response = new AMessage;
570            response->setInt32("err", err);
571
572            sp<AReplyToken> replyID;
573            CHECK(msg->senderAwaitsResponse(&replyID));
574            response->postReply(replyID);
575            break;
576        }
577
578        case kWhatSelectTrack:
579        {
580            sp<AReplyToken> replyID;
581            CHECK(msg->senderAwaitsResponse(&replyID));
582
583            size_t trackIndex;
584            int32_t select;
585            int64_t timeUs;
586            CHECK(msg->findSize("trackIndex", &trackIndex));
587            CHECK(msg->findInt32("select", &select));
588            CHECK(msg->findInt64("timeUs", &timeUs));
589
590            status_t err = INVALID_OPERATION;
591
592            size_t inbandTracks = 0;
593            if (mSource != NULL) {
594                inbandTracks = mSource->getTrackCount();
595            }
596            size_t ccTracks = 0;
597            if (mCCDecoder != NULL) {
598                ccTracks = mCCDecoder->getTrackCount();
599            }
600
601            if (trackIndex < inbandTracks) {
602                err = mSource->selectTrack(trackIndex, select, timeUs);
603
604                if (!select && err == OK) {
605                    int32_t type;
606                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
607                    if (info != NULL
608                            && info->findInt32("type", &type)
609                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
610                        ++mTimedTextGeneration;
611                    }
612                }
613            } else {
614                trackIndex -= inbandTracks;
615
616                if (trackIndex < ccTracks) {
617                    err = mCCDecoder->selectTrack(trackIndex, select);
618                }
619            }
620
621            sp<AMessage> response = new AMessage;
622            response->setInt32("err", err);
623
624            response->postReply(replyID);
625            break;
626        }
627
628        case kWhatPollDuration:
629        {
630            int32_t generation;
631            CHECK(msg->findInt32("generation", &generation));
632
633            if (generation != mPollDurationGeneration) {
634                // stale
635                break;
636            }
637
638            int64_t durationUs;
639            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
640                sp<NuPlayerDriver> driver = mDriver.promote();
641                if (driver != NULL) {
642                    driver->notifyDuration(durationUs);
643                }
644            }
645
646            msg->post(1000000ll);  // poll again in a second.
647            break;
648        }
649
650        case kWhatSetVideoSurface:
651        {
652
653            sp<RefBase> obj;
654            CHECK(msg->findObject("surface", &obj));
655            sp<Surface> surface = static_cast<Surface *>(obj.get());
656
657            ALOGD("onSetVideoSurface(%p, %s video decoder)",
658                    surface.get(),
659                    (mSource != NULL && mStarted && mSource->getFormat(false /* audio */) != NULL
660                            && mVideoDecoder != NULL) ? "have" : "no");
661
662            // Need to check mStarted before calling mSource->getFormat because NuPlayer might
663            // be in preparing state and it could take long time.
664            // When mStarted is true, mSource must have been set.
665            if (mSource == NULL || !mStarted || mSource->getFormat(false /* audio */) == NULL
666                    // NOTE: mVideoDecoder's mSurface is always non-null
667                    || (mVideoDecoder != NULL && mVideoDecoder->setVideoSurface(surface) == OK)) {
668                performSetSurface(surface);
669                break;
670            }
671
672            mDeferredActions.push_back(
673                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
674                                           FLUSH_CMD_SHUTDOWN /* video */));
675
676            mDeferredActions.push_back(new SetSurfaceAction(surface));
677
678            if (obj != NULL || mAudioDecoder != NULL) {
679                if (mStarted) {
680                    // Issue a seek to refresh the video screen only if started otherwise
681                    // the extractor may not yet be started and will assert.
682                    // If the video decoder is not set (perhaps audio only in this case)
683                    // do not perform a seek as it is not needed.
684                    int64_t currentPositionUs = 0;
685                    if (getCurrentPosition(&currentPositionUs) == OK) {
686                        mDeferredActions.push_back(
687                                new SeekAction(currentPositionUs,
688                                        MediaPlayerSeekMode::SEEK_PREVIOUS_SYNC /* mode */));
689                    }
690                }
691
692                // If there is a new surface texture, instantiate decoders
693                // again if possible.
694                mDeferredActions.push_back(
695                        new SimpleAction(&NuPlayer::performScanSources));
696            }
697
698            // After a flush without shutdown, decoder is paused.
699            // Don't resume it until source seek is done, otherwise it could
700            // start pulling stale data too soon.
701            mDeferredActions.push_back(
702                    new ResumeDecoderAction(false /* needNotify */));
703
704            processDeferredActions();
705            break;
706        }
707
708        case kWhatSetAudioSink:
709        {
710            ALOGV("kWhatSetAudioSink");
711
712            sp<RefBase> obj;
713            CHECK(msg->findObject("sink", &obj));
714
715            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
716            break;
717        }
718
719        case kWhatStart:
720        {
721            ALOGV("kWhatStart");
722            if (mStarted) {
723                // do not resume yet if the source is still buffering
724                if (!mPausedForBuffering) {
725                    onResume();
726                }
727            } else {
728                onStart();
729            }
730            mPausedByClient = false;
731            break;
732        }
733
734        case kWhatConfigPlayback:
735        {
736            sp<AReplyToken> replyID;
737            CHECK(msg->senderAwaitsResponse(&replyID));
738            AudioPlaybackRate rate /* sanitized */;
739            readFromAMessage(msg, &rate);
740            status_t err = OK;
741            if (mRenderer != NULL) {
742                // AudioSink allows only 1.f and 0.f for offload mode.
743                // For other speed, switch to non-offload mode.
744                if (mOffloadAudio && ((rate.mSpeed != 0.f && rate.mSpeed != 1.f)
745                        || rate.mPitch != 1.f)) {
746                    int64_t currentPositionUs;
747                    if (getCurrentPosition(&currentPositionUs) != OK) {
748                        currentPositionUs = mPreviousSeekTimeUs;
749                    }
750
751                    // Set mPlaybackSettings so that the new audio decoder can
752                    // be created correctly.
753                    mPlaybackSettings = rate;
754                    if (!mPaused) {
755                        mRenderer->pause();
756                    }
757                    restartAudio(
758                            currentPositionUs, true /* forceNonOffload */,
759                            true /* needsToCreateAudioDecoder */);
760                    if (!mPaused) {
761                        mRenderer->resume();
762                    }
763                }
764
765                err = mRenderer->setPlaybackSettings(rate);
766            }
767            if (err == OK) {
768                if (rate.mSpeed == 0.f) {
769                    onPause();
770                    mPausedByClient = true;
771                    // save all other settings (using non-paused speed)
772                    // so we can restore them on start
773                    AudioPlaybackRate newRate = rate;
774                    newRate.mSpeed = mPlaybackSettings.mSpeed;
775                    mPlaybackSettings = newRate;
776                } else { /* rate.mSpeed != 0.f */
777                    mPlaybackSettings = rate;
778                    if (mStarted) {
779                        // do not resume yet if the source is still buffering
780                        if (!mPausedForBuffering) {
781                            onResume();
782                        }
783                    } else if (mPrepared) {
784                        onStart();
785                    }
786
787                    mPausedByClient = false;
788                }
789            }
790
791            if (mVideoDecoder != NULL) {
792                sp<AMessage> params = new AMessage();
793                params->setFloat("playback-speed", mPlaybackSettings.mSpeed);
794                mVideoDecoder->setParameters(params);
795            }
796
797            sp<AMessage> response = new AMessage;
798            response->setInt32("err", err);
799            response->postReply(replyID);
800            break;
801        }
802
803        case kWhatGetPlaybackSettings:
804        {
805            sp<AReplyToken> replyID;
806            CHECK(msg->senderAwaitsResponse(&replyID));
807            AudioPlaybackRate rate = mPlaybackSettings;
808            status_t err = OK;
809            if (mRenderer != NULL) {
810                err = mRenderer->getPlaybackSettings(&rate);
811            }
812            if (err == OK) {
813                // get playback settings used by renderer, as it may be
814                // slightly off due to audiosink not taking small changes.
815                mPlaybackSettings = rate;
816                if (mPaused) {
817                    rate.mSpeed = 0.f;
818                }
819            }
820            sp<AMessage> response = new AMessage;
821            if (err == OK) {
822                writeToAMessage(response, rate);
823            }
824            response->setInt32("err", err);
825            response->postReply(replyID);
826            break;
827        }
828
829        case kWhatConfigSync:
830        {
831            sp<AReplyToken> replyID;
832            CHECK(msg->senderAwaitsResponse(&replyID));
833
834            ALOGV("kWhatConfigSync");
835            AVSyncSettings sync;
836            float videoFpsHint;
837            readFromAMessage(msg, &sync, &videoFpsHint);
838            status_t err = OK;
839            if (mRenderer != NULL) {
840                err = mRenderer->setSyncSettings(sync, videoFpsHint);
841            }
842            if (err == OK) {
843                mSyncSettings = sync;
844                mVideoFpsHint = videoFpsHint;
845            }
846            sp<AMessage> response = new AMessage;
847            response->setInt32("err", err);
848            response->postReply(replyID);
849            break;
850        }
851
852        case kWhatGetSyncSettings:
853        {
854            sp<AReplyToken> replyID;
855            CHECK(msg->senderAwaitsResponse(&replyID));
856            AVSyncSettings sync = mSyncSettings;
857            float videoFps = mVideoFpsHint;
858            status_t err = OK;
859            if (mRenderer != NULL) {
860                err = mRenderer->getSyncSettings(&sync, &videoFps);
861                if (err == OK) {
862                    mSyncSettings = sync;
863                    mVideoFpsHint = videoFps;
864                }
865            }
866            sp<AMessage> response = new AMessage;
867            if (err == OK) {
868                writeToAMessage(response, sync, videoFps);
869            }
870            response->setInt32("err", err);
871            response->postReply(replyID);
872            break;
873        }
874
875        case kWhatScanSources:
876        {
877            int32_t generation;
878            CHECK(msg->findInt32("generation", &generation));
879            if (generation != mScanSourcesGeneration) {
880                // Drop obsolete msg.
881                break;
882            }
883
884            mScanSourcesPending = false;
885
886            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
887                 mAudioDecoder != NULL, mVideoDecoder != NULL);
888
889            bool mHadAnySourcesBefore =
890                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
891            bool rescan = false;
892
893            // initialize video before audio because successful initialization of
894            // video may change deep buffer mode of audio.
895            if (mSurface != NULL) {
896                if (instantiateDecoder(false, &mVideoDecoder) == -EWOULDBLOCK) {
897                    rescan = true;
898                }
899            }
900
901            // Don't try to re-open audio sink if there's an existing decoder.
902            if (mAudioSink != NULL && mAudioDecoder == NULL) {
903                if (instantiateDecoder(true, &mAudioDecoder) == -EWOULDBLOCK) {
904                    rescan = true;
905                }
906            }
907
908            if (!mHadAnySourcesBefore
909                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
910                // This is the first time we've found anything playable.
911
912                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
913                    schedulePollDuration();
914                }
915            }
916
917            status_t err;
918            if ((err = mSource->feedMoreTSData()) != OK) {
919                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
920                    // We're not currently decoding anything (no audio or
921                    // video tracks found) and we just ran out of input data.
922
923                    if (err == ERROR_END_OF_STREAM) {
924                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
925                    } else {
926                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
927                    }
928                }
929                break;
930            }
931
932            if (rescan) {
933                msg->post(100000ll);
934                mScanSourcesPending = true;
935            }
936            break;
937        }
938
939        case kWhatVideoNotify:
940        case kWhatAudioNotify:
941        {
942            bool audio = msg->what() == kWhatAudioNotify;
943
944            int32_t currentDecoderGeneration =
945                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
946            int32_t requesterGeneration = currentDecoderGeneration - 1;
947            CHECK(msg->findInt32("generation", &requesterGeneration));
948
949            if (requesterGeneration != currentDecoderGeneration) {
950                ALOGV("got message from old %s decoder, generation(%d:%d)",
951                        audio ? "audio" : "video", requesterGeneration,
952                        currentDecoderGeneration);
953                sp<AMessage> reply;
954                if (!(msg->findMessage("reply", &reply))) {
955                    return;
956                }
957
958                reply->setInt32("err", INFO_DISCONTINUITY);
959                reply->post();
960                return;
961            }
962
963            int32_t what;
964            CHECK(msg->findInt32("what", &what));
965
966            if (what == DecoderBase::kWhatInputDiscontinuity) {
967                int32_t formatChange;
968                CHECK(msg->findInt32("formatChange", &formatChange));
969
970                ALOGV("%s discontinuity: formatChange %d",
971                        audio ? "audio" : "video", formatChange);
972
973                if (formatChange) {
974                    mDeferredActions.push_back(
975                            new FlushDecoderAction(
976                                audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
977                                audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN));
978                }
979
980                mDeferredActions.push_back(
981                        new SimpleAction(
982                                &NuPlayer::performScanSources));
983
984                processDeferredActions();
985            } else if (what == DecoderBase::kWhatEOS) {
986                int32_t err;
987                CHECK(msg->findInt32("err", &err));
988
989                if (err == ERROR_END_OF_STREAM) {
990                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
991                } else {
992                    ALOGV("got %s decoder EOS w/ error %d",
993                         audio ? "audio" : "video",
994                         err);
995                }
996
997                mRenderer->queueEOS(audio, err);
998            } else if (what == DecoderBase::kWhatFlushCompleted) {
999                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
1000
1001                handleFlushComplete(audio, true /* isDecoder */);
1002                finishFlushIfPossible();
1003            } else if (what == DecoderBase::kWhatVideoSizeChanged) {
1004                sp<AMessage> format;
1005                CHECK(msg->findMessage("format", &format));
1006
1007                sp<AMessage> inputFormat =
1008                        mSource->getFormat(false /* audio */);
1009
1010                setVideoScalingMode(mVideoScalingMode);
1011                updateVideoSize(inputFormat, format);
1012            } else if (what == DecoderBase::kWhatShutdownCompleted) {
1013                ALOGV("%s shutdown completed", audio ? "audio" : "video");
1014                if (audio) {
1015                    mAudioDecoder.clear();
1016                    ++mAudioDecoderGeneration;
1017
1018                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
1019                    mFlushingAudio = SHUT_DOWN;
1020                } else {
1021                    mVideoDecoder.clear();
1022                    ++mVideoDecoderGeneration;
1023
1024                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
1025                    mFlushingVideo = SHUT_DOWN;
1026                }
1027
1028                finishFlushIfPossible();
1029            } else if (what == DecoderBase::kWhatResumeCompleted) {
1030                finishResume();
1031            } else if (what == DecoderBase::kWhatError) {
1032                status_t err;
1033                if (!msg->findInt32("err", &err) || err == OK) {
1034                    err = UNKNOWN_ERROR;
1035                }
1036
1037                // Decoder errors can be due to Source (e.g. from streaming),
1038                // or from decoding corrupted bitstreams, or from other decoder
1039                // MediaCodec operations (e.g. from an ongoing reset or seek).
1040                // They may also be due to openAudioSink failure at
1041                // decoder start or after a format change.
1042                //
1043                // We try to gracefully shut down the affected decoder if possible,
1044                // rather than trying to force the shutdown with something
1045                // similar to performReset(). This method can lead to a hang
1046                // if MediaCodec functions block after an error, but they should
1047                // typically return INVALID_OPERATION instead of blocking.
1048
1049                FlushStatus *flushing = audio ? &mFlushingAudio : &mFlushingVideo;
1050                ALOGE("received error(%#x) from %s decoder, flushing(%d), now shutting down",
1051                        err, audio ? "audio" : "video", *flushing);
1052
1053                switch (*flushing) {
1054                    case NONE:
1055                        mDeferredActions.push_back(
1056                                new FlushDecoderAction(
1057                                    audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
1058                                    audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN));
1059                        processDeferredActions();
1060                        break;
1061                    case FLUSHING_DECODER:
1062                        *flushing = FLUSHING_DECODER_SHUTDOWN; // initiate shutdown after flush.
1063                        break; // Wait for flush to complete.
1064                    case FLUSHING_DECODER_SHUTDOWN:
1065                        break; // Wait for flush to complete.
1066                    case SHUTTING_DOWN_DECODER:
1067                        break; // Wait for shutdown to complete.
1068                    case FLUSHED:
1069                        // Widevine source reads must stop before releasing the video decoder.
1070                        if (!audio && mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
1071                            mSource->stop();
1072                            mSourceStarted = false;
1073                        }
1074                        getDecoder(audio)->initiateShutdown(); // In the middle of a seek.
1075                        *flushing = SHUTTING_DOWN_DECODER;     // Shut down.
1076                        break;
1077                    case SHUT_DOWN:
1078                        finishFlushIfPossible();  // Should not occur.
1079                        break;                    // Finish anyways.
1080                }
1081                notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
1082            } else {
1083                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
1084                      what,
1085                      what >> 24,
1086                      (what >> 16) & 0xff,
1087                      (what >> 8) & 0xff,
1088                      what & 0xff);
1089            }
1090
1091            break;
1092        }
1093
1094        case kWhatRendererNotify:
1095        {
1096            int32_t requesterGeneration = mRendererGeneration - 1;
1097            CHECK(msg->findInt32("generation", &requesterGeneration));
1098            if (requesterGeneration != mRendererGeneration) {
1099                ALOGV("got message from old renderer, generation(%d:%d)",
1100                        requesterGeneration, mRendererGeneration);
1101                return;
1102            }
1103
1104            int32_t what;
1105            CHECK(msg->findInt32("what", &what));
1106
1107            if (what == Renderer::kWhatEOS) {
1108                int32_t audio;
1109                CHECK(msg->findInt32("audio", &audio));
1110
1111                int32_t finalResult;
1112                CHECK(msg->findInt32("finalResult", &finalResult));
1113
1114                if (audio) {
1115                    mAudioEOS = true;
1116                } else {
1117                    mVideoEOS = true;
1118                }
1119
1120                if (finalResult == ERROR_END_OF_STREAM) {
1121                    ALOGV("reached %s EOS", audio ? "audio" : "video");
1122                } else {
1123                    ALOGE("%s track encountered an error (%d)",
1124                         audio ? "audio" : "video", finalResult);
1125
1126                    notifyListener(
1127                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
1128                }
1129
1130                if ((mAudioEOS || mAudioDecoder == NULL)
1131                        && (mVideoEOS || mVideoDecoder == NULL)) {
1132                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
1133                }
1134            } else if (what == Renderer::kWhatFlushComplete) {
1135                int32_t audio;
1136                CHECK(msg->findInt32("audio", &audio));
1137
1138                if (audio) {
1139                    mAudioEOS = false;
1140                } else {
1141                    mVideoEOS = false;
1142                }
1143
1144                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
1145                if (audio && (mFlushingAudio == NONE || mFlushingAudio == FLUSHED
1146                        || mFlushingAudio == SHUT_DOWN)) {
1147                    // Flush has been handled by tear down.
1148                    break;
1149                }
1150                handleFlushComplete(audio, false /* isDecoder */);
1151                finishFlushIfPossible();
1152            } else if (what == Renderer::kWhatVideoRenderingStart) {
1153                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
1154            } else if (what == Renderer::kWhatMediaRenderingStart) {
1155                ALOGV("media rendering started");
1156                notifyListener(MEDIA_STARTED, 0, 0);
1157            } else if (what == Renderer::kWhatAudioTearDown) {
1158                int32_t reason;
1159                CHECK(msg->findInt32("reason", &reason));
1160                ALOGV("Tear down audio with reason %d.", reason);
1161                if (reason == Renderer::kDueToTimeout && !(mPaused && mOffloadAudio)) {
1162                    // TimeoutWhenPaused is only for offload mode.
1163                    ALOGW("Receive a stale message for teardown.");
1164                    break;
1165                }
1166                int64_t positionUs;
1167                if (!msg->findInt64("positionUs", &positionUs)) {
1168                    positionUs = mPreviousSeekTimeUs;
1169                }
1170
1171                restartAudio(
1172                        positionUs, reason == Renderer::kForceNonOffload /* forceNonOffload */,
1173                        reason != Renderer::kDueToTimeout /* needsToCreateAudioDecoder */);
1174            }
1175            break;
1176        }
1177
1178        case kWhatMoreDataQueued:
1179        {
1180            break;
1181        }
1182
1183        case kWhatReset:
1184        {
1185            ALOGV("kWhatReset");
1186
1187            mResetting = true;
1188
1189            mDeferredActions.push_back(
1190                    new FlushDecoderAction(
1191                        FLUSH_CMD_SHUTDOWN /* audio */,
1192                        FLUSH_CMD_SHUTDOWN /* video */));
1193
1194            mDeferredActions.push_back(
1195                    new SimpleAction(&NuPlayer::performReset));
1196
1197            processDeferredActions();
1198            break;
1199        }
1200
1201        case kWhatSeek:
1202        {
1203            int64_t seekTimeUs;
1204            int32_t mode;
1205            int32_t needNotify;
1206            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
1207            CHECK(msg->findInt32("mode", &mode));
1208            CHECK(msg->findInt32("needNotify", &needNotify));
1209
1210            ALOGV("kWhatSeek seekTimeUs=%lld us, mode=%d, needNotify=%d",
1211                    (long long)seekTimeUs, mode, needNotify);
1212
1213            if (!mStarted) {
1214                // Seek before the player is started. In order to preview video,
1215                // need to start the player and pause it. This branch is called
1216                // only once if needed. After the player is started, any seek
1217                // operation will go through normal path.
1218                // Audio-only cases are handled separately.
1219                onStart(seekTimeUs, (MediaPlayerSeekMode)mode);
1220                if (mStarted) {
1221                    onPause();
1222                    mPausedByClient = true;
1223                }
1224                if (needNotify) {
1225                    notifyDriverSeekComplete();
1226                }
1227                break;
1228            }
1229
1230            mDeferredActions.push_back(
1231                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
1232                                           FLUSH_CMD_FLUSH /* video */));
1233
1234            mDeferredActions.push_back(
1235                    new SeekAction(seekTimeUs, (MediaPlayerSeekMode)mode));
1236
1237            // After a flush without shutdown, decoder is paused.
1238            // Don't resume it until source seek is done, otherwise it could
1239            // start pulling stale data too soon.
1240            mDeferredActions.push_back(
1241                    new ResumeDecoderAction(needNotify));
1242
1243            processDeferredActions();
1244            break;
1245        }
1246
1247        case kWhatPause:
1248        {
1249            onPause();
1250            mPausedByClient = true;
1251            break;
1252        }
1253
1254        case kWhatSourceNotify:
1255        {
1256            onSourceNotify(msg);
1257            break;
1258        }
1259
1260        case kWhatClosedCaptionNotify:
1261        {
1262            onClosedCaptionNotify(msg);
1263            break;
1264        }
1265
1266        default:
1267            TRESPASS();
1268            break;
1269    }
1270}
1271
1272void NuPlayer::onResume() {
1273    if (!mPaused || mResetting) {
1274        ALOGD_IF(mResetting, "resetting, onResume discarded");
1275        return;
1276    }
1277    mPaused = false;
1278    if (mSource != NULL) {
1279        mSource->resume();
1280    } else {
1281        ALOGW("resume called when source is gone or not set");
1282    }
1283    // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if
1284    // needed.
1285    if (audioDecoderStillNeeded() && mAudioDecoder == NULL) {
1286        instantiateDecoder(true /* audio */, &mAudioDecoder);
1287    }
1288    if (mRenderer != NULL) {
1289        mRenderer->resume();
1290    } else {
1291        ALOGW("resume called when renderer is gone or not set");
1292    }
1293
1294    mLastStartedPlayingTimeNs = systemTime();
1295}
1296
1297status_t NuPlayer::onInstantiateSecureDecoders() {
1298    status_t err;
1299    if (!(mSourceFlags & Source::FLAG_SECURE)) {
1300        return BAD_TYPE;
1301    }
1302
1303    if (mRenderer != NULL) {
1304        ALOGE("renderer should not be set when instantiating secure decoders");
1305        return UNKNOWN_ERROR;
1306    }
1307
1308    // TRICKY: We rely on mRenderer being null, so that decoder does not start requesting
1309    // data on instantiation.
1310    if (mSurface != NULL) {
1311        err = instantiateDecoder(false, &mVideoDecoder);
1312        if (err != OK) {
1313            return err;
1314        }
1315    }
1316
1317    if (mAudioSink != NULL) {
1318        err = instantiateDecoder(true, &mAudioDecoder);
1319        if (err != OK) {
1320            return err;
1321        }
1322    }
1323    return OK;
1324}
1325
1326void NuPlayer::onStart(int64_t startPositionUs, MediaPlayerSeekMode mode) {
1327    if (!mSourceStarted) {
1328        mSourceStarted = true;
1329        mSource->start();
1330    }
1331    if (startPositionUs > 0) {
1332        performSeek(startPositionUs, mode);
1333        if (mSource->getFormat(false /* audio */) == NULL) {
1334            return;
1335        }
1336    }
1337
1338    mOffloadAudio = false;
1339    mAudioEOS = false;
1340    mVideoEOS = false;
1341    mStarted = true;
1342    mPaused = false;
1343
1344    uint32_t flags = 0;
1345
1346    if (mSource->isRealTime()) {
1347        flags |= Renderer::FLAG_REAL_TIME;
1348    }
1349
1350    sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
1351    sp<MetaData> videoMeta = mSource->getFormatMeta(false /* audio */);
1352    if (audioMeta == NULL && videoMeta == NULL) {
1353        ALOGE("no metadata for either audio or video source");
1354        mSource->stop();
1355        mSourceStarted = false;
1356        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, ERROR_MALFORMED);
1357        return;
1358    }
1359    ALOGV_IF(audioMeta == NULL, "no metadata for audio source");  // video only stream
1360
1361    audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
1362    if (mAudioSink != NULL) {
1363        streamType = mAudioSink->getAudioStreamType();
1364    }
1365
1366    sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
1367
1368    mOffloadAudio =
1369        canOffloadStream(audioMeta, (videoFormat != NULL), mSource->isStreaming(), streamType)
1370                && (mPlaybackSettings.mSpeed == 1.f && mPlaybackSettings.mPitch == 1.f);
1371    if (mOffloadAudio) {
1372        flags |= Renderer::FLAG_OFFLOAD_AUDIO;
1373    }
1374
1375    sp<AMessage> notify = new AMessage(kWhatRendererNotify, this);
1376    ++mRendererGeneration;
1377    notify->setInt32("generation", mRendererGeneration);
1378    mRenderer = new Renderer(mAudioSink, notify, flags);
1379    mRendererLooper = new ALooper;
1380    mRendererLooper->setName("NuPlayerRenderer");
1381    mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
1382    mRendererLooper->registerHandler(mRenderer);
1383
1384    status_t err = mRenderer->setPlaybackSettings(mPlaybackSettings);
1385    if (err != OK) {
1386        mSource->stop();
1387        mSourceStarted = false;
1388        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
1389        return;
1390    }
1391
1392    float rate = getFrameRate();
1393    if (rate > 0) {
1394        mRenderer->setVideoFrameRate(rate);
1395    }
1396
1397    if (mVideoDecoder != NULL) {
1398        mVideoDecoder->setRenderer(mRenderer);
1399    }
1400    if (mAudioDecoder != NULL) {
1401        mAudioDecoder->setRenderer(mRenderer);
1402    }
1403
1404    mLastStartedPlayingTimeNs = systemTime();
1405
1406    postScanSources();
1407}
1408
1409void NuPlayer::onPause() {
1410    if (mPaused) {
1411        return;
1412    }
1413    mPaused = true;
1414    if (mSource != NULL) {
1415        mSource->pause();
1416    } else {
1417        ALOGW("pause called when source is gone or not set");
1418    }
1419    if (mRenderer != NULL) {
1420        mRenderer->pause();
1421    } else {
1422        ALOGW("pause called when renderer is gone or not set");
1423    }
1424
1425    sp<NuPlayerDriver> driver = mDriver.promote();
1426    if (driver != NULL) {
1427        int64_t now = systemTime();
1428        int64_t played = now - mLastStartedPlayingTimeNs;
1429        ALOGD("played from %" PRId64 " to %" PRId64 " = %" PRId64 ,
1430              mLastStartedPlayingTimeNs, now, played);
1431
1432        driver->notifyMorePlayingTimeUs((played+500)/1000);
1433    }
1434}
1435
1436bool NuPlayer::audioDecoderStillNeeded() {
1437    // Audio decoder is no longer needed if it's in shut/shutting down status.
1438    return ((mFlushingAudio != SHUT_DOWN) && (mFlushingAudio != SHUTTING_DOWN_DECODER));
1439}
1440
1441void NuPlayer::handleFlushComplete(bool audio, bool isDecoder) {
1442    // We wait for both the decoder flush and the renderer flush to complete
1443    // before entering either the FLUSHED or the SHUTTING_DOWN_DECODER state.
1444
1445    mFlushComplete[audio][isDecoder] = true;
1446    if (!mFlushComplete[audio][!isDecoder]) {
1447        return;
1448    }
1449
1450    FlushStatus *state = audio ? &mFlushingAudio : &mFlushingVideo;
1451    switch (*state) {
1452        case FLUSHING_DECODER:
1453        {
1454            *state = FLUSHED;
1455            break;
1456        }
1457
1458        case FLUSHING_DECODER_SHUTDOWN:
1459        {
1460            *state = SHUTTING_DOWN_DECODER;
1461
1462            ALOGV("initiating %s decoder shutdown", audio ? "audio" : "video");
1463            if (!audio) {
1464                // Widevine source reads must stop before releasing the video decoder.
1465                if (mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
1466                    mSource->stop();
1467                    mSourceStarted = false;
1468                }
1469            }
1470            getDecoder(audio)->initiateShutdown();
1471            break;
1472        }
1473
1474        default:
1475            // decoder flush completes only occur in a flushing state.
1476            LOG_ALWAYS_FATAL_IF(isDecoder, "decoder flush in invalid state %d", *state);
1477            break;
1478    }
1479}
1480
1481void NuPlayer::finishFlushIfPossible() {
1482    if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED
1483            && mFlushingAudio != SHUT_DOWN) {
1484        return;
1485    }
1486
1487    if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED
1488            && mFlushingVideo != SHUT_DOWN) {
1489        return;
1490    }
1491
1492    ALOGV("both audio and video are flushed now.");
1493
1494    mFlushingAudio = NONE;
1495    mFlushingVideo = NONE;
1496
1497    clearFlushComplete();
1498
1499    processDeferredActions();
1500}
1501
1502void NuPlayer::postScanSources() {
1503    if (mScanSourcesPending) {
1504        return;
1505    }
1506
1507    sp<AMessage> msg = new AMessage(kWhatScanSources, this);
1508    msg->setInt32("generation", mScanSourcesGeneration);
1509    msg->post();
1510
1511    mScanSourcesPending = true;
1512}
1513
1514void NuPlayer::tryOpenAudioSinkForOffload(
1515        const sp<AMessage> &format, const sp<MetaData> &audioMeta, bool hasVideo) {
1516    // Note: This is called early in NuPlayer to determine whether offloading
1517    // is possible; otherwise the decoders call the renderer openAudioSink directly.
1518
1519    status_t err = mRenderer->openAudioSink(
1520            format, true /* offloadOnly */, hasVideo, AUDIO_OUTPUT_FLAG_NONE, &mOffloadAudio);
1521    if (err != OK) {
1522        // Any failure we turn off mOffloadAudio.
1523        mOffloadAudio = false;
1524    } else if (mOffloadAudio) {
1525        sendMetaDataToHal(mAudioSink, audioMeta);
1526    }
1527}
1528
1529void NuPlayer::closeAudioSink() {
1530    mRenderer->closeAudioSink();
1531}
1532
1533void NuPlayer::restartAudio(
1534        int64_t currentPositionUs, bool forceNonOffload, bool needsToCreateAudioDecoder) {
1535    if (mAudioDecoder != NULL) {
1536        mAudioDecoder->pause();
1537        mAudioDecoder.clear();
1538        ++mAudioDecoderGeneration;
1539    }
1540    if (mFlushingAudio == FLUSHING_DECODER) {
1541        mFlushComplete[1 /* audio */][1 /* isDecoder */] = true;
1542        mFlushingAudio = FLUSHED;
1543        finishFlushIfPossible();
1544    } else if (mFlushingAudio == FLUSHING_DECODER_SHUTDOWN
1545            || mFlushingAudio == SHUTTING_DOWN_DECODER) {
1546        mFlushComplete[1 /* audio */][1 /* isDecoder */] = true;
1547        mFlushingAudio = SHUT_DOWN;
1548        finishFlushIfPossible();
1549        needsToCreateAudioDecoder = false;
1550    }
1551    if (mRenderer == NULL) {
1552        return;
1553    }
1554    closeAudioSink();
1555    mRenderer->flush(true /* audio */, false /* notifyComplete */);
1556    if (mVideoDecoder != NULL) {
1557        mRenderer->flush(false /* audio */, false /* notifyComplete */);
1558    }
1559
1560    performSeek(currentPositionUs, MediaPlayerSeekMode::SEEK_PREVIOUS_SYNC /* mode */);
1561
1562    if (forceNonOffload) {
1563        mRenderer->signalDisableOffloadAudio();
1564        mOffloadAudio = false;
1565    }
1566    if (needsToCreateAudioDecoder) {
1567        instantiateDecoder(true /* audio */, &mAudioDecoder, !forceNonOffload);
1568    }
1569}
1570
1571void NuPlayer::determineAudioModeChange(const sp<AMessage> &audioFormat) {
1572    if (mSource == NULL || mAudioSink == NULL) {
1573        return;
1574    }
1575
1576    if (mRenderer == NULL) {
1577        ALOGW("No renderer can be used to determine audio mode. Use non-offload for safety.");
1578        mOffloadAudio = false;
1579        return;
1580    }
1581
1582    sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
1583    sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
1584    audio_stream_type_t streamType = mAudioSink->getAudioStreamType();
1585    const bool hasVideo = (videoFormat != NULL);
1586    const bool canOffload = canOffloadStream(
1587            audioMeta, hasVideo, mSource->isStreaming(), streamType)
1588                    && (mPlaybackSettings.mSpeed == 1.f && mPlaybackSettings.mPitch == 1.f);
1589    if (canOffload) {
1590        if (!mOffloadAudio) {
1591            mRenderer->signalEnableOffloadAudio();
1592        }
1593        // open audio sink early under offload mode.
1594        tryOpenAudioSinkForOffload(audioFormat, audioMeta, hasVideo);
1595    } else {
1596        if (mOffloadAudio) {
1597            mRenderer->signalDisableOffloadAudio();
1598            mOffloadAudio = false;
1599        }
1600    }
1601}
1602
1603status_t NuPlayer::instantiateDecoder(
1604        bool audio, sp<DecoderBase> *decoder, bool checkAudioModeChange) {
1605    // The audio decoder could be cleared by tear down. If still in shut down
1606    // process, no need to create a new audio decoder.
1607    if (*decoder != NULL || (audio && mFlushingAudio == SHUT_DOWN)) {
1608        return OK;
1609    }
1610
1611    sp<AMessage> format = mSource->getFormat(audio);
1612
1613    if (format == NULL) {
1614        return UNKNOWN_ERROR;
1615    } else {
1616        status_t err;
1617        if (format->findInt32("err", &err) && err) {
1618            return err;
1619        }
1620    }
1621
1622    format->setInt32("priority", 0 /* realtime */);
1623
1624    if (!audio) {
1625        AString mime;
1626        CHECK(format->findString("mime", &mime));
1627
1628        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, this);
1629        if (mCCDecoder == NULL) {
1630            mCCDecoder = new CCDecoder(ccNotify);
1631        }
1632
1633        if (mSourceFlags & Source::FLAG_SECURE) {
1634            format->setInt32("secure", true);
1635        }
1636
1637        if (mSourceFlags & Source::FLAG_PROTECTED) {
1638            format->setInt32("protected", true);
1639        }
1640
1641        float rate = getFrameRate();
1642        if (rate > 0) {
1643            format->setFloat("operating-rate", rate * mPlaybackSettings.mSpeed);
1644        }
1645    }
1646
1647    if (audio) {
1648        sp<AMessage> notify = new AMessage(kWhatAudioNotify, this);
1649        ++mAudioDecoderGeneration;
1650        notify->setInt32("generation", mAudioDecoderGeneration);
1651
1652        if (checkAudioModeChange) {
1653            determineAudioModeChange(format);
1654        }
1655        if (mOffloadAudio) {
1656            mSource->setOffloadAudio(true /* offload */);
1657
1658            const bool hasVideo = (mSource->getFormat(false /*audio */) != NULL);
1659            format->setInt32("has-video", hasVideo);
1660            *decoder = new DecoderPassThrough(notify, mSource, mRenderer);
1661        } else {
1662            mSource->setOffloadAudio(false /* offload */);
1663
1664            *decoder = new Decoder(notify, mSource, mPID, mUID, mRenderer);
1665        }
1666    } else {
1667        sp<AMessage> notify = new AMessage(kWhatVideoNotify, this);
1668        ++mVideoDecoderGeneration;
1669        notify->setInt32("generation", mVideoDecoderGeneration);
1670
1671        *decoder = new Decoder(
1672                notify, mSource, mPID, mUID, mRenderer, mSurface, mCCDecoder);
1673
1674        // enable FRC if high-quality AV sync is requested, even if not
1675        // directly queuing to display, as this will even improve textureview
1676        // playback.
1677        {
1678            if (property_get_bool("persist.sys.media.avsync", false)) {
1679                format->setInt32("auto-frc", 1);
1680            }
1681        }
1682    }
1683    (*decoder)->init();
1684    (*decoder)->configure(format);
1685
1686    if (!audio) {
1687        sp<AMessage> params = new AMessage();
1688        float rate = getFrameRate();
1689        if (rate > 0) {
1690            params->setFloat("frame-rate-total", rate);
1691        }
1692
1693        sp<MetaData> fileMeta = getFileMeta();
1694        if (fileMeta != NULL) {
1695            int32_t videoTemporalLayerCount;
1696            if (fileMeta->findInt32(kKeyTemporalLayerCount, &videoTemporalLayerCount)
1697                    && videoTemporalLayerCount > 0) {
1698                params->setInt32("temporal-layer-count", videoTemporalLayerCount);
1699            }
1700        }
1701
1702        if (params->countEntries() > 0) {
1703            (*decoder)->setParameters(params);
1704        }
1705    }
1706    return OK;
1707}
1708
1709void NuPlayer::updateVideoSize(
1710        const sp<AMessage> &inputFormat,
1711        const sp<AMessage> &outputFormat) {
1712    if (inputFormat == NULL) {
1713        ALOGW("Unknown video size, reporting 0x0!");
1714        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1715        return;
1716    }
1717
1718    int32_t displayWidth, displayHeight;
1719    if (outputFormat != NULL) {
1720        int32_t width, height;
1721        CHECK(outputFormat->findInt32("width", &width));
1722        CHECK(outputFormat->findInt32("height", &height));
1723
1724        int32_t cropLeft, cropTop, cropRight, cropBottom;
1725        CHECK(outputFormat->findRect(
1726                    "crop",
1727                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1728
1729        displayWidth = cropRight - cropLeft + 1;
1730        displayHeight = cropBottom - cropTop + 1;
1731
1732        ALOGV("Video output format changed to %d x %d "
1733             "(crop: %d x %d @ (%d, %d))",
1734             width, height,
1735             displayWidth,
1736             displayHeight,
1737             cropLeft, cropTop);
1738    } else {
1739        CHECK(inputFormat->findInt32("width", &displayWidth));
1740        CHECK(inputFormat->findInt32("height", &displayHeight));
1741
1742        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1743    }
1744
1745    // Take into account sample aspect ratio if necessary:
1746    int32_t sarWidth, sarHeight;
1747    if (inputFormat->findInt32("sar-width", &sarWidth)
1748            && inputFormat->findInt32("sar-height", &sarHeight)) {
1749        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1750
1751        displayWidth = (displayWidth * sarWidth) / sarHeight;
1752
1753        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1754    } else {
1755        int32_t width, height;
1756        if (inputFormat->findInt32("display-width", &width)
1757                && inputFormat->findInt32("display-height", &height)
1758                && width > 0 && height > 0
1759                && displayWidth > 0 && displayHeight > 0) {
1760            if (displayHeight * (int64_t)width / height > (int64_t)displayWidth) {
1761                displayHeight = (int32_t)(displayWidth * (int64_t)height / width);
1762            } else {
1763                displayWidth = (int32_t)(displayHeight * (int64_t)width / height);
1764            }
1765            ALOGV("Video display width and height are overridden to %d x %d",
1766                 displayWidth, displayHeight);
1767        }
1768    }
1769
1770    int32_t rotationDegrees;
1771    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1772        rotationDegrees = 0;
1773    }
1774
1775    if (rotationDegrees == 90 || rotationDegrees == 270) {
1776        int32_t tmp = displayWidth;
1777        displayWidth = displayHeight;
1778        displayHeight = tmp;
1779    }
1780
1781    notifyListener(
1782            MEDIA_SET_VIDEO_SIZE,
1783            displayWidth,
1784            displayHeight);
1785}
1786
1787void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1788    if (mDriver == NULL) {
1789        return;
1790    }
1791
1792    sp<NuPlayerDriver> driver = mDriver.promote();
1793
1794    if (driver == NULL) {
1795        return;
1796    }
1797
1798    driver->notifyListener(msg, ext1, ext2, in);
1799}
1800
1801void NuPlayer::flushDecoder(bool audio, bool needShutdown) {
1802    ALOGV("[%s] flushDecoder needShutdown=%d",
1803          audio ? "audio" : "video", needShutdown);
1804
1805    const sp<DecoderBase> &decoder = getDecoder(audio);
1806    if (decoder == NULL) {
1807        ALOGI("flushDecoder %s without decoder present",
1808             audio ? "audio" : "video");
1809        return;
1810    }
1811
1812    // Make sure we don't continue to scan sources until we finish flushing.
1813    ++mScanSourcesGeneration;
1814    if (mScanSourcesPending) {
1815        if (!needShutdown) {
1816            mDeferredActions.push_back(
1817                    new SimpleAction(&NuPlayer::performScanSources));
1818        }
1819        mScanSourcesPending = false;
1820    }
1821
1822    decoder->signalFlush();
1823
1824    FlushStatus newStatus =
1825        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1826
1827    mFlushComplete[audio][false /* isDecoder */] = (mRenderer == NULL);
1828    mFlushComplete[audio][true /* isDecoder */] = false;
1829    if (audio) {
1830        ALOGE_IF(mFlushingAudio != NONE,
1831                "audio flushDecoder() is called in state %d", mFlushingAudio);
1832        mFlushingAudio = newStatus;
1833    } else {
1834        ALOGE_IF(mFlushingVideo != NONE,
1835                "video flushDecoder() is called in state %d", mFlushingVideo);
1836        mFlushingVideo = newStatus;
1837    }
1838}
1839
1840void NuPlayer::queueDecoderShutdown(
1841        bool audio, bool video, const sp<AMessage> &reply) {
1842    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1843
1844    mDeferredActions.push_back(
1845            new FlushDecoderAction(
1846                audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
1847                video ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE));
1848
1849    mDeferredActions.push_back(
1850            new SimpleAction(&NuPlayer::performScanSources));
1851
1852    mDeferredActions.push_back(new PostMessageAction(reply));
1853
1854    processDeferredActions();
1855}
1856
1857status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1858    mVideoScalingMode = mode;
1859    if (mSurface != NULL) {
1860        status_t ret = native_window_set_scaling_mode(mSurface.get(), mVideoScalingMode);
1861        if (ret != OK) {
1862            ALOGE("Failed to set scaling mode (%d): %s",
1863                -ret, strerror(-ret));
1864            return ret;
1865        }
1866    }
1867    return OK;
1868}
1869
1870status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1871    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, this);
1872    msg->setPointer("reply", reply);
1873
1874    sp<AMessage> response;
1875    status_t err = msg->postAndAwaitResponse(&response);
1876    return err;
1877}
1878
1879status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1880    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, this);
1881    msg->setPointer("reply", reply);
1882    msg->setInt32("type", type);
1883
1884    sp<AMessage> response;
1885    status_t err = msg->postAndAwaitResponse(&response);
1886    if (err == OK && response != NULL) {
1887        CHECK(response->findInt32("err", &err));
1888    }
1889    return err;
1890}
1891
1892status_t NuPlayer::selectTrack(size_t trackIndex, bool select, int64_t timeUs) {
1893    sp<AMessage> msg = new AMessage(kWhatSelectTrack, this);
1894    msg->setSize("trackIndex", trackIndex);
1895    msg->setInt32("select", select);
1896    msg->setInt64("timeUs", timeUs);
1897
1898    sp<AMessage> response;
1899    status_t err = msg->postAndAwaitResponse(&response);
1900
1901    if (err != OK) {
1902        return err;
1903    }
1904
1905    if (!response->findInt32("err", &err)) {
1906        err = OK;
1907    }
1908
1909    return err;
1910}
1911
1912status_t NuPlayer::getCurrentPosition(int64_t *mediaUs) {
1913    sp<Renderer> renderer = mRenderer;
1914    if (renderer == NULL) {
1915        return NO_INIT;
1916    }
1917
1918    return renderer->getCurrentPosition(mediaUs);
1919}
1920
1921void NuPlayer::getStats(Vector<sp<AMessage> > *mTrackStats) {
1922    CHECK(mTrackStats != NULL);
1923
1924    mTrackStats->clear();
1925    if (mVideoDecoder != NULL) {
1926        mTrackStats->push_back(mVideoDecoder->getStats());
1927    }
1928    if (mAudioDecoder != NULL) {
1929        mTrackStats->push_back(mAudioDecoder->getStats());
1930    }
1931}
1932
1933sp<MetaData> NuPlayer::getFileMeta() {
1934    return mSource->getFileFormatMeta();
1935}
1936
1937float NuPlayer::getFrameRate() {
1938    sp<MetaData> meta = mSource->getFormatMeta(false /* audio */);
1939    if (meta == NULL) {
1940        return 0;
1941    }
1942    int32_t rate;
1943    if (!meta->findInt32(kKeyFrameRate, &rate)) {
1944        // fall back to try file meta
1945        sp<MetaData> fileMeta = getFileMeta();
1946        if (fileMeta == NULL) {
1947            ALOGW("source has video meta but not file meta");
1948            return -1;
1949        }
1950        int32_t fileMetaRate;
1951        if (!fileMeta->findInt32(kKeyFrameRate, &fileMetaRate)) {
1952            return -1;
1953        }
1954        return fileMetaRate;
1955    }
1956    return rate;
1957}
1958
1959void NuPlayer::schedulePollDuration() {
1960    sp<AMessage> msg = new AMessage(kWhatPollDuration, this);
1961    msg->setInt32("generation", mPollDurationGeneration);
1962    msg->post();
1963}
1964
1965void NuPlayer::cancelPollDuration() {
1966    ++mPollDurationGeneration;
1967}
1968
1969void NuPlayer::processDeferredActions() {
1970    while (!mDeferredActions.empty()) {
1971        // We won't execute any deferred actions until we're no longer in
1972        // an intermediate state, i.e. one more more decoders are currently
1973        // flushing or shutting down.
1974
1975        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1976            // We're currently flushing, postpone the reset until that's
1977            // completed.
1978
1979            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1980                  mFlushingAudio, mFlushingVideo);
1981
1982            break;
1983        }
1984
1985        sp<Action> action = *mDeferredActions.begin();
1986        mDeferredActions.erase(mDeferredActions.begin());
1987
1988        action->execute(this);
1989    }
1990}
1991
1992void NuPlayer::performSeek(int64_t seekTimeUs, MediaPlayerSeekMode mode) {
1993    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs), mode=%d",
1994          (long long)seekTimeUs, seekTimeUs / 1E6, mode);
1995
1996    if (mSource == NULL) {
1997        // This happens when reset occurs right before the loop mode
1998        // asynchronously seeks to the start of the stream.
1999        LOG_ALWAYS_FATAL_IF(mAudioDecoder != NULL || mVideoDecoder != NULL,
2000                "mSource is NULL and decoders not NULL audio(%p) video(%p)",
2001                mAudioDecoder.get(), mVideoDecoder.get());
2002        return;
2003    }
2004    mPreviousSeekTimeUs = seekTimeUs;
2005    mSource->seekTo(seekTimeUs, mode);
2006    ++mTimedTextGeneration;
2007
2008    // everything's flushed, continue playback.
2009}
2010
2011void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) {
2012    ALOGV("performDecoderFlush audio=%d, video=%d", audio, video);
2013
2014    if ((audio == FLUSH_CMD_NONE || mAudioDecoder == NULL)
2015            && (video == FLUSH_CMD_NONE || mVideoDecoder == NULL)) {
2016        return;
2017    }
2018
2019    if (audio != FLUSH_CMD_NONE && mAudioDecoder != NULL) {
2020        flushDecoder(true /* audio */, (audio == FLUSH_CMD_SHUTDOWN));
2021    }
2022
2023    if (video != FLUSH_CMD_NONE && mVideoDecoder != NULL) {
2024        flushDecoder(false /* audio */, (video == FLUSH_CMD_SHUTDOWN));
2025    }
2026}
2027
2028void NuPlayer::performReset() {
2029    ALOGV("performReset");
2030
2031    CHECK(mAudioDecoder == NULL);
2032    CHECK(mVideoDecoder == NULL);
2033
2034    cancelPollDuration();
2035
2036    ++mScanSourcesGeneration;
2037    mScanSourcesPending = false;
2038
2039    if (mRendererLooper != NULL) {
2040        if (mRenderer != NULL) {
2041            mRendererLooper->unregisterHandler(mRenderer->id());
2042        }
2043        mRendererLooper->stop();
2044        mRendererLooper.clear();
2045    }
2046    mRenderer.clear();
2047    ++mRendererGeneration;
2048
2049    if (mSource != NULL) {
2050        mSource->stop();
2051
2052        Mutex::Autolock autoLock(mSourceLock);
2053        mSource.clear();
2054    }
2055
2056    if (mDriver != NULL) {
2057        sp<NuPlayerDriver> driver = mDriver.promote();
2058        if (driver != NULL) {
2059            driver->notifyResetComplete();
2060        }
2061    }
2062
2063    mStarted = false;
2064    mPrepared = false;
2065    mResetting = false;
2066    mSourceStarted = false;
2067}
2068
2069void NuPlayer::performScanSources() {
2070    ALOGV("performScanSources");
2071
2072    if (!mStarted) {
2073        return;
2074    }
2075
2076    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
2077        postScanSources();
2078    }
2079}
2080
2081void NuPlayer::performSetSurface(const sp<Surface> &surface) {
2082    ALOGV("performSetSurface");
2083
2084    mSurface = surface;
2085
2086    // XXX - ignore error from setVideoScalingMode for now
2087    setVideoScalingMode(mVideoScalingMode);
2088
2089    if (mDriver != NULL) {
2090        sp<NuPlayerDriver> driver = mDriver.promote();
2091        if (driver != NULL) {
2092            driver->notifySetSurfaceComplete();
2093        }
2094    }
2095}
2096
2097void NuPlayer::performResumeDecoders(bool needNotify) {
2098    if (needNotify) {
2099        mResumePending = true;
2100        if (mVideoDecoder == NULL) {
2101            // if audio-only, we can notify seek complete now,
2102            // as the resume operation will be relatively fast.
2103            finishResume();
2104        }
2105    }
2106
2107    if (mVideoDecoder != NULL) {
2108        // When there is continuous seek, MediaPlayer will cache the seek
2109        // position, and send down new seek request when previous seek is
2110        // complete. Let's wait for at least one video output frame before
2111        // notifying seek complete, so that the video thumbnail gets updated
2112        // when seekbar is dragged.
2113        mVideoDecoder->signalResume(needNotify);
2114    }
2115
2116    if (mAudioDecoder != NULL) {
2117        mAudioDecoder->signalResume(false /* needNotify */);
2118    }
2119}
2120
2121void NuPlayer::finishResume() {
2122    if (mResumePending) {
2123        mResumePending = false;
2124        notifyDriverSeekComplete();
2125    }
2126}
2127
2128void NuPlayer::notifyDriverSeekComplete() {
2129    if (mDriver != NULL) {
2130        sp<NuPlayerDriver> driver = mDriver.promote();
2131        if (driver != NULL) {
2132            driver->notifySeekComplete();
2133        }
2134    }
2135}
2136
2137void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
2138    int32_t what;
2139    CHECK(msg->findInt32("what", &what));
2140
2141    switch (what) {
2142        case Source::kWhatInstantiateSecureDecoders:
2143        {
2144            if (mSource == NULL) {
2145                // This is a stale notification from a source that was
2146                // asynchronously preparing when the client called reset().
2147                // We handled the reset, the source is gone.
2148                break;
2149            }
2150
2151            sp<AMessage> reply;
2152            CHECK(msg->findMessage("reply", &reply));
2153            status_t err = onInstantiateSecureDecoders();
2154            reply->setInt32("err", err);
2155            reply->post();
2156            break;
2157        }
2158
2159        case Source::kWhatPrepared:
2160        {
2161            if (mSource == NULL) {
2162                // This is a stale notification from a source that was
2163                // asynchronously preparing when the client called reset().
2164                // We handled the reset, the source is gone.
2165                break;
2166            }
2167
2168            int32_t err;
2169            CHECK(msg->findInt32("err", &err));
2170
2171            if (err != OK) {
2172                // shut down potential secure codecs in case client never calls reset
2173                mDeferredActions.push_back(
2174                        new FlushDecoderAction(FLUSH_CMD_SHUTDOWN /* audio */,
2175                                               FLUSH_CMD_SHUTDOWN /* video */));
2176                processDeferredActions();
2177            } else {
2178                mPrepared = true;
2179            }
2180
2181            sp<NuPlayerDriver> driver = mDriver.promote();
2182            if (driver != NULL) {
2183                // notify duration first, so that it's definitely set when
2184                // the app received the "prepare complete" callback.
2185                int64_t durationUs;
2186                if (mSource->getDuration(&durationUs) == OK) {
2187                    driver->notifyDuration(durationUs);
2188                }
2189                driver->notifyPrepareCompleted(err);
2190            }
2191
2192            break;
2193        }
2194
2195        case Source::kWhatFlagsChanged:
2196        {
2197            uint32_t flags;
2198            CHECK(msg->findInt32("flags", (int32_t *)&flags));
2199
2200            sp<NuPlayerDriver> driver = mDriver.promote();
2201            if (driver != NULL) {
2202                if ((flags & NuPlayer::Source::FLAG_CAN_SEEK) == 0) {
2203                    driver->notifyListener(
2204                            MEDIA_INFO, MEDIA_INFO_NOT_SEEKABLE, 0);
2205                }
2206                driver->notifyFlagsChanged(flags);
2207            }
2208
2209            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
2210                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
2211                cancelPollDuration();
2212            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
2213                    && (flags & Source::FLAG_DYNAMIC_DURATION)
2214                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
2215                schedulePollDuration();
2216            }
2217
2218            mSourceFlags = flags;
2219            break;
2220        }
2221
2222        case Source::kWhatVideoSizeChanged:
2223        {
2224            sp<AMessage> format;
2225            CHECK(msg->findMessage("format", &format));
2226
2227            updateVideoSize(format);
2228            break;
2229        }
2230
2231        case Source::kWhatBufferingUpdate:
2232        {
2233            int32_t percentage;
2234            CHECK(msg->findInt32("percentage", &percentage));
2235
2236            notifyListener(MEDIA_BUFFERING_UPDATE, percentage, 0);
2237            break;
2238        }
2239
2240        case Source::kWhatPauseOnBufferingStart:
2241        {
2242            // ignore if not playing
2243            if (mStarted) {
2244                ALOGI("buffer low, pausing...");
2245
2246                mPausedForBuffering = true;
2247                onPause();
2248            }
2249            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
2250            break;
2251        }
2252
2253        case Source::kWhatResumeOnBufferingEnd:
2254        {
2255            // ignore if not playing
2256            if (mStarted) {
2257                ALOGI("buffer ready, resuming...");
2258
2259                mPausedForBuffering = false;
2260
2261                // do not resume yet if client didn't unpause
2262                if (!mPausedByClient) {
2263                    onResume();
2264                }
2265            }
2266            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
2267            break;
2268        }
2269
2270        case Source::kWhatCacheStats:
2271        {
2272            int32_t kbps;
2273            CHECK(msg->findInt32("bandwidth", &kbps));
2274
2275            notifyListener(MEDIA_INFO, MEDIA_INFO_NETWORK_BANDWIDTH, kbps);
2276            break;
2277        }
2278
2279        case Source::kWhatSubtitleData:
2280        {
2281            sp<ABuffer> buffer;
2282            CHECK(msg->findBuffer("buffer", &buffer));
2283
2284            sendSubtitleData(buffer, 0 /* baseIndex */);
2285            break;
2286        }
2287
2288        case Source::kWhatTimedMetaData:
2289        {
2290            sp<ABuffer> buffer;
2291            if (!msg->findBuffer("buffer", &buffer)) {
2292                notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
2293            } else {
2294                sendTimedMetaData(buffer);
2295            }
2296            break;
2297        }
2298
2299        case Source::kWhatTimedTextData:
2300        {
2301            int32_t generation;
2302            if (msg->findInt32("generation", &generation)
2303                    && generation != mTimedTextGeneration) {
2304                break;
2305            }
2306
2307            sp<ABuffer> buffer;
2308            CHECK(msg->findBuffer("buffer", &buffer));
2309
2310            sp<NuPlayerDriver> driver = mDriver.promote();
2311            if (driver == NULL) {
2312                break;
2313            }
2314
2315            int posMs;
2316            int64_t timeUs, posUs;
2317            driver->getCurrentPosition(&posMs);
2318            posUs = (int64_t) posMs * 1000ll;
2319            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2320
2321            if (posUs < timeUs) {
2322                if (!msg->findInt32("generation", &generation)) {
2323                    msg->setInt32("generation", mTimedTextGeneration);
2324                }
2325                msg->post(timeUs - posUs);
2326            } else {
2327                sendTimedTextData(buffer);
2328            }
2329            break;
2330        }
2331
2332        case Source::kWhatQueueDecoderShutdown:
2333        {
2334            int32_t audio, video;
2335            CHECK(msg->findInt32("audio", &audio));
2336            CHECK(msg->findInt32("video", &video));
2337
2338            sp<AMessage> reply;
2339            CHECK(msg->findMessage("reply", &reply));
2340
2341            queueDecoderShutdown(audio, video, reply);
2342            break;
2343        }
2344
2345        case Source::kWhatDrmNoLicense:
2346        {
2347            notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, ERROR_DRM_NO_LICENSE);
2348            break;
2349        }
2350
2351        default:
2352            TRESPASS();
2353    }
2354}
2355
2356void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
2357    int32_t what;
2358    CHECK(msg->findInt32("what", &what));
2359
2360    switch (what) {
2361        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
2362        {
2363            sp<ABuffer> buffer;
2364            CHECK(msg->findBuffer("buffer", &buffer));
2365
2366            size_t inbandTracks = 0;
2367            if (mSource != NULL) {
2368                inbandTracks = mSource->getTrackCount();
2369            }
2370
2371            sendSubtitleData(buffer, inbandTracks);
2372            break;
2373        }
2374
2375        case NuPlayer::CCDecoder::kWhatTrackAdded:
2376        {
2377            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
2378
2379            break;
2380        }
2381
2382        default:
2383            TRESPASS();
2384    }
2385
2386
2387}
2388
2389void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
2390    int32_t trackIndex;
2391    int64_t timeUs, durationUs;
2392    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
2393    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2394    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
2395
2396    Parcel in;
2397    in.writeInt32(trackIndex + baseIndex);
2398    in.writeInt64(timeUs);
2399    in.writeInt64(durationUs);
2400    in.writeInt32(buffer->size());
2401    in.writeInt32(buffer->size());
2402    in.write(buffer->data(), buffer->size());
2403
2404    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
2405}
2406
2407void NuPlayer::sendTimedMetaData(const sp<ABuffer> &buffer) {
2408    int64_t timeUs;
2409    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2410
2411    Parcel in;
2412    in.writeInt64(timeUs);
2413    in.writeInt32(buffer->size());
2414    in.writeInt32(buffer->size());
2415    in.write(buffer->data(), buffer->size());
2416
2417    notifyListener(MEDIA_META_DATA, 0, 0, &in);
2418}
2419
2420void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
2421    const void *data;
2422    size_t size = 0;
2423    int64_t timeUs;
2424    int32_t flag = TextDescriptions::IN_BAND_TEXT_3GPP;
2425
2426    AString mime;
2427    CHECK(buffer->meta()->findString("mime", &mime));
2428    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
2429
2430    data = buffer->data();
2431    size = buffer->size();
2432
2433    Parcel parcel;
2434    if (size > 0) {
2435        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2436        int32_t global = 0;
2437        if (buffer->meta()->findInt32("global", &global) && global) {
2438            flag |= TextDescriptions::GLOBAL_DESCRIPTIONS;
2439        } else {
2440            flag |= TextDescriptions::LOCAL_DESCRIPTIONS;
2441        }
2442        TextDescriptions::getParcelOfDescriptions(
2443                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
2444    }
2445
2446    if ((parcel.dataSize() > 0)) {
2447        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
2448    } else {  // send an empty timed text
2449        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
2450    }
2451}
2452////////////////////////////////////////////////////////////////////////////////
2453
2454sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
2455    sp<MetaData> meta = getFormatMeta(audio);
2456
2457    if (meta == NULL) {
2458        return NULL;
2459    }
2460
2461    sp<AMessage> msg = new AMessage;
2462
2463    if(convertMetaDataToMessage(meta, &msg) == OK) {
2464        return msg;
2465    }
2466    return NULL;
2467}
2468
2469void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
2470    sp<AMessage> notify = dupNotify();
2471    notify->setInt32("what", kWhatFlagsChanged);
2472    notify->setInt32("flags", flags);
2473    notify->post();
2474}
2475
2476void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
2477    sp<AMessage> notify = dupNotify();
2478    notify->setInt32("what", kWhatVideoSizeChanged);
2479    notify->setMessage("format", format);
2480    notify->post();
2481}
2482
2483void NuPlayer::Source::notifyPrepared(status_t err) {
2484    sp<AMessage> notify = dupNotify();
2485    notify->setInt32("what", kWhatPrepared);
2486    notify->setInt32("err", err);
2487    notify->post();
2488}
2489
2490void NuPlayer::Source::notifyInstantiateSecureDecoders(const sp<AMessage> &reply) {
2491    sp<AMessage> notify = dupNotify();
2492    notify->setInt32("what", kWhatInstantiateSecureDecoders);
2493    notify->setMessage("reply", reply);
2494    notify->post();
2495}
2496
2497void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
2498    TRESPASS();
2499}
2500
2501}  // namespace android
2502