NuPlayer.cpp revision e4d18c7f84186b935ac569d5919059c455edd390 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerCCDecoder.h"
25#include "NuPlayerDecoder.h"
26#include "NuPlayerDecoderBase.h"
27#include "NuPlayerDecoderPassThrough.h"
28#include "NuPlayerDriver.h"
29#include "NuPlayerRenderer.h"
30#include "NuPlayerSource.h"
31#include "RTSPSource.h"
32#include "StreamingSource.h"
33#include "GenericSource.h"
34#include "TextDescriptions.h"
35
36#include "ATSParser.h"
37
38#include <cutils/properties.h>
39
40#include <media/AudioResamplerPublic.h>
41#include <media/AVSyncSettings.h>
42
43#include <media/stagefright/foundation/hexdump.h>
44#include <media/stagefright/foundation/ABuffer.h>
45#include <media/stagefright/foundation/ADebug.h>
46#include <media/stagefright/foundation/AMessage.h>
47#include <media/stagefright/MediaBuffer.h>
48#include <media/stagefright/MediaDefs.h>
49#include <media/stagefright/MediaErrors.h>
50#include <media/stagefright/MetaData.h>
51
52#include <gui/IGraphicBufferProducer.h>
53#include <gui/Surface.h>
54
55#include "avc_utils.h"
56
57#include "ESDS.h"
58#include <media/stagefright/Utils.h>
59
60namespace android {
61
62struct NuPlayer::Action : public RefBase {
63    Action() {}
64
65    virtual void execute(NuPlayer *player) = 0;
66
67private:
68    DISALLOW_EVIL_CONSTRUCTORS(Action);
69};
70
71struct NuPlayer::SeekAction : public Action {
72    SeekAction(int64_t seekTimeUs)
73        : mSeekTimeUs(seekTimeUs) {
74    }
75
76    virtual void execute(NuPlayer *player) {
77        player->performSeek(mSeekTimeUs);
78    }
79
80private:
81    int64_t mSeekTimeUs;
82
83    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
84};
85
86struct NuPlayer::ResumeDecoderAction : public Action {
87    ResumeDecoderAction(bool needNotify)
88        : mNeedNotify(needNotify) {
89    }
90
91    virtual void execute(NuPlayer *player) {
92        player->performResumeDecoders(mNeedNotify);
93    }
94
95private:
96    bool mNeedNotify;
97
98    DISALLOW_EVIL_CONSTRUCTORS(ResumeDecoderAction);
99};
100
101struct NuPlayer::SetSurfaceAction : public Action {
102    SetSurfaceAction(const sp<Surface> &surface)
103        : mSurface(surface) {
104    }
105
106    virtual void execute(NuPlayer *player) {
107        player->performSetSurface(mSurface);
108    }
109
110private:
111    sp<Surface> mSurface;
112
113    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
114};
115
116struct NuPlayer::FlushDecoderAction : public Action {
117    FlushDecoderAction(FlushCommand audio, FlushCommand video)
118        : mAudio(audio),
119          mVideo(video) {
120    }
121
122    virtual void execute(NuPlayer *player) {
123        player->performDecoderFlush(mAudio, mVideo);
124    }
125
126private:
127    FlushCommand mAudio;
128    FlushCommand mVideo;
129
130    DISALLOW_EVIL_CONSTRUCTORS(FlushDecoderAction);
131};
132
133struct NuPlayer::PostMessageAction : public Action {
134    PostMessageAction(const sp<AMessage> &msg)
135        : mMessage(msg) {
136    }
137
138    virtual void execute(NuPlayer *) {
139        mMessage->post();
140    }
141
142private:
143    sp<AMessage> mMessage;
144
145    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
146};
147
148// Use this if there's no state necessary to save in order to execute
149// the action.
150struct NuPlayer::SimpleAction : public Action {
151    typedef void (NuPlayer::*ActionFunc)();
152
153    SimpleAction(ActionFunc func)
154        : mFunc(func) {
155    }
156
157    virtual void execute(NuPlayer *player) {
158        (player->*mFunc)();
159    }
160
161private:
162    ActionFunc mFunc;
163
164    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
165};
166
167////////////////////////////////////////////////////////////////////////////////
168
169NuPlayer::NuPlayer()
170    : mUIDValid(false),
171      mSourceFlags(0),
172      mOffloadAudio(false),
173      mAudioDecoderGeneration(0),
174      mVideoDecoderGeneration(0),
175      mRendererGeneration(0),
176      mAudioEOS(false),
177      mVideoEOS(false),
178      mScanSourcesPending(false),
179      mScanSourcesGeneration(0),
180      mPollDurationGeneration(0),
181      mTimedTextGeneration(0),
182      mFlushingAudio(NONE),
183      mFlushingVideo(NONE),
184      mResumePending(false),
185      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
186      mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
187      mVideoFpsHint(-1.f),
188      mStarted(false),
189      mSourceStarted(false),
190      mPaused(false),
191      mPausedByClient(false),
192      mPausedForBuffering(false) {
193    clearFlushComplete();
194}
195
196NuPlayer::~NuPlayer() {
197}
198
199void NuPlayer::setUID(uid_t uid) {
200    mUIDValid = true;
201    mUID = uid;
202}
203
204void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
205    mDriver = driver;
206}
207
208void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
209    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
210
211    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
212
213    msg->setObject("source", new StreamingSource(notify, source));
214    msg->post();
215}
216
217static bool IsHTTPLiveURL(const char *url) {
218    if (!strncasecmp("http://", url, 7)
219            || !strncasecmp("https://", url, 8)
220            || !strncasecmp("file://", url, 7)) {
221        size_t len = strlen(url);
222        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
223            return true;
224        }
225
226        if (strstr(url,"m3u8")) {
227            return true;
228        }
229    }
230
231    return false;
232}
233
234void NuPlayer::setDataSourceAsync(
235        const sp<IMediaHTTPService> &httpService,
236        const char *url,
237        const KeyedVector<String8, String8> *headers) {
238
239    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
240    size_t len = strlen(url);
241
242    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
243
244    sp<Source> source;
245    if (IsHTTPLiveURL(url)) {
246        source = new HTTPLiveSource(notify, httpService, url, headers);
247    } else if (!strncasecmp(url, "rtsp://", 7)) {
248        source = new RTSPSource(
249                notify, httpService, url, headers, mUIDValid, mUID);
250    } else if ((!strncasecmp(url, "http://", 7)
251                || !strncasecmp(url, "https://", 8))
252                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
253                    || strstr(url, ".sdp?"))) {
254        source = new RTSPSource(
255                notify, httpService, url, headers, mUIDValid, mUID, true);
256    } else {
257        sp<GenericSource> genericSource =
258                new GenericSource(notify, mUIDValid, mUID);
259        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
260        // The correct flags will be updated in Source::kWhatFlagsChanged
261        // handler when  GenericSource is prepared.
262
263        status_t err = genericSource->setDataSource(httpService, url, headers);
264
265        if (err == OK) {
266            source = genericSource;
267        } else {
268            ALOGE("Failed to set data source!");
269        }
270    }
271    msg->setObject("source", source);
272    msg->post();
273}
274
275void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
276    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
277
278    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
279
280    sp<GenericSource> source =
281            new GenericSource(notify, mUIDValid, mUID);
282
283    status_t err = source->setDataSource(fd, offset, length);
284
285    if (err != OK) {
286        ALOGE("Failed to set data source!");
287        source = NULL;
288    }
289
290    msg->setObject("source", source);
291    msg->post();
292}
293
294void NuPlayer::setDataSourceAsync(const sp<DataSource> &dataSource) {
295    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);
296    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);
297
298    sp<GenericSource> source = new GenericSource(notify, mUIDValid, mUID);
299    status_t err = source->setDataSource(dataSource);
300
301    if (err != OK) {
302        ALOGE("Failed to set data source!");
303        source = NULL;
304    }
305
306    msg->setObject("source", source);
307    msg->post();
308}
309
310void NuPlayer::prepareAsync() {
311    (new AMessage(kWhatPrepare, this))->post();
312}
313
314void NuPlayer::setVideoSurfaceTextureAsync(
315        const sp<IGraphicBufferProducer> &bufferProducer) {
316    sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
317
318    if (bufferProducer == NULL) {
319        msg->setObject("surface", NULL);
320    } else {
321        msg->setObject("surface", new Surface(bufferProducer, true /* controlledByApp */));
322    }
323
324    msg->post();
325}
326
327void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
328    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, this);
329    msg->setObject("sink", sink);
330    msg->post();
331}
332
333void NuPlayer::start() {
334    (new AMessage(kWhatStart, this))->post();
335}
336
337status_t NuPlayer::setPlaybackSettings(const AudioPlaybackRate &rate) {
338    // do some cursory validation of the settings here. audio modes are
339    // only validated when set on the audiosink.
340     if ((rate.mSpeed != 0.f && rate.mSpeed < AUDIO_TIMESTRETCH_SPEED_MIN)
341            || rate.mSpeed > AUDIO_TIMESTRETCH_SPEED_MAX
342            || rate.mPitch < AUDIO_TIMESTRETCH_SPEED_MIN
343            || rate.mPitch > AUDIO_TIMESTRETCH_SPEED_MAX) {
344        return BAD_VALUE;
345    }
346    sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
347    writeToAMessage(msg, rate);
348    sp<AMessage> response;
349    status_t err = msg->postAndAwaitResponse(&response);
350    if (err == OK && response != NULL) {
351        CHECK(response->findInt32("err", &err));
352    }
353    return err;
354}
355
356status_t NuPlayer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
357    sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
358    sp<AMessage> response;
359    status_t err = msg->postAndAwaitResponse(&response);
360    if (err == OK && response != NULL) {
361        CHECK(response->findInt32("err", &err));
362        if (err == OK) {
363            readFromAMessage(response, rate);
364        }
365    }
366    return err;
367}
368
369status_t NuPlayer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
370    sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
371    writeToAMessage(msg, sync, videoFpsHint);
372    sp<AMessage> response;
373    status_t err = msg->postAndAwaitResponse(&response);
374    if (err == OK && response != NULL) {
375        CHECK(response->findInt32("err", &err));
376    }
377    return err;
378}
379
380status_t NuPlayer::getSyncSettings(
381        AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
382    sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
383    sp<AMessage> response;
384    status_t err = msg->postAndAwaitResponse(&response);
385    if (err == OK && response != NULL) {
386        CHECK(response->findInt32("err", &err));
387        if (err == OK) {
388            readFromAMessage(response, sync, videoFps);
389        }
390    }
391    return err;
392}
393
394void NuPlayer::pause() {
395    (new AMessage(kWhatPause, this))->post();
396}
397
398void NuPlayer::resetAsync() {
399    if (mSource != NULL) {
400        // During a reset, the data source might be unresponsive already, we need to
401        // disconnect explicitly so that reads exit promptly.
402        // We can't queue the disconnect request to the looper, as it might be
403        // queued behind a stuck read and never gets processed.
404        // Doing a disconnect outside the looper to allows the pending reads to exit
405        // (either successfully or with error).
406        mSource->disconnect();
407    }
408
409    (new AMessage(kWhatReset, this))->post();
410}
411
412void NuPlayer::seekToAsync(int64_t seekTimeUs, bool needNotify) {
413    sp<AMessage> msg = new AMessage(kWhatSeek, this);
414    msg->setInt64("seekTimeUs", seekTimeUs);
415    msg->setInt32("needNotify", needNotify);
416    msg->post();
417}
418
419
420void NuPlayer::writeTrackInfo(
421        Parcel* reply, const sp<AMessage> format) const {
422    int32_t trackType;
423    CHECK(format->findInt32("type", &trackType));
424
425    AString mime;
426    if (!format->findString("mime", &mime)) {
427        // Java MediaPlayer only uses mimetype for subtitle and timedtext tracks.
428        // If we can't find the mimetype here it means that we wouldn't be needing
429        // the mimetype on the Java end. We still write a placeholder mime to keep the
430        // (de)serialization logic simple.
431        if (trackType == MEDIA_TRACK_TYPE_AUDIO) {
432            mime = "audio/";
433        } else if (trackType == MEDIA_TRACK_TYPE_VIDEO) {
434            mime = "video/";
435        } else {
436            TRESPASS();
437        }
438    }
439
440    AString lang;
441    CHECK(format->findString("language", &lang));
442
443    reply->writeInt32(2); // write something non-zero
444    reply->writeInt32(trackType);
445    reply->writeString16(String16(mime.c_str()));
446    reply->writeString16(String16(lang.c_str()));
447
448    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
449        int32_t isAuto, isDefault, isForced;
450        CHECK(format->findInt32("auto", &isAuto));
451        CHECK(format->findInt32("default", &isDefault));
452        CHECK(format->findInt32("forced", &isForced));
453
454        reply->writeInt32(isAuto);
455        reply->writeInt32(isDefault);
456        reply->writeInt32(isForced);
457    }
458}
459
460void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
461    switch (msg->what()) {
462        case kWhatSetDataSource:
463        {
464            ALOGV("kWhatSetDataSource");
465
466            CHECK(mSource == NULL);
467
468            status_t err = OK;
469            sp<RefBase> obj;
470            CHECK(msg->findObject("source", &obj));
471            if (obj != NULL) {
472                mSource = static_cast<Source *>(obj.get());
473            } else {
474                err = UNKNOWN_ERROR;
475            }
476
477            CHECK(mDriver != NULL);
478            sp<NuPlayerDriver> driver = mDriver.promote();
479            if (driver != NULL) {
480                driver->notifySetDataSourceCompleted(err);
481            }
482            break;
483        }
484
485        case kWhatPrepare:
486        {
487            mSource->prepareAsync();
488            break;
489        }
490
491        case kWhatGetTrackInfo:
492        {
493            sp<AReplyToken> replyID;
494            CHECK(msg->senderAwaitsResponse(&replyID));
495
496            Parcel* reply;
497            CHECK(msg->findPointer("reply", (void**)&reply));
498
499            size_t inbandTracks = 0;
500            if (mSource != NULL) {
501                inbandTracks = mSource->getTrackCount();
502            }
503
504            size_t ccTracks = 0;
505            if (mCCDecoder != NULL) {
506                ccTracks = mCCDecoder->getTrackCount();
507            }
508
509            // total track count
510            reply->writeInt32(inbandTracks + ccTracks);
511
512            // write inband tracks
513            for (size_t i = 0; i < inbandTracks; ++i) {
514                writeTrackInfo(reply, mSource->getTrackInfo(i));
515            }
516
517            // write CC track
518            for (size_t i = 0; i < ccTracks; ++i) {
519                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
520            }
521
522            sp<AMessage> response = new AMessage;
523            response->postReply(replyID);
524            break;
525        }
526
527        case kWhatGetSelectedTrack:
528        {
529            status_t err = INVALID_OPERATION;
530            if (mSource != NULL) {
531                err = OK;
532
533                int32_t type32;
534                CHECK(msg->findInt32("type", (int32_t*)&type32));
535                media_track_type type = (media_track_type)type32;
536                ssize_t selectedTrack = mSource->getSelectedTrack(type);
537
538                Parcel* reply;
539                CHECK(msg->findPointer("reply", (void**)&reply));
540                reply->writeInt32(selectedTrack);
541            }
542
543            sp<AMessage> response = new AMessage;
544            response->setInt32("err", err);
545
546            sp<AReplyToken> replyID;
547            CHECK(msg->senderAwaitsResponse(&replyID));
548            response->postReply(replyID);
549            break;
550        }
551
552        case kWhatSelectTrack:
553        {
554            sp<AReplyToken> replyID;
555            CHECK(msg->senderAwaitsResponse(&replyID));
556
557            size_t trackIndex;
558            int32_t select;
559            int64_t timeUs;
560            CHECK(msg->findSize("trackIndex", &trackIndex));
561            CHECK(msg->findInt32("select", &select));
562            CHECK(msg->findInt64("timeUs", &timeUs));
563
564            status_t err = INVALID_OPERATION;
565
566            size_t inbandTracks = 0;
567            if (mSource != NULL) {
568                inbandTracks = mSource->getTrackCount();
569            }
570            size_t ccTracks = 0;
571            if (mCCDecoder != NULL) {
572                ccTracks = mCCDecoder->getTrackCount();
573            }
574
575            if (trackIndex < inbandTracks) {
576                err = mSource->selectTrack(trackIndex, select, timeUs);
577
578                if (!select && err == OK) {
579                    int32_t type;
580                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
581                    if (info != NULL
582                            && info->findInt32("type", &type)
583                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
584                        ++mTimedTextGeneration;
585                    }
586                }
587            } else {
588                trackIndex -= inbandTracks;
589
590                if (trackIndex < ccTracks) {
591                    err = mCCDecoder->selectTrack(trackIndex, select);
592                }
593            }
594
595            sp<AMessage> response = new AMessage;
596            response->setInt32("err", err);
597
598            response->postReply(replyID);
599            break;
600        }
601
602        case kWhatPollDuration:
603        {
604            int32_t generation;
605            CHECK(msg->findInt32("generation", &generation));
606
607            if (generation != mPollDurationGeneration) {
608                // stale
609                break;
610            }
611
612            int64_t durationUs;
613            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
614                sp<NuPlayerDriver> driver = mDriver.promote();
615                if (driver != NULL) {
616                    driver->notifyDuration(durationUs);
617                }
618            }
619
620            msg->post(1000000ll);  // poll again in a second.
621            break;
622        }
623
624        case kWhatSetVideoSurface:
625        {
626            ALOGV("kWhatSetVideoSurface");
627
628            sp<RefBase> obj;
629            CHECK(msg->findObject("surface", &obj));
630            sp<Surface> surface = static_cast<Surface *>(obj.get());
631            if (mSource == NULL || mSource->getFormat(false /* audio */) == NULL) {
632                performSetSurface(surface);
633                break;
634            }
635
636            mDeferredActions.push_back(
637                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
638                                           FLUSH_CMD_SHUTDOWN /* video */));
639
640            mDeferredActions.push_back(new SetSurfaceAction(surface));
641
642            if (obj != NULL || mAudioDecoder != NULL) {
643                if (mStarted) {
644                    // Issue a seek to refresh the video screen only if started otherwise
645                    // the extractor may not yet be started and will assert.
646                    // If the video decoder is not set (perhaps audio only in this case)
647                    // do not perform a seek as it is not needed.
648                    int64_t currentPositionUs = 0;
649                    if (getCurrentPosition(&currentPositionUs) == OK) {
650                        mDeferredActions.push_back(
651                                new SeekAction(currentPositionUs));
652                    }
653                }
654
655                // If there is a new surface texture, instantiate decoders
656                // again if possible.
657                mDeferredActions.push_back(
658                        new SimpleAction(&NuPlayer::performScanSources));
659            }
660
661            // After a flush without shutdown, decoder is paused.
662            // Don't resume it until source seek is done, otherwise it could
663            // start pulling stale data too soon.
664            mDeferredActions.push_back(
665                    new ResumeDecoderAction(false /* needNotify */));
666
667            processDeferredActions();
668            break;
669        }
670
671        case kWhatSetAudioSink:
672        {
673            ALOGV("kWhatSetAudioSink");
674
675            sp<RefBase> obj;
676            CHECK(msg->findObject("sink", &obj));
677
678            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
679            break;
680        }
681
682        case kWhatStart:
683        {
684            ALOGV("kWhatStart");
685            if (mStarted) {
686                // do not resume yet if the source is still buffering
687                if (!mPausedForBuffering) {
688                    onResume();
689                }
690            } else {
691                onStart();
692            }
693            mPausedByClient = false;
694            break;
695        }
696
697        case kWhatConfigPlayback:
698        {
699            sp<AReplyToken> replyID;
700            CHECK(msg->senderAwaitsResponse(&replyID));
701            AudioPlaybackRate rate /* sanitized */;
702            readFromAMessage(msg, &rate);
703            status_t err = OK;
704            if (mRenderer != NULL) {
705                err = mRenderer->setPlaybackSettings(rate);
706            }
707            if (err == OK) {
708                if (rate.mSpeed == 0.f) {
709                    onPause();
710                    // save all other settings (using non-paused speed)
711                    // so we can restore them on start
712                    AudioPlaybackRate newRate = rate;
713                    newRate.mSpeed = mPlaybackSettings.mSpeed;
714                    mPlaybackSettings = newRate;
715                } else { /* rate.mSpeed != 0.f */
716                    onResume();
717                    mPlaybackSettings = rate;
718                }
719            }
720
721            if (mVideoDecoder != NULL) {
722                float rate = getFrameRate();
723                if (rate > 0) {
724                    sp<AMessage> params = new AMessage();
725                    params->setFloat("operating-rate", rate * mPlaybackSettings.mSpeed);
726                    mVideoDecoder->setParameters(params);
727                }
728            }
729
730            sp<AMessage> response = new AMessage;
731            response->setInt32("err", err);
732            response->postReply(replyID);
733            break;
734        }
735
736        case kWhatGetPlaybackSettings:
737        {
738            sp<AReplyToken> replyID;
739            CHECK(msg->senderAwaitsResponse(&replyID));
740            AudioPlaybackRate rate = mPlaybackSettings;
741            status_t err = OK;
742            if (mRenderer != NULL) {
743                err = mRenderer->getPlaybackSettings(&rate);
744            }
745            if (err == OK) {
746                // get playback settings used by renderer, as it may be
747                // slightly off due to audiosink not taking small changes.
748                mPlaybackSettings = rate;
749                if (mPaused) {
750                    rate.mSpeed = 0.f;
751                }
752            }
753            sp<AMessage> response = new AMessage;
754            if (err == OK) {
755                writeToAMessage(response, rate);
756            }
757            response->setInt32("err", err);
758            response->postReply(replyID);
759            break;
760        }
761
762        case kWhatConfigSync:
763        {
764            sp<AReplyToken> replyID;
765            CHECK(msg->senderAwaitsResponse(&replyID));
766
767            ALOGV("kWhatConfigSync");
768            AVSyncSettings sync;
769            float videoFpsHint;
770            readFromAMessage(msg, &sync, &videoFpsHint);
771            status_t err = OK;
772            if (mRenderer != NULL) {
773                err = mRenderer->setSyncSettings(sync, videoFpsHint);
774            }
775            if (err == OK) {
776                mSyncSettings = sync;
777                mVideoFpsHint = videoFpsHint;
778            }
779            sp<AMessage> response = new AMessage;
780            response->setInt32("err", err);
781            response->postReply(replyID);
782            break;
783        }
784
785        case kWhatGetSyncSettings:
786        {
787            sp<AReplyToken> replyID;
788            CHECK(msg->senderAwaitsResponse(&replyID));
789            AVSyncSettings sync = mSyncSettings;
790            float videoFps = mVideoFpsHint;
791            status_t err = OK;
792            if (mRenderer != NULL) {
793                err = mRenderer->getSyncSettings(&sync, &videoFps);
794                if (err == OK) {
795                    mSyncSettings = sync;
796                    mVideoFpsHint = videoFps;
797                }
798            }
799            sp<AMessage> response = new AMessage;
800            if (err == OK) {
801                writeToAMessage(response, sync, videoFps);
802            }
803            response->setInt32("err", err);
804            response->postReply(replyID);
805            break;
806        }
807
808        case kWhatScanSources:
809        {
810            int32_t generation;
811            CHECK(msg->findInt32("generation", &generation));
812            if (generation != mScanSourcesGeneration) {
813                // Drop obsolete msg.
814                break;
815            }
816
817            mScanSourcesPending = false;
818
819            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
820                 mAudioDecoder != NULL, mVideoDecoder != NULL);
821
822            bool mHadAnySourcesBefore =
823                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
824
825            // initialize video before audio because successful initialization of
826            // video may change deep buffer mode of audio.
827            if (mSurface != NULL) {
828                instantiateDecoder(false, &mVideoDecoder);
829            }
830
831            // Don't try to re-open audio sink if there's an existing decoder.
832            if (mAudioSink != NULL && mAudioDecoder == NULL) {
833                instantiateDecoder(true, &mAudioDecoder);
834            }
835
836            if (!mHadAnySourcesBefore
837                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
838                // This is the first time we've found anything playable.
839
840                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
841                    schedulePollDuration();
842                }
843            }
844
845            status_t err;
846            if ((err = mSource->feedMoreTSData()) != OK) {
847                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
848                    // We're not currently decoding anything (no audio or
849                    // video tracks found) and we just ran out of input data.
850
851                    if (err == ERROR_END_OF_STREAM) {
852                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
853                    } else {
854                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
855                    }
856                }
857                break;
858            }
859
860            if ((mAudioDecoder == NULL && mAudioSink != NULL)
861                    || (mVideoDecoder == NULL && mSurface != NULL)) {
862                msg->post(100000ll);
863                mScanSourcesPending = true;
864            }
865            break;
866        }
867
868        case kWhatVideoNotify:
869        case kWhatAudioNotify:
870        {
871            bool audio = msg->what() == kWhatAudioNotify;
872
873            int32_t currentDecoderGeneration =
874                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
875            int32_t requesterGeneration = currentDecoderGeneration - 1;
876            CHECK(msg->findInt32("generation", &requesterGeneration));
877
878            if (requesterGeneration != currentDecoderGeneration) {
879                ALOGV("got message from old %s decoder, generation(%d:%d)",
880                        audio ? "audio" : "video", requesterGeneration,
881                        currentDecoderGeneration);
882                sp<AMessage> reply;
883                if (!(msg->findMessage("reply", &reply))) {
884                    return;
885                }
886
887                reply->setInt32("err", INFO_DISCONTINUITY);
888                reply->post();
889                return;
890            }
891
892            int32_t what;
893            CHECK(msg->findInt32("what", &what));
894
895            if (what == DecoderBase::kWhatInputDiscontinuity) {
896                int32_t formatChange;
897                CHECK(msg->findInt32("formatChange", &formatChange));
898
899                ALOGV("%s discontinuity: formatChange %d",
900                        audio ? "audio" : "video", formatChange);
901
902                if (formatChange) {
903                    mDeferredActions.push_back(
904                            new FlushDecoderAction(
905                                audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
906                                audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN));
907                }
908
909                mDeferredActions.push_back(
910                        new SimpleAction(
911                                &NuPlayer::performScanSources));
912
913                processDeferredActions();
914            } else if (what == DecoderBase::kWhatEOS) {
915                int32_t err;
916                CHECK(msg->findInt32("err", &err));
917
918                if (err == ERROR_END_OF_STREAM) {
919                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
920                } else {
921                    ALOGV("got %s decoder EOS w/ error %d",
922                         audio ? "audio" : "video",
923                         err);
924                }
925
926                mRenderer->queueEOS(audio, err);
927            } else if (what == DecoderBase::kWhatFlushCompleted) {
928                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
929
930                handleFlushComplete(audio, true /* isDecoder */);
931                finishFlushIfPossible();
932            } else if (what == DecoderBase::kWhatVideoSizeChanged) {
933                sp<AMessage> format;
934                CHECK(msg->findMessage("format", &format));
935
936                sp<AMessage> inputFormat =
937                        mSource->getFormat(false /* audio */);
938
939                updateVideoSize(inputFormat, format);
940            } else if (what == DecoderBase::kWhatShutdownCompleted) {
941                ALOGV("%s shutdown completed", audio ? "audio" : "video");
942                if (audio) {
943                    mAudioDecoder.clear();
944                    ++mAudioDecoderGeneration;
945
946                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
947                    mFlushingAudio = SHUT_DOWN;
948                } else {
949                    mVideoDecoder.clear();
950                    ++mVideoDecoderGeneration;
951
952                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
953                    mFlushingVideo = SHUT_DOWN;
954                }
955
956                finishFlushIfPossible();
957            } else if (what == DecoderBase::kWhatResumeCompleted) {
958                finishResume();
959            } else if (what == DecoderBase::kWhatError) {
960                status_t err;
961                if (!msg->findInt32("err", &err) || err == OK) {
962                    err = UNKNOWN_ERROR;
963                }
964
965                // Decoder errors can be due to Source (e.g. from streaming),
966                // or from decoding corrupted bitstreams, or from other decoder
967                // MediaCodec operations (e.g. from an ongoing reset or seek).
968                // They may also be due to openAudioSink failure at
969                // decoder start or after a format change.
970                //
971                // We try to gracefully shut down the affected decoder if possible,
972                // rather than trying to force the shutdown with something
973                // similar to performReset(). This method can lead to a hang
974                // if MediaCodec functions block after an error, but they should
975                // typically return INVALID_OPERATION instead of blocking.
976
977                FlushStatus *flushing = audio ? &mFlushingAudio : &mFlushingVideo;
978                ALOGE("received error(%#x) from %s decoder, flushing(%d), now shutting down",
979                        err, audio ? "audio" : "video", *flushing);
980
981                switch (*flushing) {
982                    case NONE:
983                        mDeferredActions.push_back(
984                                new FlushDecoderAction(
985                                    audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
986                                    audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN));
987                        processDeferredActions();
988                        break;
989                    case FLUSHING_DECODER:
990                        *flushing = FLUSHING_DECODER_SHUTDOWN; // initiate shutdown after flush.
991                        break; // Wait for flush to complete.
992                    case FLUSHING_DECODER_SHUTDOWN:
993                        break; // Wait for flush to complete.
994                    case SHUTTING_DOWN_DECODER:
995                        break; // Wait for shutdown to complete.
996                    case FLUSHED:
997                        // Widevine source reads must stop before releasing the video decoder.
998                        if (!audio && mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
999                            mSource->stop();
1000                            mSourceStarted = false;
1001                        }
1002                        getDecoder(audio)->initiateShutdown(); // In the middle of a seek.
1003                        *flushing = SHUTTING_DOWN_DECODER;     // Shut down.
1004                        break;
1005                    case SHUT_DOWN:
1006                        finishFlushIfPossible();  // Should not occur.
1007                        break;                    // Finish anyways.
1008                }
1009                notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
1010            } else {
1011                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
1012                      what,
1013                      what >> 24,
1014                      (what >> 16) & 0xff,
1015                      (what >> 8) & 0xff,
1016                      what & 0xff);
1017            }
1018
1019            break;
1020        }
1021
1022        case kWhatRendererNotify:
1023        {
1024            int32_t requesterGeneration = mRendererGeneration - 1;
1025            CHECK(msg->findInt32("generation", &requesterGeneration));
1026            if (requesterGeneration != mRendererGeneration) {
1027                ALOGV("got message from old renderer, generation(%d:%d)",
1028                        requesterGeneration, mRendererGeneration);
1029                return;
1030            }
1031
1032            int32_t what;
1033            CHECK(msg->findInt32("what", &what));
1034
1035            if (what == Renderer::kWhatEOS) {
1036                int32_t audio;
1037                CHECK(msg->findInt32("audio", &audio));
1038
1039                int32_t finalResult;
1040                CHECK(msg->findInt32("finalResult", &finalResult));
1041
1042                if (audio) {
1043                    mAudioEOS = true;
1044                } else {
1045                    mVideoEOS = true;
1046                }
1047
1048                if (finalResult == ERROR_END_OF_STREAM) {
1049                    ALOGV("reached %s EOS", audio ? "audio" : "video");
1050                } else {
1051                    ALOGE("%s track encountered an error (%d)",
1052                         audio ? "audio" : "video", finalResult);
1053
1054                    notifyListener(
1055                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
1056                }
1057
1058                if ((mAudioEOS || mAudioDecoder == NULL)
1059                        && (mVideoEOS || mVideoDecoder == NULL)) {
1060                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
1061                }
1062            } else if (what == Renderer::kWhatFlushComplete) {
1063                int32_t audio;
1064                CHECK(msg->findInt32("audio", &audio));
1065
1066                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
1067                handleFlushComplete(audio, false /* isDecoder */);
1068                finishFlushIfPossible();
1069            } else if (what == Renderer::kWhatVideoRenderingStart) {
1070                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
1071            } else if (what == Renderer::kWhatMediaRenderingStart) {
1072                ALOGV("media rendering started");
1073                notifyListener(MEDIA_STARTED, 0, 0);
1074            } else if (what == Renderer::kWhatAudioTearDown) {
1075                int64_t positionUs;
1076                CHECK(msg->findInt64("positionUs", &positionUs));
1077                int32_t reason;
1078                CHECK(msg->findInt32("reason", &reason));
1079                ALOGV("Tear down audio with reason %d.", reason);
1080                closeAudioSink();
1081                mAudioDecoder.clear();
1082                ++mAudioDecoderGeneration;
1083                mRenderer->flush(
1084                        true /* audio */, false /* notifyComplete */);
1085                if (mVideoDecoder != NULL) {
1086                    mRenderer->flush(
1087                            false /* audio */, false /* notifyComplete */);
1088                }
1089
1090                performSeek(positionUs);
1091
1092                if (reason == Renderer::kDueToError) {
1093                    instantiateDecoder(true /* audio */, &mAudioDecoder);
1094                }
1095            }
1096            break;
1097        }
1098
1099        case kWhatMoreDataQueued:
1100        {
1101            break;
1102        }
1103
1104        case kWhatReset:
1105        {
1106            ALOGV("kWhatReset");
1107
1108            mDeferredActions.push_back(
1109                    new FlushDecoderAction(
1110                        FLUSH_CMD_SHUTDOWN /* audio */,
1111                        FLUSH_CMD_SHUTDOWN /* video */));
1112
1113            mDeferredActions.push_back(
1114                    new SimpleAction(&NuPlayer::performReset));
1115
1116            processDeferredActions();
1117            break;
1118        }
1119
1120        case kWhatSeek:
1121        {
1122            int64_t seekTimeUs;
1123            int32_t needNotify;
1124            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
1125            CHECK(msg->findInt32("needNotify", &needNotify));
1126
1127            ALOGV("kWhatSeek seekTimeUs=%lld us, needNotify=%d",
1128                    (long long)seekTimeUs, needNotify);
1129
1130            if (!mStarted) {
1131                // Seek before the player is started. In order to preview video,
1132                // need to start the player and pause it. This branch is called
1133                // only once if needed. After the player is started, any seek
1134                // operation will go through normal path.
1135                // Audio-only cases are handled separately.
1136                onStart(seekTimeUs);
1137                if (mStarted) {
1138                    onPause();
1139                    mPausedByClient = true;
1140                }
1141                if (needNotify) {
1142                    notifyDriverSeekComplete();
1143                }
1144                break;
1145            }
1146
1147            mDeferredActions.push_back(
1148                    new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */,
1149                                           FLUSH_CMD_FLUSH /* video */));
1150
1151            mDeferredActions.push_back(
1152                    new SeekAction(seekTimeUs));
1153
1154            // After a flush without shutdown, decoder is paused.
1155            // Don't resume it until source seek is done, otherwise it could
1156            // start pulling stale data too soon.
1157            mDeferredActions.push_back(
1158                    new ResumeDecoderAction(needNotify));
1159
1160            processDeferredActions();
1161            break;
1162        }
1163
1164        case kWhatPause:
1165        {
1166            onPause();
1167            mPausedByClient = true;
1168            break;
1169        }
1170
1171        case kWhatSourceNotify:
1172        {
1173            onSourceNotify(msg);
1174            break;
1175        }
1176
1177        case kWhatClosedCaptionNotify:
1178        {
1179            onClosedCaptionNotify(msg);
1180            break;
1181        }
1182
1183        default:
1184            TRESPASS();
1185            break;
1186    }
1187}
1188
1189void NuPlayer::onResume() {
1190    if (!mPaused) {
1191        return;
1192    }
1193    mPaused = false;
1194    if (mSource != NULL) {
1195        mSource->resume();
1196    } else {
1197        ALOGW("resume called when source is gone or not set");
1198    }
1199    // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if
1200    // needed.
1201    if (audioDecoderStillNeeded() && mAudioDecoder == NULL) {
1202        instantiateDecoder(true /* audio */, &mAudioDecoder);
1203    }
1204    if (mRenderer != NULL) {
1205        mRenderer->resume();
1206    } else {
1207        ALOGW("resume called when renderer is gone or not set");
1208    }
1209}
1210
1211status_t NuPlayer::onInstantiateSecureDecoders() {
1212    status_t err;
1213    if (!(mSourceFlags & Source::FLAG_SECURE)) {
1214        return BAD_TYPE;
1215    }
1216
1217    if (mRenderer != NULL) {
1218        ALOGE("renderer should not be set when instantiating secure decoders");
1219        return UNKNOWN_ERROR;
1220    }
1221
1222    // TRICKY: We rely on mRenderer being null, so that decoder does not start requesting
1223    // data on instantiation.
1224    if (mSurface != NULL) {
1225        err = instantiateDecoder(false, &mVideoDecoder);
1226        if (err != OK) {
1227            return err;
1228        }
1229    }
1230
1231    if (mAudioSink != NULL) {
1232        err = instantiateDecoder(true, &mAudioDecoder);
1233        if (err != OK) {
1234            return err;
1235        }
1236    }
1237    return OK;
1238}
1239
1240void NuPlayer::onStart(int64_t startPositionUs) {
1241    if (!mSourceStarted) {
1242        mSourceStarted = true;
1243        mSource->start();
1244    }
1245    if (startPositionUs > 0) {
1246        performSeek(startPositionUs);
1247        if (mSource->getFormat(false /* audio */) == NULL) {
1248            return;
1249        }
1250    }
1251
1252    mOffloadAudio = false;
1253    mAudioEOS = false;
1254    mVideoEOS = false;
1255    mStarted = true;
1256
1257    uint32_t flags = 0;
1258
1259    if (mSource->isRealTime()) {
1260        flags |= Renderer::FLAG_REAL_TIME;
1261    }
1262
1263    sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
1264    audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
1265    if (mAudioSink != NULL) {
1266        streamType = mAudioSink->getAudioStreamType();
1267    }
1268
1269    sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
1270
1271    mOffloadAudio =
1272        canOffloadStream(audioMeta, (videoFormat != NULL), mSource->isStreaming(), streamType);
1273    if (mOffloadAudio) {
1274        flags |= Renderer::FLAG_OFFLOAD_AUDIO;
1275    }
1276
1277    sp<AMessage> notify = new AMessage(kWhatRendererNotify, this);
1278    ++mRendererGeneration;
1279    notify->setInt32("generation", mRendererGeneration);
1280    mRenderer = new Renderer(mAudioSink, notify, flags);
1281    mRendererLooper = new ALooper;
1282    mRendererLooper->setName("NuPlayerRenderer");
1283    mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
1284    mRendererLooper->registerHandler(mRenderer);
1285
1286    status_t err = mRenderer->setPlaybackSettings(mPlaybackSettings);
1287    if (err != OK) {
1288        mSource->stop();
1289        mSourceStarted = false;
1290        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
1291        return;
1292    }
1293
1294    float rate = getFrameRate();
1295    if (rate > 0) {
1296        mRenderer->setVideoFrameRate(rate);
1297    }
1298
1299    if (mVideoDecoder != NULL) {
1300        mVideoDecoder->setRenderer(mRenderer);
1301    }
1302    if (mAudioDecoder != NULL) {
1303        mAudioDecoder->setRenderer(mRenderer);
1304    }
1305
1306    postScanSources();
1307}
1308
1309void NuPlayer::onPause() {
1310    if (mPaused) {
1311        return;
1312    }
1313    mPaused = true;
1314    if (mSource != NULL) {
1315        mSource->pause();
1316    } else {
1317        ALOGW("pause called when source is gone or not set");
1318    }
1319    if (mRenderer != NULL) {
1320        mRenderer->pause();
1321    } else {
1322        ALOGW("pause called when renderer is gone or not set");
1323    }
1324}
1325
1326bool NuPlayer::audioDecoderStillNeeded() {
1327    // Audio decoder is no longer needed if it's in shut/shutting down status.
1328    return ((mFlushingAudio != SHUT_DOWN) && (mFlushingAudio != SHUTTING_DOWN_DECODER));
1329}
1330
1331void NuPlayer::handleFlushComplete(bool audio, bool isDecoder) {
1332    // We wait for both the decoder flush and the renderer flush to complete
1333    // before entering either the FLUSHED or the SHUTTING_DOWN_DECODER state.
1334
1335    mFlushComplete[audio][isDecoder] = true;
1336    if (!mFlushComplete[audio][!isDecoder]) {
1337        return;
1338    }
1339
1340    FlushStatus *state = audio ? &mFlushingAudio : &mFlushingVideo;
1341    switch (*state) {
1342        case FLUSHING_DECODER:
1343        {
1344            *state = FLUSHED;
1345            break;
1346        }
1347
1348        case FLUSHING_DECODER_SHUTDOWN:
1349        {
1350            *state = SHUTTING_DOWN_DECODER;
1351
1352            ALOGV("initiating %s decoder shutdown", audio ? "audio" : "video");
1353            if (!audio) {
1354                // Widevine source reads must stop before releasing the video decoder.
1355                if (mSource != NULL && mSourceFlags & Source::FLAG_SECURE) {
1356                    mSource->stop();
1357                    mSourceStarted = false;
1358                }
1359            }
1360            getDecoder(audio)->initiateShutdown();
1361            break;
1362        }
1363
1364        default:
1365            // decoder flush completes only occur in a flushing state.
1366            LOG_ALWAYS_FATAL_IF(isDecoder, "decoder flush in invalid state %d", *state);
1367            break;
1368    }
1369}
1370
1371void NuPlayer::finishFlushIfPossible() {
1372    if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED
1373            && mFlushingAudio != SHUT_DOWN) {
1374        return;
1375    }
1376
1377    if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED
1378            && mFlushingVideo != SHUT_DOWN) {
1379        return;
1380    }
1381
1382    ALOGV("both audio and video are flushed now.");
1383
1384    mFlushingAudio = NONE;
1385    mFlushingVideo = NONE;
1386
1387    clearFlushComplete();
1388
1389    processDeferredActions();
1390}
1391
1392void NuPlayer::postScanSources() {
1393    if (mScanSourcesPending) {
1394        return;
1395    }
1396
1397    sp<AMessage> msg = new AMessage(kWhatScanSources, this);
1398    msg->setInt32("generation", mScanSourcesGeneration);
1399    msg->post();
1400
1401    mScanSourcesPending = true;
1402}
1403
1404void NuPlayer::tryOpenAudioSinkForOffload(const sp<AMessage> &format, bool hasVideo) {
1405    // Note: This is called early in NuPlayer to determine whether offloading
1406    // is possible; otherwise the decoders call the renderer openAudioSink directly.
1407
1408    status_t err = mRenderer->openAudioSink(
1409            format, true /* offloadOnly */, hasVideo, AUDIO_OUTPUT_FLAG_NONE, &mOffloadAudio);
1410    if (err != OK) {
1411        // Any failure we turn off mOffloadAudio.
1412        mOffloadAudio = false;
1413    } else if (mOffloadAudio) {
1414        sp<MetaData> audioMeta =
1415                mSource->getFormatMeta(true /* audio */);
1416        sendMetaDataToHal(mAudioSink, audioMeta);
1417    }
1418}
1419
1420void NuPlayer::closeAudioSink() {
1421    mRenderer->closeAudioSink();
1422}
1423
1424void NuPlayer::determineAudioModeChange() {
1425    if (mSource == NULL || mAudioSink == NULL) {
1426        return;
1427    }
1428
1429    if (mRenderer == NULL) {
1430        ALOGW("No renderer can be used to determine audio mode. Use non-offload for safety.");
1431        mOffloadAudio = false;
1432        return;
1433    }
1434
1435    sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
1436    sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
1437    audio_stream_type_t streamType = mAudioSink->getAudioStreamType();
1438    const bool hasVideo = (videoFormat != NULL);
1439    const bool canOffload = canOffloadStream(
1440            audioMeta, hasVideo, mSource->isStreaming(), streamType);
1441    if (canOffload) {
1442        if (!mOffloadAudio) {
1443            mRenderer->signalEnableOffloadAudio();
1444        }
1445        // open audio sink early under offload mode.
1446        sp<AMessage> format = mSource->getFormat(true /*audio*/);
1447        tryOpenAudioSinkForOffload(format, hasVideo);
1448    } else {
1449        mRenderer->signalDisableOffloadAudio();
1450        mOffloadAudio = false;
1451    }
1452}
1453
1454status_t NuPlayer::instantiateDecoder(bool audio, sp<DecoderBase> *decoder) {
1455    if (*decoder != NULL) {
1456        return OK;
1457    }
1458
1459    sp<AMessage> format = mSource->getFormat(audio);
1460
1461    if (format == NULL) {
1462        return -EWOULDBLOCK;
1463    }
1464
1465    format->setInt32("priority", 0 /* realtime */);
1466
1467    if (!audio) {
1468        AString mime;
1469        CHECK(format->findString("mime", &mime));
1470
1471        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, this);
1472        if (mCCDecoder == NULL) {
1473            mCCDecoder = new CCDecoder(ccNotify);
1474        }
1475
1476        if (mSourceFlags & Source::FLAG_SECURE) {
1477            format->setInt32("secure", true);
1478        }
1479
1480        if (mSourceFlags & Source::FLAG_PROTECTED) {
1481            format->setInt32("protected", true);
1482        }
1483
1484        float rate = getFrameRate();
1485        if (rate > 0) {
1486            format->setFloat("operating-rate", rate * mPlaybackSettings.mSpeed);
1487        }
1488    }
1489
1490    if (audio) {
1491        sp<AMessage> notify = new AMessage(kWhatAudioNotify, this);
1492        ++mAudioDecoderGeneration;
1493        notify->setInt32("generation", mAudioDecoderGeneration);
1494
1495        determineAudioModeChange();
1496        if (mOffloadAudio) {
1497            const bool hasVideo = (mSource->getFormat(false /*audio */) != NULL);
1498            format->setInt32("has-video", hasVideo);
1499            *decoder = new DecoderPassThrough(notify, mSource, mRenderer);
1500        } else {
1501            *decoder = new Decoder(notify, mSource, mRenderer);
1502        }
1503    } else {
1504        sp<AMessage> notify = new AMessage(kWhatVideoNotify, this);
1505        ++mVideoDecoderGeneration;
1506        notify->setInt32("generation", mVideoDecoderGeneration);
1507
1508        *decoder = new Decoder(
1509                notify, mSource, mRenderer, mSurface, mCCDecoder);
1510
1511        // enable FRC if high-quality AV sync is requested, even if not
1512        // directly queuing to display, as this will even improve textureview
1513        // playback.
1514        {
1515            char value[PROPERTY_VALUE_MAX];
1516            if (property_get("persist.sys.media.avsync", value, NULL) &&
1517                    (!strcmp("1", value) || !strcasecmp("true", value))) {
1518                format->setInt32("auto-frc", 1);
1519            }
1520        }
1521    }
1522    (*decoder)->init();
1523    (*decoder)->configure(format);
1524
1525    // allocate buffers to decrypt widevine source buffers
1526    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1527        Vector<sp<ABuffer> > inputBufs;
1528        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1529
1530        Vector<MediaBuffer *> mediaBufs;
1531        for (size_t i = 0; i < inputBufs.size(); i++) {
1532            const sp<ABuffer> &buffer = inputBufs[i];
1533            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1534            mediaBufs.push(mbuf);
1535        }
1536
1537        status_t err = mSource->setBuffers(audio, mediaBufs);
1538        if (err != OK) {
1539            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1540                mediaBufs[i]->release();
1541            }
1542            mediaBufs.clear();
1543            ALOGE("Secure source didn't support secure mediaBufs.");
1544            return err;
1545        }
1546    }
1547    return OK;
1548}
1549
1550void NuPlayer::updateVideoSize(
1551        const sp<AMessage> &inputFormat,
1552        const sp<AMessage> &outputFormat) {
1553    if (inputFormat == NULL) {
1554        ALOGW("Unknown video size, reporting 0x0!");
1555        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1556        return;
1557    }
1558
1559    int32_t displayWidth, displayHeight;
1560    if (outputFormat != NULL) {
1561        int32_t width, height;
1562        CHECK(outputFormat->findInt32("width", &width));
1563        CHECK(outputFormat->findInt32("height", &height));
1564
1565        int32_t cropLeft, cropTop, cropRight, cropBottom;
1566        CHECK(outputFormat->findRect(
1567                    "crop",
1568                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1569
1570        displayWidth = cropRight - cropLeft + 1;
1571        displayHeight = cropBottom - cropTop + 1;
1572
1573        ALOGV("Video output format changed to %d x %d "
1574             "(crop: %d x %d @ (%d, %d))",
1575             width, height,
1576             displayWidth,
1577             displayHeight,
1578             cropLeft, cropTop);
1579    } else {
1580        CHECK(inputFormat->findInt32("width", &displayWidth));
1581        CHECK(inputFormat->findInt32("height", &displayHeight));
1582
1583        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1584    }
1585
1586    // Take into account sample aspect ratio if necessary:
1587    int32_t sarWidth, sarHeight;
1588    if (inputFormat->findInt32("sar-width", &sarWidth)
1589            && inputFormat->findInt32("sar-height", &sarHeight)) {
1590        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1591
1592        displayWidth = (displayWidth * sarWidth) / sarHeight;
1593
1594        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1595    }
1596
1597    int32_t rotationDegrees;
1598    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1599        rotationDegrees = 0;
1600    }
1601
1602    if (rotationDegrees == 90 || rotationDegrees == 270) {
1603        int32_t tmp = displayWidth;
1604        displayWidth = displayHeight;
1605        displayHeight = tmp;
1606    }
1607
1608    notifyListener(
1609            MEDIA_SET_VIDEO_SIZE,
1610            displayWidth,
1611            displayHeight);
1612}
1613
1614void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1615    if (mDriver == NULL) {
1616        return;
1617    }
1618
1619    sp<NuPlayerDriver> driver = mDriver.promote();
1620
1621    if (driver == NULL) {
1622        return;
1623    }
1624
1625    driver->notifyListener(msg, ext1, ext2, in);
1626}
1627
1628void NuPlayer::flushDecoder(bool audio, bool needShutdown) {
1629    ALOGV("[%s] flushDecoder needShutdown=%d",
1630          audio ? "audio" : "video", needShutdown);
1631
1632    const sp<DecoderBase> &decoder = getDecoder(audio);
1633    if (decoder == NULL) {
1634        ALOGI("flushDecoder %s without decoder present",
1635             audio ? "audio" : "video");
1636        return;
1637    }
1638
1639    // Make sure we don't continue to scan sources until we finish flushing.
1640    ++mScanSourcesGeneration;
1641    if (mScanSourcesPending) {
1642        mDeferredActions.push_back(
1643                new SimpleAction(&NuPlayer::performScanSources));
1644        mScanSourcesPending = false;
1645    }
1646
1647    decoder->signalFlush();
1648
1649    FlushStatus newStatus =
1650        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1651
1652    mFlushComplete[audio][false /* isDecoder */] = (mRenderer == NULL);
1653    mFlushComplete[audio][true /* isDecoder */] = false;
1654    if (audio) {
1655        ALOGE_IF(mFlushingAudio != NONE,
1656                "audio flushDecoder() is called in state %d", mFlushingAudio);
1657        mFlushingAudio = newStatus;
1658    } else {
1659        ALOGE_IF(mFlushingVideo != NONE,
1660                "video flushDecoder() is called in state %d", mFlushingVideo);
1661        mFlushingVideo = newStatus;
1662    }
1663}
1664
1665void NuPlayer::queueDecoderShutdown(
1666        bool audio, bool video, const sp<AMessage> &reply) {
1667    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1668
1669    mDeferredActions.push_back(
1670            new FlushDecoderAction(
1671                audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE,
1672                video ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE));
1673
1674    mDeferredActions.push_back(
1675            new SimpleAction(&NuPlayer::performScanSources));
1676
1677    mDeferredActions.push_back(new PostMessageAction(reply));
1678
1679    processDeferredActions();
1680}
1681
1682status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1683    mVideoScalingMode = mode;
1684    if (mSurface != NULL) {
1685        status_t ret = native_window_set_scaling_mode(mSurface.get(), mVideoScalingMode);
1686        if (ret != OK) {
1687            ALOGE("Failed to set scaling mode (%d): %s",
1688                -ret, strerror(-ret));
1689            return ret;
1690        }
1691    }
1692    return OK;
1693}
1694
1695status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1696    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, this);
1697    msg->setPointer("reply", reply);
1698
1699    sp<AMessage> response;
1700    status_t err = msg->postAndAwaitResponse(&response);
1701    return err;
1702}
1703
1704status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1705    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, this);
1706    msg->setPointer("reply", reply);
1707    msg->setInt32("type", type);
1708
1709    sp<AMessage> response;
1710    status_t err = msg->postAndAwaitResponse(&response);
1711    if (err == OK && response != NULL) {
1712        CHECK(response->findInt32("err", &err));
1713    }
1714    return err;
1715}
1716
1717status_t NuPlayer::selectTrack(size_t trackIndex, bool select, int64_t timeUs) {
1718    sp<AMessage> msg = new AMessage(kWhatSelectTrack, this);
1719    msg->setSize("trackIndex", trackIndex);
1720    msg->setInt32("select", select);
1721    msg->setInt64("timeUs", timeUs);
1722
1723    sp<AMessage> response;
1724    status_t err = msg->postAndAwaitResponse(&response);
1725
1726    if (err != OK) {
1727        return err;
1728    }
1729
1730    if (!response->findInt32("err", &err)) {
1731        err = OK;
1732    }
1733
1734    return err;
1735}
1736
1737status_t NuPlayer::getCurrentPosition(int64_t *mediaUs) {
1738    sp<Renderer> renderer = mRenderer;
1739    if (renderer == NULL) {
1740        return NO_INIT;
1741    }
1742
1743    return renderer->getCurrentPosition(mediaUs);
1744}
1745
1746void NuPlayer::getStats(Vector<sp<AMessage> > *mTrackStats) {
1747    CHECK(mTrackStats != NULL);
1748
1749    mTrackStats->clear();
1750    if (mVideoDecoder != NULL) {
1751        mTrackStats->push_back(mVideoDecoder->getStats());
1752    }
1753    if (mAudioDecoder != NULL) {
1754        mTrackStats->push_back(mAudioDecoder->getStats());
1755    }
1756}
1757
1758sp<MetaData> NuPlayer::getFileMeta() {
1759    return mSource->getFileFormatMeta();
1760}
1761
1762float NuPlayer::getFrameRate() {
1763    sp<MetaData> meta = mSource->getFormatMeta(false /* audio */);
1764    if (meta == NULL) {
1765        return 0;
1766    }
1767    int32_t rate;
1768    if (!meta->findInt32(kKeyFrameRate, &rate)) {
1769        // fall back to try file meta
1770        sp<MetaData> fileMeta = getFileMeta();
1771        if (fileMeta == NULL) {
1772            ALOGW("source has video meta but not file meta");
1773            return -1;
1774        }
1775        int32_t fileMetaRate;
1776        if (!fileMeta->findInt32(kKeyFrameRate, &fileMetaRate)) {
1777            return -1;
1778        }
1779        return fileMetaRate;
1780    }
1781    return rate;
1782}
1783
1784void NuPlayer::schedulePollDuration() {
1785    sp<AMessage> msg = new AMessage(kWhatPollDuration, this);
1786    msg->setInt32("generation", mPollDurationGeneration);
1787    msg->post();
1788}
1789
1790void NuPlayer::cancelPollDuration() {
1791    ++mPollDurationGeneration;
1792}
1793
1794void NuPlayer::processDeferredActions() {
1795    while (!mDeferredActions.empty()) {
1796        // We won't execute any deferred actions until we're no longer in
1797        // an intermediate state, i.e. one more more decoders are currently
1798        // flushing or shutting down.
1799
1800        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1801            // We're currently flushing, postpone the reset until that's
1802            // completed.
1803
1804            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1805                  mFlushingAudio, mFlushingVideo);
1806
1807            break;
1808        }
1809
1810        sp<Action> action = *mDeferredActions.begin();
1811        mDeferredActions.erase(mDeferredActions.begin());
1812
1813        action->execute(this);
1814    }
1815}
1816
1817void NuPlayer::performSeek(int64_t seekTimeUs) {
1818    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs)",
1819          (long long)seekTimeUs,
1820          seekTimeUs / 1E6);
1821
1822    if (mSource == NULL) {
1823        // This happens when reset occurs right before the loop mode
1824        // asynchronously seeks to the start of the stream.
1825        LOG_ALWAYS_FATAL_IF(mAudioDecoder != NULL || mVideoDecoder != NULL,
1826                "mSource is NULL and decoders not NULL audio(%p) video(%p)",
1827                mAudioDecoder.get(), mVideoDecoder.get());
1828        return;
1829    }
1830    mSource->seekTo(seekTimeUs);
1831    ++mTimedTextGeneration;
1832
1833    // everything's flushed, continue playback.
1834}
1835
1836void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) {
1837    ALOGV("performDecoderFlush audio=%d, video=%d", audio, video);
1838
1839    if ((audio == FLUSH_CMD_NONE || mAudioDecoder == NULL)
1840            && (video == FLUSH_CMD_NONE || mVideoDecoder == NULL)) {
1841        return;
1842    }
1843
1844    if (audio != FLUSH_CMD_NONE && mAudioDecoder != NULL) {
1845        flushDecoder(true /* audio */, (audio == FLUSH_CMD_SHUTDOWN));
1846    }
1847
1848    if (video != FLUSH_CMD_NONE && mVideoDecoder != NULL) {
1849        flushDecoder(false /* audio */, (video == FLUSH_CMD_SHUTDOWN));
1850    }
1851}
1852
1853void NuPlayer::performReset() {
1854    ALOGV("performReset");
1855
1856    CHECK(mAudioDecoder == NULL);
1857    CHECK(mVideoDecoder == NULL);
1858
1859    cancelPollDuration();
1860
1861    ++mScanSourcesGeneration;
1862    mScanSourcesPending = false;
1863
1864    if (mRendererLooper != NULL) {
1865        if (mRenderer != NULL) {
1866            mRendererLooper->unregisterHandler(mRenderer->id());
1867        }
1868        mRendererLooper->stop();
1869        mRendererLooper.clear();
1870    }
1871    mRenderer.clear();
1872    ++mRendererGeneration;
1873
1874    if (mSource != NULL) {
1875        mSource->stop();
1876
1877        mSource.clear();
1878    }
1879
1880    if (mDriver != NULL) {
1881        sp<NuPlayerDriver> driver = mDriver.promote();
1882        if (driver != NULL) {
1883            driver->notifyResetComplete();
1884        }
1885    }
1886
1887    mStarted = false;
1888    mSourceStarted = false;
1889}
1890
1891void NuPlayer::performScanSources() {
1892    ALOGV("performScanSources");
1893
1894    if (!mStarted) {
1895        return;
1896    }
1897
1898    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1899        postScanSources();
1900    }
1901}
1902
1903void NuPlayer::performSetSurface(const sp<Surface> &surface) {
1904    ALOGV("performSetSurface");
1905
1906    mSurface = surface;
1907
1908    // XXX - ignore error from setVideoScalingMode for now
1909    setVideoScalingMode(mVideoScalingMode);
1910
1911    if (mDriver != NULL) {
1912        sp<NuPlayerDriver> driver = mDriver.promote();
1913        if (driver != NULL) {
1914            driver->notifySetSurfaceComplete();
1915        }
1916    }
1917}
1918
1919void NuPlayer::performResumeDecoders(bool needNotify) {
1920    if (needNotify) {
1921        mResumePending = true;
1922        if (mVideoDecoder == NULL) {
1923            // if audio-only, we can notify seek complete now,
1924            // as the resume operation will be relatively fast.
1925            finishResume();
1926        }
1927    }
1928
1929    if (mVideoDecoder != NULL) {
1930        // When there is continuous seek, MediaPlayer will cache the seek
1931        // position, and send down new seek request when previous seek is
1932        // complete. Let's wait for at least one video output frame before
1933        // notifying seek complete, so that the video thumbnail gets updated
1934        // when seekbar is dragged.
1935        mVideoDecoder->signalResume(needNotify);
1936    }
1937
1938    if (mAudioDecoder != NULL) {
1939        mAudioDecoder->signalResume(false /* needNotify */);
1940    }
1941}
1942
1943void NuPlayer::finishResume() {
1944    if (mResumePending) {
1945        mResumePending = false;
1946        notifyDriverSeekComplete();
1947    }
1948}
1949
1950void NuPlayer::notifyDriverSeekComplete() {
1951    if (mDriver != NULL) {
1952        sp<NuPlayerDriver> driver = mDriver.promote();
1953        if (driver != NULL) {
1954            driver->notifySeekComplete();
1955        }
1956    }
1957}
1958
1959void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1960    int32_t what;
1961    CHECK(msg->findInt32("what", &what));
1962
1963    switch (what) {
1964        case Source::kWhatInstantiateSecureDecoders:
1965        {
1966            if (mSource == NULL) {
1967                // This is a stale notification from a source that was
1968                // asynchronously preparing when the client called reset().
1969                // We handled the reset, the source is gone.
1970                break;
1971            }
1972
1973            sp<AMessage> reply;
1974            CHECK(msg->findMessage("reply", &reply));
1975            status_t err = onInstantiateSecureDecoders();
1976            reply->setInt32("err", err);
1977            reply->post();
1978            break;
1979        }
1980
1981        case Source::kWhatPrepared:
1982        {
1983            if (mSource == NULL) {
1984                // This is a stale notification from a source that was
1985                // asynchronously preparing when the client called reset().
1986                // We handled the reset, the source is gone.
1987                break;
1988            }
1989
1990            int32_t err;
1991            CHECK(msg->findInt32("err", &err));
1992
1993            if (err != OK) {
1994                // shut down potential secure codecs in case client never calls reset
1995                mDeferredActions.push_back(
1996                        new FlushDecoderAction(FLUSH_CMD_SHUTDOWN /* audio */,
1997                                               FLUSH_CMD_SHUTDOWN /* video */));
1998                processDeferredActions();
1999            }
2000
2001            sp<NuPlayerDriver> driver = mDriver.promote();
2002            if (driver != NULL) {
2003                // notify duration first, so that it's definitely set when
2004                // the app received the "prepare complete" callback.
2005                int64_t durationUs;
2006                if (mSource->getDuration(&durationUs) == OK) {
2007                    driver->notifyDuration(durationUs);
2008                }
2009                driver->notifyPrepareCompleted(err);
2010            }
2011
2012            break;
2013        }
2014
2015        case Source::kWhatFlagsChanged:
2016        {
2017            uint32_t flags;
2018            CHECK(msg->findInt32("flags", (int32_t *)&flags));
2019
2020            sp<NuPlayerDriver> driver = mDriver.promote();
2021            if (driver != NULL) {
2022                if ((flags & NuPlayer::Source::FLAG_CAN_SEEK) == 0) {
2023                    driver->notifyListener(
2024                            MEDIA_INFO, MEDIA_INFO_NOT_SEEKABLE, 0);
2025                }
2026                driver->notifyFlagsChanged(flags);
2027            }
2028
2029            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
2030                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
2031                cancelPollDuration();
2032            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
2033                    && (flags & Source::FLAG_DYNAMIC_DURATION)
2034                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
2035                schedulePollDuration();
2036            }
2037
2038            mSourceFlags = flags;
2039            break;
2040        }
2041
2042        case Source::kWhatVideoSizeChanged:
2043        {
2044            sp<AMessage> format;
2045            CHECK(msg->findMessage("format", &format));
2046
2047            updateVideoSize(format);
2048            break;
2049        }
2050
2051        case Source::kWhatBufferingUpdate:
2052        {
2053            int32_t percentage;
2054            CHECK(msg->findInt32("percentage", &percentage));
2055
2056            notifyListener(MEDIA_BUFFERING_UPDATE, percentage, 0);
2057            break;
2058        }
2059
2060        case Source::kWhatPauseOnBufferingStart:
2061        {
2062            // ignore if not playing
2063            if (mStarted) {
2064                ALOGI("buffer low, pausing...");
2065
2066                mPausedForBuffering = true;
2067                onPause();
2068            }
2069            // fall-thru
2070        }
2071
2072        case Source::kWhatBufferingStart:
2073        {
2074            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
2075            break;
2076        }
2077
2078        case Source::kWhatResumeOnBufferingEnd:
2079        {
2080            // ignore if not playing
2081            if (mStarted) {
2082                ALOGI("buffer ready, resuming...");
2083
2084                mPausedForBuffering = false;
2085
2086                // do not resume yet if client didn't unpause
2087                if (!mPausedByClient) {
2088                    onResume();
2089                }
2090            }
2091            // fall-thru
2092        }
2093
2094        case Source::kWhatBufferingEnd:
2095        {
2096            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
2097            break;
2098        }
2099
2100        case Source::kWhatCacheStats:
2101        {
2102            int32_t kbps;
2103            CHECK(msg->findInt32("bandwidth", &kbps));
2104
2105            notifyListener(MEDIA_INFO, MEDIA_INFO_NETWORK_BANDWIDTH, kbps);
2106            break;
2107        }
2108
2109        case Source::kWhatSubtitleData:
2110        {
2111            sp<ABuffer> buffer;
2112            CHECK(msg->findBuffer("buffer", &buffer));
2113
2114            sendSubtitleData(buffer, 0 /* baseIndex */);
2115            break;
2116        }
2117
2118        case Source::kWhatTimedMetaData:
2119        {
2120            sp<ABuffer> buffer;
2121            if (!msg->findBuffer("buffer", &buffer)) {
2122                notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
2123            } else {
2124                sendTimedMetaData(buffer);
2125            }
2126            break;
2127        }
2128
2129        case Source::kWhatTimedTextData:
2130        {
2131            int32_t generation;
2132            if (msg->findInt32("generation", &generation)
2133                    && generation != mTimedTextGeneration) {
2134                break;
2135            }
2136
2137            sp<ABuffer> buffer;
2138            CHECK(msg->findBuffer("buffer", &buffer));
2139
2140            sp<NuPlayerDriver> driver = mDriver.promote();
2141            if (driver == NULL) {
2142                break;
2143            }
2144
2145            int posMs;
2146            int64_t timeUs, posUs;
2147            driver->getCurrentPosition(&posMs);
2148            posUs = posMs * 1000;
2149            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2150
2151            if (posUs < timeUs) {
2152                if (!msg->findInt32("generation", &generation)) {
2153                    msg->setInt32("generation", mTimedTextGeneration);
2154                }
2155                msg->post(timeUs - posUs);
2156            } else {
2157                sendTimedTextData(buffer);
2158            }
2159            break;
2160        }
2161
2162        case Source::kWhatQueueDecoderShutdown:
2163        {
2164            int32_t audio, video;
2165            CHECK(msg->findInt32("audio", &audio));
2166            CHECK(msg->findInt32("video", &video));
2167
2168            sp<AMessage> reply;
2169            CHECK(msg->findMessage("reply", &reply));
2170
2171            queueDecoderShutdown(audio, video, reply);
2172            break;
2173        }
2174
2175        case Source::kWhatDrmNoLicense:
2176        {
2177            notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, ERROR_DRM_NO_LICENSE);
2178            break;
2179        }
2180
2181        default:
2182            TRESPASS();
2183    }
2184}
2185
2186void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
2187    int32_t what;
2188    CHECK(msg->findInt32("what", &what));
2189
2190    switch (what) {
2191        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
2192        {
2193            sp<ABuffer> buffer;
2194            CHECK(msg->findBuffer("buffer", &buffer));
2195
2196            size_t inbandTracks = 0;
2197            if (mSource != NULL) {
2198                inbandTracks = mSource->getTrackCount();
2199            }
2200
2201            sendSubtitleData(buffer, inbandTracks);
2202            break;
2203        }
2204
2205        case NuPlayer::CCDecoder::kWhatTrackAdded:
2206        {
2207            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
2208
2209            break;
2210        }
2211
2212        default:
2213            TRESPASS();
2214    }
2215
2216
2217}
2218
2219void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
2220    int32_t trackIndex;
2221    int64_t timeUs, durationUs;
2222    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
2223    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2224    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
2225
2226    Parcel in;
2227    in.writeInt32(trackIndex + baseIndex);
2228    in.writeInt64(timeUs);
2229    in.writeInt64(durationUs);
2230    in.writeInt32(buffer->size());
2231    in.writeInt32(buffer->size());
2232    in.write(buffer->data(), buffer->size());
2233
2234    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
2235}
2236
2237void NuPlayer::sendTimedMetaData(const sp<ABuffer> &buffer) {
2238    int64_t timeUs;
2239    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2240
2241    Parcel in;
2242    in.writeInt64(timeUs);
2243    in.writeInt32(buffer->size());
2244    in.writeInt32(buffer->size());
2245    in.write(buffer->data(), buffer->size());
2246
2247    notifyListener(MEDIA_META_DATA, 0, 0, &in);
2248}
2249
2250void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
2251    const void *data;
2252    size_t size = 0;
2253    int64_t timeUs;
2254    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
2255
2256    AString mime;
2257    CHECK(buffer->meta()->findString("mime", &mime));
2258    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
2259
2260    data = buffer->data();
2261    size = buffer->size();
2262
2263    Parcel parcel;
2264    if (size > 0) {
2265        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2266        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
2267        TextDescriptions::getParcelOfDescriptions(
2268                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
2269    }
2270
2271    if ((parcel.dataSize() > 0)) {
2272        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
2273    } else {  // send an empty timed text
2274        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
2275    }
2276}
2277////////////////////////////////////////////////////////////////////////////////
2278
2279sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
2280    sp<MetaData> meta = getFormatMeta(audio);
2281
2282    if (meta == NULL) {
2283        return NULL;
2284    }
2285
2286    sp<AMessage> msg = new AMessage;
2287
2288    if(convertMetaDataToMessage(meta, &msg) == OK) {
2289        return msg;
2290    }
2291    return NULL;
2292}
2293
2294void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
2295    sp<AMessage> notify = dupNotify();
2296    notify->setInt32("what", kWhatFlagsChanged);
2297    notify->setInt32("flags", flags);
2298    notify->post();
2299}
2300
2301void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
2302    sp<AMessage> notify = dupNotify();
2303    notify->setInt32("what", kWhatVideoSizeChanged);
2304    notify->setMessage("format", format);
2305    notify->post();
2306}
2307
2308void NuPlayer::Source::notifyPrepared(status_t err) {
2309    sp<AMessage> notify = dupNotify();
2310    notify->setInt32("what", kWhatPrepared);
2311    notify->setInt32("err", err);
2312    notify->post();
2313}
2314
2315void NuPlayer::Source::notifyInstantiateSecureDecoders(const sp<AMessage> &reply) {
2316    sp<AMessage> notify = dupNotify();
2317    notify->setInt32("what", kWhatInstantiateSecureDecoders);
2318    notify->setMessage("reply", reply);
2319    notify->post();
2320}
2321
2322void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
2323    TRESPASS();
2324}
2325
2326}  // namespace android
2327