NuPlayer.cpp revision 13d6faa02087ce3bb0d4a02b8495f1822f211433 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerDecoder.h"
25#include "NuPlayerDecoderPassThrough.h"
26#include "NuPlayerDriver.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29#include "RTSPSource.h"
30#include "StreamingSource.h"
31#include "GenericSource.h"
32#include "TextDescriptions.h"
33
34#include "ATSParser.h"
35
36#include <media/stagefright/foundation/hexdump.h>
37#include <media/stagefright/foundation/ABuffer.h>
38#include <media/stagefright/foundation/ADebug.h>
39#include <media/stagefright/foundation/AMessage.h>
40#include <media/stagefright/MediaBuffer.h>
41#include <media/stagefright/MediaDefs.h>
42#include <media/stagefright/MediaErrors.h>
43#include <media/stagefright/MetaData.h>
44#include <gui/IGraphicBufferProducer.h>
45
46#include "avc_utils.h"
47
48#include "ESDS.h"
49#include <media/stagefright/Utils.h>
50
51namespace android {
52
53struct NuPlayer::Action : public RefBase {
54    Action() {}
55
56    virtual void execute(NuPlayer *player) = 0;
57
58private:
59    DISALLOW_EVIL_CONSTRUCTORS(Action);
60};
61
62struct NuPlayer::SeekAction : public Action {
63    SeekAction(int64_t seekTimeUs)
64        : mSeekTimeUs(seekTimeUs) {
65    }
66
67    virtual void execute(NuPlayer *player) {
68        player->performSeek(mSeekTimeUs);
69    }
70
71private:
72    int64_t mSeekTimeUs;
73
74    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
75};
76
77struct NuPlayer::SetSurfaceAction : public Action {
78    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
79        : mWrapper(wrapper) {
80    }
81
82    virtual void execute(NuPlayer *player) {
83        player->performSetSurface(mWrapper);
84    }
85
86private:
87    sp<NativeWindowWrapper> mWrapper;
88
89    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
90};
91
92struct NuPlayer::ShutdownDecoderAction : public Action {
93    ShutdownDecoderAction(bool audio, bool video)
94        : mAudio(audio),
95          mVideo(video) {
96    }
97
98    virtual void execute(NuPlayer *player) {
99        player->performDecoderShutdown(mAudio, mVideo);
100    }
101
102private:
103    bool mAudio;
104    bool mVideo;
105
106    DISALLOW_EVIL_CONSTRUCTORS(ShutdownDecoderAction);
107};
108
109struct NuPlayer::PostMessageAction : public Action {
110    PostMessageAction(const sp<AMessage> &msg)
111        : mMessage(msg) {
112    }
113
114    virtual void execute(NuPlayer *) {
115        mMessage->post();
116    }
117
118private:
119    sp<AMessage> mMessage;
120
121    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
122};
123
124// Use this if there's no state necessary to save in order to execute
125// the action.
126struct NuPlayer::SimpleAction : public Action {
127    typedef void (NuPlayer::*ActionFunc)();
128
129    SimpleAction(ActionFunc func)
130        : mFunc(func) {
131    }
132
133    virtual void execute(NuPlayer *player) {
134        (player->*mFunc)();
135    }
136
137private:
138    ActionFunc mFunc;
139
140    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
141};
142
143////////////////////////////////////////////////////////////////////////////////
144
145NuPlayer::NuPlayer()
146    : mUIDValid(false),
147      mSourceFlags(0),
148      mVideoIsAVC(false),
149      mOffloadAudio(false),
150      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
151      mAudioDecoderGeneration(0),
152      mVideoDecoderGeneration(0),
153      mAudioEOS(false),
154      mVideoEOS(false),
155      mScanSourcesPending(false),
156      mScanSourcesGeneration(0),
157      mPollDurationGeneration(0),
158      mTimedTextGeneration(0),
159      mTimeDiscontinuityPending(false),
160      mFlushingAudio(NONE),
161      mFlushingVideo(NONE),
162      mSkipRenderingAudioUntilMediaTimeUs(-1ll),
163      mSkipRenderingVideoUntilMediaTimeUs(-1ll),
164      mVideoLateByUs(0ll),
165      mNumFramesTotal(0ll),
166      mNumFramesDropped(0ll),
167      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
168      mStarted(false) {
169}
170
171NuPlayer::~NuPlayer() {
172}
173
174void NuPlayer::setUID(uid_t uid) {
175    mUIDValid = true;
176    mUID = uid;
177}
178
179void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
180    mDriver = driver;
181}
182
183void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
184    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
185
186    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
187
188    msg->setObject("source", new StreamingSource(notify, source));
189    msg->post();
190}
191
192static bool IsHTTPLiveURL(const char *url) {
193    if (!strncasecmp("http://", url, 7)
194            || !strncasecmp("https://", url, 8)
195            || !strncasecmp("file://", url, 7)) {
196        size_t len = strlen(url);
197        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
198            return true;
199        }
200
201        if (strstr(url,"m3u8")) {
202            return true;
203        }
204    }
205
206    return false;
207}
208
209void NuPlayer::setDataSourceAsync(
210        const sp<IMediaHTTPService> &httpService,
211        const char *url,
212        const KeyedVector<String8, String8> *headers) {
213
214    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
215    size_t len = strlen(url);
216
217    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
218
219    sp<Source> source;
220    if (IsHTTPLiveURL(url)) {
221        source = new HTTPLiveSource(notify, httpService, url, headers);
222    } else if (!strncasecmp(url, "rtsp://", 7)) {
223        source = new RTSPSource(
224                notify, httpService, url, headers, mUIDValid, mUID);
225    } else if ((!strncasecmp(url, "http://", 7)
226                || !strncasecmp(url, "https://", 8))
227                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
228                    || strstr(url, ".sdp?"))) {
229        source = new RTSPSource(
230                notify, httpService, url, headers, mUIDValid, mUID, true);
231    } else {
232        sp<GenericSource> genericSource =
233                new GenericSource(notify, mUIDValid, mUID);
234        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
235        // The correct flags will be updated in Source::kWhatFlagsChanged
236        // handler when  GenericSource is prepared.
237
238        status_t err = genericSource->setDataSource(httpService, url, headers);
239
240        if (err == OK) {
241            source = genericSource;
242        } else {
243            ALOGE("Failed to set data source!");
244        }
245    }
246    msg->setObject("source", source);
247    msg->post();
248}
249
250void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
251    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
252
253    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
254
255    sp<GenericSource> source =
256            new GenericSource(notify, mUIDValid, mUID);
257
258    status_t err = source->setDataSource(fd, offset, length);
259
260    if (err != OK) {
261        ALOGE("Failed to set data source!");
262        source = NULL;
263    }
264
265    msg->setObject("source", source);
266    msg->post();
267}
268
269void NuPlayer::prepareAsync() {
270    (new AMessage(kWhatPrepare, id()))->post();
271}
272
273void NuPlayer::setVideoSurfaceTextureAsync(
274        const sp<IGraphicBufferProducer> &bufferProducer) {
275    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id());
276
277    if (bufferProducer == NULL) {
278        msg->setObject("native-window", NULL);
279    } else {
280        msg->setObject(
281                "native-window",
282                new NativeWindowWrapper(
283                    new Surface(bufferProducer)));
284    }
285
286    msg->post();
287}
288
289void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
290    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id());
291    msg->setObject("sink", sink);
292    msg->post();
293}
294
295void NuPlayer::start() {
296    (new AMessage(kWhatStart, id()))->post();
297}
298
299void NuPlayer::pause() {
300    (new AMessage(kWhatPause, id()))->post();
301}
302
303void NuPlayer::resume() {
304    (new AMessage(kWhatResume, id()))->post();
305}
306
307void NuPlayer::resetAsync() {
308    (new AMessage(kWhatReset, id()))->post();
309}
310
311void NuPlayer::seekToAsync(int64_t seekTimeUs) {
312    sp<AMessage> msg = new AMessage(kWhatSeek, id());
313    msg->setInt64("seekTimeUs", seekTimeUs);
314    msg->post();
315}
316
317// static
318bool NuPlayer::IsFlushingState(FlushStatus state, bool *needShutdown) {
319    switch (state) {
320        case FLUSHING_DECODER:
321            if (needShutdown != NULL) {
322                *needShutdown = false;
323            }
324            return true;
325
326        case FLUSHING_DECODER_SHUTDOWN:
327            if (needShutdown != NULL) {
328                *needShutdown = true;
329            }
330            return true;
331
332        default:
333            return false;
334    }
335}
336
337void NuPlayer::writeTrackInfo(
338        Parcel* reply, const sp<AMessage> format) const {
339    int32_t trackType;
340    CHECK(format->findInt32("type", &trackType));
341
342    AString lang;
343    CHECK(format->findString("language", &lang));
344
345    reply->writeInt32(2); // write something non-zero
346    reply->writeInt32(trackType);
347    reply->writeString16(String16(lang.c_str()));
348
349    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
350        AString mime;
351        CHECK(format->findString("mime", &mime));
352
353        int32_t isAuto, isDefault, isForced;
354        CHECK(format->findInt32("auto", &isAuto));
355        CHECK(format->findInt32("default", &isDefault));
356        CHECK(format->findInt32("forced", &isForced));
357
358        reply->writeString16(String16(mime.c_str()));
359        reply->writeInt32(isAuto);
360        reply->writeInt32(isDefault);
361        reply->writeInt32(isForced);
362    }
363}
364
365void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
366    switch (msg->what()) {
367        case kWhatSetDataSource:
368        {
369            ALOGV("kWhatSetDataSource");
370
371            CHECK(mSource == NULL);
372
373            status_t err = OK;
374            sp<RefBase> obj;
375            CHECK(msg->findObject("source", &obj));
376            if (obj != NULL) {
377                mSource = static_cast<Source *>(obj.get());
378            } else {
379                err = UNKNOWN_ERROR;
380            }
381
382            CHECK(mDriver != NULL);
383            sp<NuPlayerDriver> driver = mDriver.promote();
384            if (driver != NULL) {
385                driver->notifySetDataSourceCompleted(err);
386            }
387            break;
388        }
389
390        case kWhatPrepare:
391        {
392            mSource->prepareAsync();
393            break;
394        }
395
396        case kWhatGetTrackInfo:
397        {
398            uint32_t replyID;
399            CHECK(msg->senderAwaitsResponse(&replyID));
400
401            Parcel* reply;
402            CHECK(msg->findPointer("reply", (void**)&reply));
403
404            size_t inbandTracks = 0;
405            if (mSource != NULL) {
406                inbandTracks = mSource->getTrackCount();
407            }
408
409            size_t ccTracks = 0;
410            if (mCCDecoder != NULL) {
411                ccTracks = mCCDecoder->getTrackCount();
412            }
413
414            // total track count
415            reply->writeInt32(inbandTracks + ccTracks);
416
417            // write inband tracks
418            for (size_t i = 0; i < inbandTracks; ++i) {
419                writeTrackInfo(reply, mSource->getTrackInfo(i));
420            }
421
422            // write CC track
423            for (size_t i = 0; i < ccTracks; ++i) {
424                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
425            }
426
427            sp<AMessage> response = new AMessage;
428            response->postReply(replyID);
429            break;
430        }
431
432        case kWhatGetSelectedTrack:
433        {
434            status_t err = INVALID_OPERATION;
435            if (mSource != NULL) {
436                err = OK;
437
438                int32_t type32;
439                CHECK(msg->findInt32("type", (int32_t*)&type32));
440                media_track_type type = (media_track_type)type32;
441                ssize_t selectedTrack = mSource->getSelectedTrack(type);
442
443                Parcel* reply;
444                CHECK(msg->findPointer("reply", (void**)&reply));
445                reply->writeInt32(selectedTrack);
446            }
447
448            sp<AMessage> response = new AMessage;
449            response->setInt32("err", err);
450
451            uint32_t replyID;
452            CHECK(msg->senderAwaitsResponse(&replyID));
453            response->postReply(replyID);
454            break;
455        }
456
457        case kWhatSelectTrack:
458        {
459            uint32_t replyID;
460            CHECK(msg->senderAwaitsResponse(&replyID));
461
462            size_t trackIndex;
463            int32_t select;
464            CHECK(msg->findSize("trackIndex", &trackIndex));
465            CHECK(msg->findInt32("select", &select));
466
467            status_t err = INVALID_OPERATION;
468
469            size_t inbandTracks = 0;
470            if (mSource != NULL) {
471                inbandTracks = mSource->getTrackCount();
472            }
473            size_t ccTracks = 0;
474            if (mCCDecoder != NULL) {
475                ccTracks = mCCDecoder->getTrackCount();
476            }
477
478            if (trackIndex < inbandTracks) {
479                err = mSource->selectTrack(trackIndex, select);
480
481                if (!select && err == OK) {
482                    int32_t type;
483                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
484                    if (info != NULL
485                            && info->findInt32("type", &type)
486                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
487                        ++mTimedTextGeneration;
488                    }
489                }
490            } else {
491                trackIndex -= inbandTracks;
492
493                if (trackIndex < ccTracks) {
494                    err = mCCDecoder->selectTrack(trackIndex, select);
495                }
496            }
497
498            sp<AMessage> response = new AMessage;
499            response->setInt32("err", err);
500
501            response->postReply(replyID);
502            break;
503        }
504
505        case kWhatPollDuration:
506        {
507            int32_t generation;
508            CHECK(msg->findInt32("generation", &generation));
509
510            if (generation != mPollDurationGeneration) {
511                // stale
512                break;
513            }
514
515            int64_t durationUs;
516            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
517                sp<NuPlayerDriver> driver = mDriver.promote();
518                if (driver != NULL) {
519                    driver->notifyDuration(durationUs);
520                }
521            }
522
523            msg->post(1000000ll);  // poll again in a second.
524            break;
525        }
526
527        case kWhatSetVideoNativeWindow:
528        {
529            ALOGV("kWhatSetVideoNativeWindow");
530
531            mDeferredActions.push_back(
532                    new ShutdownDecoderAction(
533                        false /* audio */, true /* video */));
534
535            sp<RefBase> obj;
536            CHECK(msg->findObject("native-window", &obj));
537
538            mDeferredActions.push_back(
539                    new SetSurfaceAction(
540                        static_cast<NativeWindowWrapper *>(obj.get())));
541
542            if (obj != NULL) {
543                // If there is a new surface texture, instantiate decoders
544                // again if possible.
545                mDeferredActions.push_back(
546                        new SimpleAction(&NuPlayer::performScanSources));
547            }
548
549            processDeferredActions();
550            break;
551        }
552
553        case kWhatSetAudioSink:
554        {
555            ALOGV("kWhatSetAudioSink");
556
557            sp<RefBase> obj;
558            CHECK(msg->findObject("sink", &obj));
559
560            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
561            break;
562        }
563
564        case kWhatStart:
565        {
566            ALOGV("kWhatStart");
567
568            mVideoIsAVC = false;
569            mOffloadAudio = false;
570            mAudioEOS = false;
571            mVideoEOS = false;
572            mSkipRenderingAudioUntilMediaTimeUs = -1;
573            mSkipRenderingVideoUntilMediaTimeUs = -1;
574            mVideoLateByUs = 0;
575            mNumFramesTotal = 0;
576            mNumFramesDropped = 0;
577            mStarted = true;
578
579            /* instantiate decoders now for secure playback */
580            if (mSourceFlags & Source::FLAG_SECURE) {
581                if (mNativeWindow != NULL) {
582                    instantiateDecoder(false, &mVideoDecoder);
583                }
584
585                if (mAudioSink != NULL) {
586                    instantiateDecoder(true, &mAudioDecoder);
587                }
588            }
589
590            mSource->start();
591
592            uint32_t flags = 0;
593
594            if (mSource->isRealTime()) {
595                flags |= Renderer::FLAG_REAL_TIME;
596            }
597
598            sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
599            audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
600            if (mAudioSink != NULL) {
601                streamType = mAudioSink->getAudioStreamType();
602            }
603
604            sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
605
606            mOffloadAudio =
607                canOffloadStream(audioMeta, (videoFormat != NULL),
608                                 true /* is_streaming */, streamType);
609            if (mOffloadAudio) {
610                flags |= Renderer::FLAG_OFFLOAD_AUDIO;
611            }
612
613            mRenderer = new Renderer(
614                    mAudioSink,
615                    new AMessage(kWhatRendererNotify, id()),
616                    flags);
617
618            mRendererLooper = new ALooper;
619            mRendererLooper->setName("NuPlayerRenderer");
620            mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
621            mRendererLooper->registerHandler(mRenderer);
622
623            postScanSources();
624            break;
625        }
626
627        case kWhatScanSources:
628        {
629            int32_t generation;
630            CHECK(msg->findInt32("generation", &generation));
631            if (generation != mScanSourcesGeneration) {
632                // Drop obsolete msg.
633                break;
634            }
635
636            mScanSourcesPending = false;
637
638            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
639                 mAudioDecoder != NULL, mVideoDecoder != NULL);
640
641            bool mHadAnySourcesBefore =
642                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
643
644            // initialize video before audio because successful initialization of
645            // video may change deep buffer mode of audio.
646            if (mNativeWindow != NULL) {
647                instantiateDecoder(false, &mVideoDecoder);
648            }
649
650            if (mAudioSink != NULL) {
651                if (mOffloadAudio) {
652                    // open audio sink early under offload mode.
653                    sp<AMessage> format = mSource->getFormat(true /*audio*/);
654                    openAudioSink(format, true /*offloadOnly*/);
655                }
656                instantiateDecoder(true, &mAudioDecoder);
657            }
658
659            if (!mHadAnySourcesBefore
660                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
661                // This is the first time we've found anything playable.
662
663                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
664                    schedulePollDuration();
665                }
666            }
667
668            status_t err;
669            if ((err = mSource->feedMoreTSData()) != OK) {
670                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
671                    // We're not currently decoding anything (no audio or
672                    // video tracks found) and we just ran out of input data.
673
674                    if (err == ERROR_END_OF_STREAM) {
675                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
676                    } else {
677                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
678                    }
679                }
680                break;
681            }
682
683            if ((mAudioDecoder == NULL && mAudioSink != NULL)
684                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
685                msg->post(100000ll);
686                mScanSourcesPending = true;
687            }
688            break;
689        }
690
691        case kWhatVideoNotify:
692        case kWhatAudioNotify:
693        {
694            bool audio = msg->what() == kWhatAudioNotify;
695
696            int32_t currentDecoderGeneration =
697                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
698            int32_t requesterGeneration = currentDecoderGeneration - 1;
699            CHECK(msg->findInt32("generation", &requesterGeneration));
700
701            if (requesterGeneration != currentDecoderGeneration) {
702                ALOGV("got message from old %s decoder, generation(%d:%d)",
703                        audio ? "audio" : "video", requesterGeneration,
704                        currentDecoderGeneration);
705                sp<AMessage> reply;
706                if (!(msg->findMessage("reply", &reply))) {
707                    return;
708                }
709
710                reply->setInt32("err", INFO_DISCONTINUITY);
711                reply->post();
712                return;
713            }
714
715            int32_t what;
716            CHECK(msg->findInt32("what", &what));
717
718            if (what == Decoder::kWhatFillThisBuffer) {
719                status_t err = feedDecoderInputData(
720                        audio, msg);
721
722                if (err == -EWOULDBLOCK) {
723                    if (mSource->feedMoreTSData() == OK) {
724                        msg->post(10000ll);
725                    }
726                }
727            } else if (what == Decoder::kWhatEOS) {
728                int32_t err;
729                CHECK(msg->findInt32("err", &err));
730
731                if (err == ERROR_END_OF_STREAM) {
732                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
733                } else {
734                    ALOGV("got %s decoder EOS w/ error %d",
735                         audio ? "audio" : "video",
736                         err);
737                }
738
739                mRenderer->queueEOS(audio, err);
740            } else if (what == Decoder::kWhatFlushCompleted) {
741                bool needShutdown;
742
743                if (audio) {
744                    CHECK(IsFlushingState(mFlushingAudio, &needShutdown));
745                    mFlushingAudio = FLUSHED;
746                } else {
747                    CHECK(IsFlushingState(mFlushingVideo, &needShutdown));
748                    mFlushingVideo = FLUSHED;
749
750                    mVideoLateByUs = 0;
751                }
752
753                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
754
755                if (needShutdown) {
756                    ALOGV("initiating %s decoder shutdown",
757                         audio ? "audio" : "video");
758
759                    (audio ? mAudioDecoder : mVideoDecoder)->initiateShutdown();
760
761                    if (audio) {
762                        mFlushingAudio = SHUTTING_DOWN_DECODER;
763                    } else {
764                        mFlushingVideo = SHUTTING_DOWN_DECODER;
765                    }
766                }
767
768                finishFlushIfPossible();
769            } else if (what == Decoder::kWhatOutputFormatChanged) {
770                sp<AMessage> format;
771                CHECK(msg->findMessage("format", &format));
772
773                if (audio) {
774                    openAudioSink(format, false /*offloadOnly*/);
775                } else {
776                    // video
777                    sp<AMessage> inputFormat =
778                            mSource->getFormat(false /* audio */);
779
780                    updateVideoSize(inputFormat, format);
781                }
782            } else if (what == Decoder::kWhatShutdownCompleted) {
783                ALOGV("%s shutdown completed", audio ? "audio" : "video");
784                if (audio) {
785                    mAudioDecoder.clear();
786
787                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
788                    mFlushingAudio = SHUT_DOWN;
789                } else {
790                    mVideoDecoder.clear();
791
792                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
793                    mFlushingVideo = SHUT_DOWN;
794                }
795
796                finishFlushIfPossible();
797            } else if (what == Decoder::kWhatError) {
798                ALOGE("Received error from %s decoder, aborting playback.",
799                     audio ? "audio" : "video");
800
801                status_t err;
802                if (!msg->findInt32("err", &err)) {
803                    err = UNKNOWN_ERROR;
804                }
805                mRenderer->queueEOS(audio, err);
806                if (audio && mFlushingAudio != NONE) {
807                    mAudioDecoder.clear();
808                    mFlushingAudio = SHUT_DOWN;
809                } else if (!audio && mFlushingVideo != NONE){
810                    mVideoDecoder.clear();
811                    mFlushingVideo = SHUT_DOWN;
812                }
813                finishFlushIfPossible();
814            } else if (what == Decoder::kWhatDrainThisBuffer) {
815                renderBuffer(audio, msg);
816            } else {
817                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
818                      what,
819                      what >> 24,
820                      (what >> 16) & 0xff,
821                      (what >> 8) & 0xff,
822                      what & 0xff);
823            }
824
825            break;
826        }
827
828        case kWhatRendererNotify:
829        {
830            int32_t what;
831            CHECK(msg->findInt32("what", &what));
832
833            if (what == Renderer::kWhatEOS) {
834                int32_t audio;
835                CHECK(msg->findInt32("audio", &audio));
836
837                int32_t finalResult;
838                CHECK(msg->findInt32("finalResult", &finalResult));
839
840                if (audio) {
841                    mAudioEOS = true;
842                } else {
843                    mVideoEOS = true;
844                }
845
846                if (finalResult == ERROR_END_OF_STREAM) {
847                    ALOGV("reached %s EOS", audio ? "audio" : "video");
848                } else {
849                    ALOGE("%s track encountered an error (%d)",
850                         audio ? "audio" : "video", finalResult);
851
852                    notifyListener(
853                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
854                }
855
856                if ((mAudioEOS || mAudioDecoder == NULL)
857                        && (mVideoEOS || mVideoDecoder == NULL)) {
858                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
859                }
860            } else if (what == Renderer::kWhatPosition) {
861                int64_t positionUs;
862                CHECK(msg->findInt64("positionUs", &positionUs));
863
864                CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs));
865
866                if (mDriver != NULL) {
867                    sp<NuPlayerDriver> driver = mDriver.promote();
868                    if (driver != NULL) {
869                        driver->notifyPosition(positionUs);
870
871                        driver->notifyFrameStats(
872                                mNumFramesTotal, mNumFramesDropped);
873                    }
874                }
875            } else if (what == Renderer::kWhatFlushComplete) {
876                int32_t audio;
877                CHECK(msg->findInt32("audio", &audio));
878
879                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
880            } else if (what == Renderer::kWhatVideoRenderingStart) {
881                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
882            } else if (what == Renderer::kWhatMediaRenderingStart) {
883                ALOGV("media rendering started");
884                notifyListener(MEDIA_STARTED, 0, 0);
885            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
886                ALOGV("Tear down audio offload, fall back to s/w path");
887                int64_t positionUs;
888                CHECK(msg->findInt64("positionUs", &positionUs));
889                closeAudioSink();
890                mAudioDecoder.clear();
891                mRenderer->flush(true /* audio */);
892                if (mVideoDecoder != NULL) {
893                    mRenderer->flush(false /* audio */);
894                }
895                mRenderer->signalDisableOffloadAudio();
896                mOffloadAudio = false;
897
898                performSeek(positionUs);
899                instantiateDecoder(true /* audio */, &mAudioDecoder);
900            }
901            break;
902        }
903
904        case kWhatMoreDataQueued:
905        {
906            break;
907        }
908
909        case kWhatReset:
910        {
911            ALOGV("kWhatReset");
912
913            mDeferredActions.push_back(
914                    new ShutdownDecoderAction(
915                        true /* audio */, true /* video */));
916
917            mDeferredActions.push_back(
918                    new SimpleAction(&NuPlayer::performReset));
919
920            processDeferredActions();
921            break;
922        }
923
924        case kWhatSeek:
925        {
926            int64_t seekTimeUs;
927            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
928
929            ALOGV("kWhatSeek seekTimeUs=%lld us", seekTimeUs);
930
931            mDeferredActions.push_back(
932                    new SimpleAction(&NuPlayer::performDecoderFlush));
933
934            mDeferredActions.push_back(new SeekAction(seekTimeUs));
935
936            processDeferredActions();
937            break;
938        }
939
940        case kWhatPause:
941        {
942            CHECK(mRenderer != NULL);
943            mSource->pause();
944            mRenderer->pause();
945            break;
946        }
947
948        case kWhatResume:
949        {
950            CHECK(mRenderer != NULL);
951            mSource->resume();
952            mRenderer->resume();
953            break;
954        }
955
956        case kWhatSourceNotify:
957        {
958            onSourceNotify(msg);
959            break;
960        }
961
962        case kWhatClosedCaptionNotify:
963        {
964            onClosedCaptionNotify(msg);
965            break;
966        }
967
968        default:
969            TRESPASS();
970            break;
971    }
972}
973
974void NuPlayer::finishFlushIfPossible() {
975    if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED
976            && mFlushingAudio != SHUT_DOWN) {
977        return;
978    }
979
980    if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED
981            && mFlushingVideo != SHUT_DOWN) {
982        return;
983    }
984
985    ALOGV("both audio and video are flushed now.");
986
987    if (mTimeDiscontinuityPending) {
988        mRenderer->signalTimeDiscontinuity();
989        mTimeDiscontinuityPending = false;
990    }
991
992    if (mAudioDecoder != NULL && mFlushingAudio == FLUSHED) {
993        mAudioDecoder->signalResume();
994    }
995
996    if (mVideoDecoder != NULL && mFlushingVideo == FLUSHED) {
997        mVideoDecoder->signalResume();
998    }
999
1000    mFlushingAudio = NONE;
1001    mFlushingVideo = NONE;
1002
1003    processDeferredActions();
1004}
1005
1006void NuPlayer::postScanSources() {
1007    if (mScanSourcesPending) {
1008        return;
1009    }
1010
1011    sp<AMessage> msg = new AMessage(kWhatScanSources, id());
1012    msg->setInt32("generation", mScanSourcesGeneration);
1013    msg->post();
1014
1015    mScanSourcesPending = true;
1016}
1017
1018void NuPlayer::openAudioSink(const sp<AMessage> &format, bool offloadOnly) {
1019    ALOGV("openAudioSink: offloadOnly(%d) mOffloadAudio(%d)",
1020            offloadOnly, mOffloadAudio);
1021    bool audioSinkChanged = false;
1022
1023    int32_t numChannels;
1024    CHECK(format->findInt32("channel-count", &numChannels));
1025
1026    int32_t channelMask;
1027    if (!format->findInt32("channel-mask", &channelMask)) {
1028        // signal to the AudioSink to derive the mask from count.
1029        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1030    }
1031
1032    int32_t sampleRate;
1033    CHECK(format->findInt32("sample-rate", &sampleRate));
1034
1035    uint32_t flags;
1036    int64_t durationUs;
1037    // FIXME: we should handle the case where the video decoder
1038    // is created after we receive the format change indication.
1039    // Current code will just make that we select deep buffer
1040    // with video which should not be a problem as it should
1041    // not prevent from keeping A/V sync.
1042    if (mVideoDecoder == NULL &&
1043            mSource->getDuration(&durationUs) == OK &&
1044            durationUs
1045                > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
1046        flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1047    } else {
1048        flags = AUDIO_OUTPUT_FLAG_NONE;
1049    }
1050
1051    if (mOffloadAudio) {
1052        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1053        AString mime;
1054        CHECK(format->findString("mime", &mime));
1055        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1056
1057        if (err != OK) {
1058            ALOGE("Couldn't map mime \"%s\" to a valid "
1059                    "audio_format", mime.c_str());
1060            mOffloadAudio = false;
1061        } else {
1062            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1063                    mime.c_str(), audioFormat);
1064
1065            int avgBitRate = -1;
1066            format->findInt32("bit-rate", &avgBitRate);
1067
1068            int32_t aacProfile = -1;
1069            if (audioFormat == AUDIO_FORMAT_AAC
1070                    && format->findInt32("aac-profile", &aacProfile)) {
1071                // Redefine AAC format as per aac profile
1072                mapAACProfileToAudioFormat(
1073                        audioFormat,
1074                        aacProfile);
1075            }
1076
1077            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1078            offloadInfo.duration_us = -1;
1079            format->findInt64(
1080                    "durationUs", &offloadInfo.duration_us);
1081            offloadInfo.sample_rate = sampleRate;
1082            offloadInfo.channel_mask = channelMask;
1083            offloadInfo.format = audioFormat;
1084            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1085            offloadInfo.bit_rate = avgBitRate;
1086            offloadInfo.has_video = (mVideoDecoder != NULL);
1087            offloadInfo.is_streaming = true;
1088
1089            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1090                ALOGV("openAudioSink: no change in offload mode");
1091                return;  // no change from previous configuration, everything ok.
1092            }
1093            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1094            flags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1095            flags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1096            audioSinkChanged = true;
1097            mAudioSink->close();
1098            err = mAudioSink->open(
1099                    sampleRate,
1100                    numChannels,
1101                    (audio_channel_mask_t)channelMask,
1102                    audioFormat,
1103                    8 /* bufferCount */,
1104                    &NuPlayer::Renderer::AudioSinkCallback,
1105                    mRenderer.get(),
1106                    (audio_output_flags_t)flags,
1107                    &offloadInfo);
1108
1109            if (err == OK) {
1110                // If the playback is offloaded to h/w, we pass
1111                // the HAL some metadata information.
1112                // We don't want to do this for PCM because it
1113                // will be going through the AudioFlinger mixer
1114                // before reaching the hardware.
1115                sp<MetaData> audioMeta =
1116                        mSource->getFormatMeta(true /* audio */);
1117                sendMetaDataToHal(mAudioSink, audioMeta);
1118                mCurrentOffloadInfo = offloadInfo;
1119                err = mAudioSink->start();
1120                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1121            }
1122            if (err != OK) {
1123                // Clean up, fall back to non offload mode.
1124                mAudioSink->close();
1125                mRenderer->signalDisableOffloadAudio();
1126                mOffloadAudio = false;
1127                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1128                ALOGV("openAudioSink: offload failed");
1129            }
1130        }
1131    }
1132    if (!offloadOnly && !mOffloadAudio) {
1133        flags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1134        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1135
1136        audioSinkChanged = true;
1137        mAudioSink->close();
1138        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1139        CHECK_EQ(mAudioSink->open(
1140                    sampleRate,
1141                    numChannels,
1142                    (audio_channel_mask_t)channelMask,
1143                    AUDIO_FORMAT_PCM_16_BIT,
1144                    8 /* bufferCount */,
1145                    NULL,
1146                    NULL,
1147                    (audio_output_flags_t)flags),
1148                 (status_t)OK);
1149        mAudioSink->start();
1150    }
1151    if (audioSinkChanged) {
1152        mRenderer->signalAudioSinkChanged();
1153    }
1154}
1155
1156void NuPlayer::closeAudioSink() {
1157    mAudioSink->close();
1158    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1159}
1160
1161status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
1162    if (*decoder != NULL) {
1163        return OK;
1164    }
1165
1166    sp<AMessage> format = mSource->getFormat(audio);
1167
1168    if (format == NULL) {
1169        return -EWOULDBLOCK;
1170    }
1171
1172    if (!audio) {
1173        AString mime;
1174        CHECK(format->findString("mime", &mime));
1175        mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
1176
1177        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
1178        mCCDecoder = new CCDecoder(ccNotify);
1179
1180        if (mSourceFlags & Source::FLAG_SECURE) {
1181            format->setInt32("secure", true);
1182        }
1183    }
1184
1185    if (audio) {
1186        sp<AMessage> notify = new AMessage(kWhatAudioNotify, id());
1187        ++mAudioDecoderGeneration;
1188        notify->setInt32("generation", mAudioDecoderGeneration);
1189
1190        if (mOffloadAudio) {
1191            *decoder = new DecoderPassThrough(notify);
1192        } else {
1193            *decoder = new Decoder(notify);
1194        }
1195    } else {
1196        sp<AMessage> notify = new AMessage(kWhatVideoNotify, id());
1197        ++mVideoDecoderGeneration;
1198        notify->setInt32("generation", mVideoDecoderGeneration);
1199
1200        *decoder = new Decoder(notify, mNativeWindow);
1201    }
1202    (*decoder)->init();
1203    (*decoder)->configure(format);
1204
1205    // allocate buffers to decrypt widevine source buffers
1206    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1207        Vector<sp<ABuffer> > inputBufs;
1208        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1209
1210        Vector<MediaBuffer *> mediaBufs;
1211        for (size_t i = 0; i < inputBufs.size(); i++) {
1212            const sp<ABuffer> &buffer = inputBufs[i];
1213            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1214            mediaBufs.push(mbuf);
1215        }
1216
1217        status_t err = mSource->setBuffers(audio, mediaBufs);
1218        if (err != OK) {
1219            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1220                mediaBufs[i]->release();
1221            }
1222            mediaBufs.clear();
1223            ALOGE("Secure source didn't support secure mediaBufs.");
1224            return err;
1225        }
1226    }
1227    return OK;
1228}
1229
1230status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
1231    sp<AMessage> reply;
1232    CHECK(msg->findMessage("reply", &reply));
1233
1234    if ((audio && mFlushingAudio != NONE)
1235            || (!audio && mFlushingVideo != NONE)) {
1236        reply->setInt32("err", INFO_DISCONTINUITY);
1237        reply->post();
1238        return OK;
1239    }
1240
1241    sp<ABuffer> accessUnit;
1242
1243    bool dropAccessUnit;
1244    do {
1245        status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
1246
1247        if (err == -EWOULDBLOCK) {
1248            return err;
1249        } else if (err != OK) {
1250            if (err == INFO_DISCONTINUITY) {
1251                int32_t type;
1252                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
1253
1254                bool formatChange =
1255                    (audio &&
1256                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
1257                    || (!audio &&
1258                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
1259
1260                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
1261
1262                ALOGI("%s discontinuity (formatChange=%d, time=%d)",
1263                     audio ? "audio" : "video", formatChange, timeChange);
1264
1265                if (audio) {
1266                    mSkipRenderingAudioUntilMediaTimeUs = -1;
1267                } else {
1268                    mSkipRenderingVideoUntilMediaTimeUs = -1;
1269                }
1270
1271                if (timeChange) {
1272                    sp<AMessage> extra;
1273                    if (accessUnit->meta()->findMessage("extra", &extra)
1274                            && extra != NULL) {
1275                        int64_t resumeAtMediaTimeUs;
1276                        if (extra->findInt64(
1277                                    "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
1278                            ALOGI("suppressing rendering of %s until %lld us",
1279                                    audio ? "audio" : "video", resumeAtMediaTimeUs);
1280
1281                            if (audio) {
1282                                mSkipRenderingAudioUntilMediaTimeUs =
1283                                    resumeAtMediaTimeUs;
1284                            } else {
1285                                mSkipRenderingVideoUntilMediaTimeUs =
1286                                    resumeAtMediaTimeUs;
1287                            }
1288                        }
1289                    }
1290                }
1291
1292                mTimeDiscontinuityPending =
1293                    mTimeDiscontinuityPending || timeChange;
1294
1295                if (mFlushingAudio == NONE && mFlushingVideo == NONE) {
1296                    // And we'll resume scanning sources once we're done
1297                    // flushing.
1298                    mDeferredActions.push_front(
1299                            new SimpleAction(
1300                                &NuPlayer::performScanSources));
1301                }
1302
1303                if (formatChange || timeChange) {
1304
1305                    sp<AMessage> newFormat = mSource->getFormat(audio);
1306                    sp<Decoder> &decoder = audio ? mAudioDecoder : mVideoDecoder;
1307                    if (formatChange && !decoder->supportsSeamlessFormatChange(newFormat)) {
1308                        flushDecoder(audio, /* needShutdown = */ true);
1309                    } else {
1310                        flushDecoder(audio, /* needShutdown = */ false);
1311                        err = OK;
1312                    }
1313                } else {
1314                    // This stream is unaffected by the discontinuity
1315                    return -EWOULDBLOCK;
1316                }
1317            }
1318
1319            reply->setInt32("err", err);
1320            reply->post();
1321            return OK;
1322        }
1323
1324        if (!audio) {
1325            ++mNumFramesTotal;
1326        }
1327
1328        dropAccessUnit = false;
1329        if (!audio
1330                && !(mSourceFlags & Source::FLAG_SECURE)
1331                && mVideoLateByUs > 100000ll
1332                && mVideoIsAVC
1333                && !IsAVCReferenceFrame(accessUnit)) {
1334            dropAccessUnit = true;
1335            ++mNumFramesDropped;
1336        }
1337    } while (dropAccessUnit);
1338
1339    // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
1340
1341#if 0
1342    int64_t mediaTimeUs;
1343    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
1344    ALOGV("feeding %s input buffer at media time %.2f secs",
1345         audio ? "audio" : "video",
1346         mediaTimeUs / 1E6);
1347#endif
1348
1349    if (!audio) {
1350        mCCDecoder->decode(accessUnit);
1351    }
1352
1353    reply->setBuffer("buffer", accessUnit);
1354    reply->post();
1355
1356    return OK;
1357}
1358
1359void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
1360    // ALOGV("renderBuffer %s", audio ? "audio" : "video");
1361
1362    sp<AMessage> reply;
1363    CHECK(msg->findMessage("reply", &reply));
1364
1365    if ((audio && mFlushingAudio != NONE)
1366            || (!audio && mFlushingVideo != NONE)) {
1367        // We're currently attempting to flush the decoder, in order
1368        // to complete this, the decoder wants all its buffers back,
1369        // so we don't want any output buffers it sent us (from before
1370        // we initiated the flush) to be stuck in the renderer's queue.
1371
1372        ALOGV("we're still flushing the %s decoder, sending its output buffer"
1373             " right back.", audio ? "audio" : "video");
1374
1375        reply->post();
1376        return;
1377    }
1378
1379    sp<ABuffer> buffer;
1380    CHECK(msg->findBuffer("buffer", &buffer));
1381
1382    int64_t mediaTimeUs;
1383    CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));
1384
1385    int64_t &skipUntilMediaTimeUs =
1386        audio
1387            ? mSkipRenderingAudioUntilMediaTimeUs
1388            : mSkipRenderingVideoUntilMediaTimeUs;
1389
1390    if (skipUntilMediaTimeUs >= 0) {
1391
1392        if (mediaTimeUs < skipUntilMediaTimeUs) {
1393            ALOGV("dropping %s buffer at time %lld as requested.",
1394                 audio ? "audio" : "video",
1395                 mediaTimeUs);
1396
1397            reply->post();
1398            return;
1399        }
1400
1401        skipUntilMediaTimeUs = -1;
1402    }
1403
1404    if (!audio && mCCDecoder->isSelected()) {
1405        mCCDecoder->display(mediaTimeUs);
1406    }
1407
1408    mRenderer->queueBuffer(audio, buffer, reply);
1409}
1410
1411void NuPlayer::updateVideoSize(
1412        const sp<AMessage> &inputFormat,
1413        const sp<AMessage> &outputFormat) {
1414    if (inputFormat == NULL) {
1415        ALOGW("Unknown video size, reporting 0x0!");
1416        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1417        return;
1418    }
1419
1420    int32_t displayWidth, displayHeight;
1421    int32_t cropLeft, cropTop, cropRight, cropBottom;
1422
1423    if (outputFormat != NULL) {
1424        int32_t width, height;
1425        CHECK(outputFormat->findInt32("width", &width));
1426        CHECK(outputFormat->findInt32("height", &height));
1427
1428        int32_t cropLeft, cropTop, cropRight, cropBottom;
1429        CHECK(outputFormat->findRect(
1430                    "crop",
1431                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1432
1433        displayWidth = cropRight - cropLeft + 1;
1434        displayHeight = cropBottom - cropTop + 1;
1435
1436        ALOGV("Video output format changed to %d x %d "
1437             "(crop: %d x %d @ (%d, %d))",
1438             width, height,
1439             displayWidth,
1440             displayHeight,
1441             cropLeft, cropTop);
1442    } else {
1443        CHECK(inputFormat->findInt32("width", &displayWidth));
1444        CHECK(inputFormat->findInt32("height", &displayHeight));
1445
1446        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1447    }
1448
1449    // Take into account sample aspect ratio if necessary:
1450    int32_t sarWidth, sarHeight;
1451    if (inputFormat->findInt32("sar-width", &sarWidth)
1452            && inputFormat->findInt32("sar-height", &sarHeight)) {
1453        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1454
1455        displayWidth = (displayWidth * sarWidth) / sarHeight;
1456
1457        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1458    }
1459
1460    int32_t rotationDegrees;
1461    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1462        rotationDegrees = 0;
1463    }
1464
1465    if (rotationDegrees == 90 || rotationDegrees == 270) {
1466        int32_t tmp = displayWidth;
1467        displayWidth = displayHeight;
1468        displayHeight = tmp;
1469    }
1470
1471    notifyListener(
1472            MEDIA_SET_VIDEO_SIZE,
1473            displayWidth,
1474            displayHeight);
1475}
1476
1477void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1478    if (mDriver == NULL) {
1479        return;
1480    }
1481
1482    sp<NuPlayerDriver> driver = mDriver.promote();
1483
1484    if (driver == NULL) {
1485        return;
1486    }
1487
1488    driver->notifyListener(msg, ext1, ext2, in);
1489}
1490
1491void NuPlayer::flushDecoder(bool audio, bool needShutdown) {
1492    ALOGV("[%s] flushDecoder needShutdown=%d",
1493          audio ? "audio" : "video", needShutdown);
1494
1495    if ((audio && mAudioDecoder == NULL) || (!audio && mVideoDecoder == NULL)) {
1496        ALOGI("flushDecoder %s without decoder present",
1497             audio ? "audio" : "video");
1498    }
1499
1500    // Make sure we don't continue to scan sources until we finish flushing.
1501    ++mScanSourcesGeneration;
1502    mScanSourcesPending = false;
1503
1504    (audio ? mAudioDecoder : mVideoDecoder)->signalFlush();
1505    mRenderer->flush(audio);
1506
1507    FlushStatus newStatus =
1508        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1509
1510    if (audio) {
1511        ALOGE_IF(mFlushingAudio != NONE,
1512                "audio flushDecoder() is called in state %d", mFlushingAudio);
1513        mFlushingAudio = newStatus;
1514    } else {
1515        ALOGE_IF(mFlushingVideo != NONE,
1516                "video flushDecoder() is called in state %d", mFlushingVideo);
1517        mFlushingVideo = newStatus;
1518    }
1519}
1520
1521void NuPlayer::queueDecoderShutdown(
1522        bool audio, bool video, const sp<AMessage> &reply) {
1523    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1524
1525    mDeferredActions.push_back(
1526            new ShutdownDecoderAction(audio, video));
1527
1528    mDeferredActions.push_back(
1529            new SimpleAction(&NuPlayer::performScanSources));
1530
1531    mDeferredActions.push_back(new PostMessageAction(reply));
1532
1533    processDeferredActions();
1534}
1535
1536status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1537    mVideoScalingMode = mode;
1538    if (mNativeWindow != NULL) {
1539        status_t ret = native_window_set_scaling_mode(
1540                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1541        if (ret != OK) {
1542            ALOGE("Failed to set scaling mode (%d): %s",
1543                -ret, strerror(-ret));
1544            return ret;
1545        }
1546    }
1547    return OK;
1548}
1549
1550status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1551    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id());
1552    msg->setPointer("reply", reply);
1553
1554    sp<AMessage> response;
1555    status_t err = msg->postAndAwaitResponse(&response);
1556    return err;
1557}
1558
1559status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1560    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id());
1561    msg->setPointer("reply", reply);
1562    msg->setInt32("type", type);
1563
1564    sp<AMessage> response;
1565    status_t err = msg->postAndAwaitResponse(&response);
1566    if (err == OK && response != NULL) {
1567        CHECK(response->findInt32("err", &err));
1568    }
1569    return err;
1570}
1571
1572status_t NuPlayer::selectTrack(size_t trackIndex, bool select) {
1573    sp<AMessage> msg = new AMessage(kWhatSelectTrack, id());
1574    msg->setSize("trackIndex", trackIndex);
1575    msg->setInt32("select", select);
1576
1577    sp<AMessage> response;
1578    status_t err = msg->postAndAwaitResponse(&response);
1579
1580    if (err != OK) {
1581        return err;
1582    }
1583
1584    if (!response->findInt32("err", &err)) {
1585        err = OK;
1586    }
1587
1588    return err;
1589}
1590
1591void NuPlayer::schedulePollDuration() {
1592    sp<AMessage> msg = new AMessage(kWhatPollDuration, id());
1593    msg->setInt32("generation", mPollDurationGeneration);
1594    msg->post();
1595}
1596
1597void NuPlayer::cancelPollDuration() {
1598    ++mPollDurationGeneration;
1599}
1600
1601void NuPlayer::processDeferredActions() {
1602    while (!mDeferredActions.empty()) {
1603        // We won't execute any deferred actions until we're no longer in
1604        // an intermediate state, i.e. one more more decoders are currently
1605        // flushing or shutting down.
1606
1607        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1608            // We're currently flushing, postpone the reset until that's
1609            // completed.
1610
1611            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1612                  mFlushingAudio, mFlushingVideo);
1613
1614            break;
1615        }
1616
1617        sp<Action> action = *mDeferredActions.begin();
1618        mDeferredActions.erase(mDeferredActions.begin());
1619
1620        action->execute(this);
1621    }
1622}
1623
1624void NuPlayer::performSeek(int64_t seekTimeUs) {
1625    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs)",
1626          seekTimeUs,
1627          seekTimeUs / 1E6);
1628
1629    mSource->seekTo(seekTimeUs);
1630    ++mTimedTextGeneration;
1631
1632    if (mDriver != NULL) {
1633        sp<NuPlayerDriver> driver = mDriver.promote();
1634        if (driver != NULL) {
1635            driver->notifyPosition(seekTimeUs);
1636            driver->notifySeekComplete();
1637        }
1638    }
1639
1640    // everything's flushed, continue playback.
1641}
1642
1643void NuPlayer::performDecoderFlush() {
1644    ALOGV("performDecoderFlush");
1645
1646    if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
1647        return;
1648    }
1649
1650    mTimeDiscontinuityPending = true;
1651
1652    if (mAudioDecoder != NULL) {
1653        flushDecoder(true /* audio */, false /* needShutdown */);
1654    }
1655
1656    if (mVideoDecoder != NULL) {
1657        flushDecoder(false /* audio */, false /* needShutdown */);
1658    }
1659}
1660
1661void NuPlayer::performDecoderShutdown(bool audio, bool video) {
1662    ALOGV("performDecoderShutdown audio=%d, video=%d", audio, video);
1663
1664    if ((!audio || mAudioDecoder == NULL)
1665            && (!video || mVideoDecoder == NULL)) {
1666        return;
1667    }
1668
1669    mTimeDiscontinuityPending = true;
1670
1671    if (audio && mAudioDecoder != NULL) {
1672        flushDecoder(true /* audio */, true /* needShutdown */);
1673    }
1674
1675    if (video && mVideoDecoder != NULL) {
1676        flushDecoder(false /* audio */, true /* needShutdown */);
1677    }
1678}
1679
1680void NuPlayer::performReset() {
1681    ALOGV("performReset");
1682
1683    CHECK(mAudioDecoder == NULL);
1684    CHECK(mVideoDecoder == NULL);
1685
1686    cancelPollDuration();
1687
1688    ++mScanSourcesGeneration;
1689    mScanSourcesPending = false;
1690
1691    if (mRendererLooper != NULL) {
1692        if (mRenderer != NULL) {
1693            mRendererLooper->unregisterHandler(mRenderer->id());
1694        }
1695        mRendererLooper->stop();
1696        mRendererLooper.clear();
1697    }
1698    mRenderer.clear();
1699
1700    if (mSource != NULL) {
1701        mSource->stop();
1702
1703        mSource.clear();
1704    }
1705
1706    if (mDriver != NULL) {
1707        sp<NuPlayerDriver> driver = mDriver.promote();
1708        if (driver != NULL) {
1709            driver->notifyResetComplete();
1710        }
1711    }
1712
1713    mStarted = false;
1714}
1715
1716void NuPlayer::performScanSources() {
1717    ALOGV("performScanSources");
1718
1719    if (!mStarted) {
1720        return;
1721    }
1722
1723    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1724        postScanSources();
1725    }
1726}
1727
1728void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1729    ALOGV("performSetSurface");
1730
1731    mNativeWindow = wrapper;
1732
1733    // XXX - ignore error from setVideoScalingMode for now
1734    setVideoScalingMode(mVideoScalingMode);
1735
1736    if (mDriver != NULL) {
1737        sp<NuPlayerDriver> driver = mDriver.promote();
1738        if (driver != NULL) {
1739            driver->notifySetSurfaceComplete();
1740        }
1741    }
1742}
1743
1744void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1745    int32_t what;
1746    CHECK(msg->findInt32("what", &what));
1747
1748    switch (what) {
1749        case Source::kWhatPrepared:
1750        {
1751            if (mSource == NULL) {
1752                // This is a stale notification from a source that was
1753                // asynchronously preparing when the client called reset().
1754                // We handled the reset, the source is gone.
1755                break;
1756            }
1757
1758            int32_t err;
1759            CHECK(msg->findInt32("err", &err));
1760
1761            sp<NuPlayerDriver> driver = mDriver.promote();
1762            if (driver != NULL) {
1763                // notify duration first, so that it's definitely set when
1764                // the app received the "prepare complete" callback.
1765                int64_t durationUs;
1766                if (mSource->getDuration(&durationUs) == OK) {
1767                    driver->notifyDuration(durationUs);
1768                }
1769                driver->notifyPrepareCompleted(err);
1770            }
1771
1772            break;
1773        }
1774
1775        case Source::kWhatFlagsChanged:
1776        {
1777            uint32_t flags;
1778            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1779
1780            sp<NuPlayerDriver> driver = mDriver.promote();
1781            if (driver != NULL) {
1782                driver->notifyFlagsChanged(flags);
1783            }
1784
1785            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1786                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
1787                cancelPollDuration();
1788            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1789                    && (flags & Source::FLAG_DYNAMIC_DURATION)
1790                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
1791                schedulePollDuration();
1792            }
1793
1794            mSourceFlags = flags;
1795            break;
1796        }
1797
1798        case Source::kWhatVideoSizeChanged:
1799        {
1800            sp<AMessage> format;
1801            CHECK(msg->findMessage("format", &format));
1802
1803            updateVideoSize(format);
1804            break;
1805        }
1806
1807        case Source::kWhatBufferingStart:
1808        {
1809            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
1810            break;
1811        }
1812
1813        case Source::kWhatBufferingEnd:
1814        {
1815            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
1816            break;
1817        }
1818
1819        case Source::kWhatSubtitleData:
1820        {
1821            sp<ABuffer> buffer;
1822            CHECK(msg->findBuffer("buffer", &buffer));
1823
1824            sendSubtitleData(buffer, 0 /* baseIndex */);
1825            break;
1826        }
1827
1828        case Source::kWhatTimedTextData:
1829        {
1830            int32_t generation;
1831            if (msg->findInt32("generation", &generation)
1832                    && generation != mTimedTextGeneration) {
1833                break;
1834            }
1835
1836            sp<ABuffer> buffer;
1837            CHECK(msg->findBuffer("buffer", &buffer));
1838
1839            sp<NuPlayerDriver> driver = mDriver.promote();
1840            if (driver == NULL) {
1841                break;
1842            }
1843
1844            int posMs;
1845            int64_t timeUs, posUs;
1846            driver->getCurrentPosition(&posMs);
1847            posUs = posMs * 1000;
1848            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1849
1850            if (posUs < timeUs) {
1851                if (!msg->findInt32("generation", &generation)) {
1852                    msg->setInt32("generation", mTimedTextGeneration);
1853                }
1854                msg->post(timeUs - posUs);
1855            } else {
1856                sendTimedTextData(buffer);
1857            }
1858            break;
1859        }
1860
1861        case Source::kWhatQueueDecoderShutdown:
1862        {
1863            int32_t audio, video;
1864            CHECK(msg->findInt32("audio", &audio));
1865            CHECK(msg->findInt32("video", &video));
1866
1867            sp<AMessage> reply;
1868            CHECK(msg->findMessage("reply", &reply));
1869
1870            queueDecoderShutdown(audio, video, reply);
1871            break;
1872        }
1873
1874        default:
1875            TRESPASS();
1876    }
1877}
1878
1879void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
1880    int32_t what;
1881    CHECK(msg->findInt32("what", &what));
1882
1883    switch (what) {
1884        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
1885        {
1886            sp<ABuffer> buffer;
1887            CHECK(msg->findBuffer("buffer", &buffer));
1888
1889            size_t inbandTracks = 0;
1890            if (mSource != NULL) {
1891                inbandTracks = mSource->getTrackCount();
1892            }
1893
1894            sendSubtitleData(buffer, inbandTracks);
1895            break;
1896        }
1897
1898        case NuPlayer::CCDecoder::kWhatTrackAdded:
1899        {
1900            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
1901
1902            break;
1903        }
1904
1905        default:
1906            TRESPASS();
1907    }
1908
1909
1910}
1911
1912void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
1913    int32_t trackIndex;
1914    int64_t timeUs, durationUs;
1915    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
1916    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1917    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
1918
1919    Parcel in;
1920    in.writeInt32(trackIndex + baseIndex);
1921    in.writeInt64(timeUs);
1922    in.writeInt64(durationUs);
1923    in.writeInt32(buffer->size());
1924    in.writeInt32(buffer->size());
1925    in.write(buffer->data(), buffer->size());
1926
1927    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
1928}
1929
1930void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
1931    const void *data;
1932    size_t size = 0;
1933    int64_t timeUs;
1934    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
1935
1936    AString mime;
1937    CHECK(buffer->meta()->findString("mime", &mime));
1938    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
1939
1940    data = buffer->data();
1941    size = buffer->size();
1942
1943    Parcel parcel;
1944    if (size > 0) {
1945        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1946        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
1947        TextDescriptions::getParcelOfDescriptions(
1948                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
1949    }
1950
1951    if ((parcel.dataSize() > 0)) {
1952        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
1953    } else {  // send an empty timed text
1954        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
1955    }
1956}
1957////////////////////////////////////////////////////////////////////////////////
1958
1959sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
1960    sp<MetaData> meta = getFormatMeta(audio);
1961
1962    if (meta == NULL) {
1963        return NULL;
1964    }
1965
1966    sp<AMessage> msg = new AMessage;
1967
1968    if(convertMetaDataToMessage(meta, &msg) == OK) {
1969        return msg;
1970    }
1971    return NULL;
1972}
1973
1974void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
1975    sp<AMessage> notify = dupNotify();
1976    notify->setInt32("what", kWhatFlagsChanged);
1977    notify->setInt32("flags", flags);
1978    notify->post();
1979}
1980
1981void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
1982    sp<AMessage> notify = dupNotify();
1983    notify->setInt32("what", kWhatVideoSizeChanged);
1984    notify->setMessage("format", format);
1985    notify->post();
1986}
1987
1988void NuPlayer::Source::notifyPrepared(status_t err) {
1989    sp<AMessage> notify = dupNotify();
1990    notify->setInt32("what", kWhatPrepared);
1991    notify->setInt32("err", err);
1992    notify->post();
1993}
1994
1995void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
1996    TRESPASS();
1997}
1998
1999}  // namespace android
2000