NuPlayer.cpp revision 5596d7c4ad388d1757398181b3a1453d731a1b41 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerDecoder.h"
25#include "NuPlayerDecoderPassThrough.h"
26#include "NuPlayerDriver.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29#include "RTSPSource.h"
30#include "StreamingSource.h"
31#include "GenericSource.h"
32#include "TextDescriptions.h"
33
34#include "ATSParser.h"
35
36#include <media/stagefright/foundation/hexdump.h>
37#include <media/stagefright/foundation/ABuffer.h>
38#include <media/stagefright/foundation/ADebug.h>
39#include <media/stagefright/foundation/AMessage.h>
40#include <media/stagefright/MediaBuffer.h>
41#include <media/stagefright/MediaDefs.h>
42#include <media/stagefright/MediaErrors.h>
43#include <media/stagefright/MetaData.h>
44#include <gui/IGraphicBufferProducer.h>
45
46#include "avc_utils.h"
47
48#include "ESDS.h"
49#include <media/stagefright/Utils.h>
50
51namespace android {
52
53struct NuPlayer::Action : public RefBase {
54    Action() {}
55
56    virtual void execute(NuPlayer *player) = 0;
57
58private:
59    DISALLOW_EVIL_CONSTRUCTORS(Action);
60};
61
62struct NuPlayer::SeekAction : public Action {
63    SeekAction(int64_t seekTimeUs)
64        : mSeekTimeUs(seekTimeUs) {
65    }
66
67    virtual void execute(NuPlayer *player) {
68        player->performSeek(mSeekTimeUs);
69    }
70
71private:
72    int64_t mSeekTimeUs;
73
74    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
75};
76
77struct NuPlayer::SetSurfaceAction : public Action {
78    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
79        : mWrapper(wrapper) {
80    }
81
82    virtual void execute(NuPlayer *player) {
83        player->performSetSurface(mWrapper);
84    }
85
86private:
87    sp<NativeWindowWrapper> mWrapper;
88
89    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
90};
91
92struct NuPlayer::ShutdownDecoderAction : public Action {
93    ShutdownDecoderAction(bool audio, bool video)
94        : mAudio(audio),
95          mVideo(video) {
96    }
97
98    virtual void execute(NuPlayer *player) {
99        player->performDecoderShutdown(mAudio, mVideo);
100    }
101
102private:
103    bool mAudio;
104    bool mVideo;
105
106    DISALLOW_EVIL_CONSTRUCTORS(ShutdownDecoderAction);
107};
108
109struct NuPlayer::PostMessageAction : public Action {
110    PostMessageAction(const sp<AMessage> &msg)
111        : mMessage(msg) {
112    }
113
114    virtual void execute(NuPlayer *) {
115        mMessage->post();
116    }
117
118private:
119    sp<AMessage> mMessage;
120
121    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
122};
123
124// Use this if there's no state necessary to save in order to execute
125// the action.
126struct NuPlayer::SimpleAction : public Action {
127    typedef void (NuPlayer::*ActionFunc)();
128
129    SimpleAction(ActionFunc func)
130        : mFunc(func) {
131    }
132
133    virtual void execute(NuPlayer *player) {
134        (player->*mFunc)();
135    }
136
137private:
138    ActionFunc mFunc;
139
140    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
141};
142
143////////////////////////////////////////////////////////////////////////////////
144
145NuPlayer::NuPlayer()
146    : mUIDValid(false),
147      mSourceFlags(0),
148      mVideoIsAVC(false),
149      mOffloadAudio(false),
150      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
151      mAudioDecoderGeneration(0),
152      mVideoDecoderGeneration(0),
153      mAudioEOS(false),
154      mVideoEOS(false),
155      mScanSourcesPending(false),
156      mScanSourcesGeneration(0),
157      mPollDurationGeneration(0),
158      mTimedTextGeneration(0),
159      mTimeDiscontinuityPending(false),
160      mFlushingAudio(NONE),
161      mFlushingVideo(NONE),
162      mSkipRenderingAudioUntilMediaTimeUs(-1ll),
163      mSkipRenderingVideoUntilMediaTimeUs(-1ll),
164      mVideoLateByUs(0ll),
165      mNumFramesTotal(0ll),
166      mNumFramesDropped(0ll),
167      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
168      mStarted(false) {
169}
170
171NuPlayer::~NuPlayer() {
172}
173
174void NuPlayer::setUID(uid_t uid) {
175    mUIDValid = true;
176    mUID = uid;
177}
178
179void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
180    mDriver = driver;
181}
182
183void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
184    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
185
186    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
187
188    msg->setObject("source", new StreamingSource(notify, source));
189    msg->post();
190}
191
192static bool IsHTTPLiveURL(const char *url) {
193    if (!strncasecmp("http://", url, 7)
194            || !strncasecmp("https://", url, 8)
195            || !strncasecmp("file://", url, 7)) {
196        size_t len = strlen(url);
197        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
198            return true;
199        }
200
201        if (strstr(url,"m3u8")) {
202            return true;
203        }
204    }
205
206    return false;
207}
208
209void NuPlayer::setDataSourceAsync(
210        const sp<IMediaHTTPService> &httpService,
211        const char *url,
212        const KeyedVector<String8, String8> *headers) {
213
214    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
215    size_t len = strlen(url);
216
217    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
218
219    sp<Source> source;
220    if (IsHTTPLiveURL(url)) {
221        source = new HTTPLiveSource(notify, httpService, url, headers);
222    } else if (!strncasecmp(url, "rtsp://", 7)) {
223        source = new RTSPSource(
224                notify, httpService, url, headers, mUIDValid, mUID);
225    } else if ((!strncasecmp(url, "http://", 7)
226                || !strncasecmp(url, "https://", 8))
227                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
228                    || strstr(url, ".sdp?"))) {
229        source = new RTSPSource(
230                notify, httpService, url, headers, mUIDValid, mUID, true);
231    } else {
232        sp<GenericSource> genericSource =
233                new GenericSource(notify, mUIDValid, mUID);
234        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
235        // The correct flags will be updated in Source::kWhatFlagsChanged
236        // handler when  GenericSource is prepared.
237
238        status_t err = genericSource->setDataSource(httpService, url, headers);
239
240        if (err == OK) {
241            source = genericSource;
242        } else {
243            ALOGE("Failed to set data source!");
244        }
245    }
246    msg->setObject("source", source);
247    msg->post();
248}
249
250void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
251    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
252
253    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
254
255    sp<GenericSource> source =
256            new GenericSource(notify, mUIDValid, mUID);
257
258    status_t err = source->setDataSource(fd, offset, length);
259
260    if (err != OK) {
261        ALOGE("Failed to set data source!");
262        source = NULL;
263    }
264
265    msg->setObject("source", source);
266    msg->post();
267}
268
269void NuPlayer::prepareAsync() {
270    (new AMessage(kWhatPrepare, id()))->post();
271}
272
273void NuPlayer::setVideoSurfaceTextureAsync(
274        const sp<IGraphicBufferProducer> &bufferProducer) {
275    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id());
276
277    if (bufferProducer == NULL) {
278        msg->setObject("native-window", NULL);
279    } else {
280        msg->setObject(
281                "native-window",
282                new NativeWindowWrapper(
283                    new Surface(bufferProducer)));
284    }
285
286    msg->post();
287}
288
289void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
290    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id());
291    msg->setObject("sink", sink);
292    msg->post();
293}
294
295void NuPlayer::start() {
296    (new AMessage(kWhatStart, id()))->post();
297}
298
299void NuPlayer::pause() {
300    (new AMessage(kWhatPause, id()))->post();
301}
302
303void NuPlayer::resume() {
304    (new AMessage(kWhatResume, id()))->post();
305}
306
307void NuPlayer::resetAsync() {
308    (new AMessage(kWhatReset, id()))->post();
309}
310
311void NuPlayer::seekToAsync(int64_t seekTimeUs) {
312    sp<AMessage> msg = new AMessage(kWhatSeek, id());
313    msg->setInt64("seekTimeUs", seekTimeUs);
314    msg->post();
315}
316
317// static
318bool NuPlayer::IsFlushingState(FlushStatus state, bool *needShutdown) {
319    switch (state) {
320        case FLUSHING_DECODER:
321            if (needShutdown != NULL) {
322                *needShutdown = false;
323            }
324            return true;
325
326        case FLUSHING_DECODER_SHUTDOWN:
327            if (needShutdown != NULL) {
328                *needShutdown = true;
329            }
330            return true;
331
332        default:
333            return false;
334    }
335}
336
337void NuPlayer::writeTrackInfo(
338        Parcel* reply, const sp<AMessage> format) const {
339    int32_t trackType;
340    CHECK(format->findInt32("type", &trackType));
341
342    AString lang;
343    CHECK(format->findString("language", &lang));
344
345    reply->writeInt32(2); // write something non-zero
346    reply->writeInt32(trackType);
347    reply->writeString16(String16(lang.c_str()));
348
349    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
350        AString mime;
351        CHECK(format->findString("mime", &mime));
352
353        int32_t isAuto, isDefault, isForced;
354        CHECK(format->findInt32("auto", &isAuto));
355        CHECK(format->findInt32("default", &isDefault));
356        CHECK(format->findInt32("forced", &isForced));
357
358        reply->writeString16(String16(mime.c_str()));
359        reply->writeInt32(isAuto);
360        reply->writeInt32(isDefault);
361        reply->writeInt32(isForced);
362    }
363}
364
365void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
366    switch (msg->what()) {
367        case kWhatSetDataSource:
368        {
369            ALOGV("kWhatSetDataSource");
370
371            CHECK(mSource == NULL);
372
373            status_t err = OK;
374            sp<RefBase> obj;
375            CHECK(msg->findObject("source", &obj));
376            if (obj != NULL) {
377                mSource = static_cast<Source *>(obj.get());
378            } else {
379                err = UNKNOWN_ERROR;
380            }
381
382            CHECK(mDriver != NULL);
383            sp<NuPlayerDriver> driver = mDriver.promote();
384            if (driver != NULL) {
385                driver->notifySetDataSourceCompleted(err);
386            }
387            break;
388        }
389
390        case kWhatPrepare:
391        {
392            mSource->prepareAsync();
393            break;
394        }
395
396        case kWhatGetTrackInfo:
397        {
398            uint32_t replyID;
399            CHECK(msg->senderAwaitsResponse(&replyID));
400
401            Parcel* reply;
402            CHECK(msg->findPointer("reply", (void**)&reply));
403
404            size_t inbandTracks = 0;
405            if (mSource != NULL) {
406                inbandTracks = mSource->getTrackCount();
407            }
408
409            size_t ccTracks = 0;
410            if (mCCDecoder != NULL) {
411                ccTracks = mCCDecoder->getTrackCount();
412            }
413
414            // total track count
415            reply->writeInt32(inbandTracks + ccTracks);
416
417            // write inband tracks
418            for (size_t i = 0; i < inbandTracks; ++i) {
419                writeTrackInfo(reply, mSource->getTrackInfo(i));
420            }
421
422            // write CC track
423            for (size_t i = 0; i < ccTracks; ++i) {
424                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
425            }
426
427            sp<AMessage> response = new AMessage;
428            response->postReply(replyID);
429            break;
430        }
431
432        case kWhatGetSelectedTrack:
433        {
434            status_t err = INVALID_OPERATION;
435            if (mSource != NULL) {
436                err = OK;
437
438                int32_t type32;
439                CHECK(msg->findInt32("type", (int32_t*)&type32));
440                media_track_type type = (media_track_type)type32;
441                ssize_t selectedTrack = mSource->getSelectedTrack(type);
442
443                Parcel* reply;
444                CHECK(msg->findPointer("reply", (void**)&reply));
445                reply->writeInt32(selectedTrack);
446            }
447
448            sp<AMessage> response = new AMessage;
449            response->setInt32("err", err);
450
451            uint32_t replyID;
452            CHECK(msg->senderAwaitsResponse(&replyID));
453            response->postReply(replyID);
454            break;
455        }
456
457        case kWhatSelectTrack:
458        {
459            uint32_t replyID;
460            CHECK(msg->senderAwaitsResponse(&replyID));
461
462            size_t trackIndex;
463            int32_t select;
464            CHECK(msg->findSize("trackIndex", &trackIndex));
465            CHECK(msg->findInt32("select", &select));
466
467            status_t err = INVALID_OPERATION;
468
469            size_t inbandTracks = 0;
470            if (mSource != NULL) {
471                inbandTracks = mSource->getTrackCount();
472            }
473            size_t ccTracks = 0;
474            if (mCCDecoder != NULL) {
475                ccTracks = mCCDecoder->getTrackCount();
476            }
477
478            if (trackIndex < inbandTracks) {
479                err = mSource->selectTrack(trackIndex, select);
480
481                if (!select && err == OK) {
482                    int32_t type;
483                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
484                    if (info != NULL
485                            && info->findInt32("type", &type)
486                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
487                        ++mTimedTextGeneration;
488                    }
489                }
490            } else {
491                trackIndex -= inbandTracks;
492
493                if (trackIndex < ccTracks) {
494                    err = mCCDecoder->selectTrack(trackIndex, select);
495                }
496            }
497
498            sp<AMessage> response = new AMessage;
499            response->setInt32("err", err);
500
501            response->postReply(replyID);
502            break;
503        }
504
505        case kWhatPollDuration:
506        {
507            int32_t generation;
508            CHECK(msg->findInt32("generation", &generation));
509
510            if (generation != mPollDurationGeneration) {
511                // stale
512                break;
513            }
514
515            int64_t durationUs;
516            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
517                sp<NuPlayerDriver> driver = mDriver.promote();
518                if (driver != NULL) {
519                    driver->notifyDuration(durationUs);
520                }
521            }
522
523            msg->post(1000000ll);  // poll again in a second.
524            break;
525        }
526
527        case kWhatSetVideoNativeWindow:
528        {
529            ALOGV("kWhatSetVideoNativeWindow");
530
531            mDeferredActions.push_back(
532                    new ShutdownDecoderAction(
533                        false /* audio */, true /* video */));
534
535            sp<RefBase> obj;
536            CHECK(msg->findObject("native-window", &obj));
537
538            mDeferredActions.push_back(
539                    new SetSurfaceAction(
540                        static_cast<NativeWindowWrapper *>(obj.get())));
541
542            if (obj != NULL) {
543                // If there is a new surface texture, instantiate decoders
544                // again if possible.
545                mDeferredActions.push_back(
546                        new SimpleAction(&NuPlayer::performScanSources));
547            }
548
549            processDeferredActions();
550            break;
551        }
552
553        case kWhatSetAudioSink:
554        {
555            ALOGV("kWhatSetAudioSink");
556
557            sp<RefBase> obj;
558            CHECK(msg->findObject("sink", &obj));
559
560            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
561            break;
562        }
563
564        case kWhatStart:
565        {
566            ALOGV("kWhatStart");
567
568            mVideoIsAVC = false;
569            mOffloadAudio = false;
570            mAudioEOS = false;
571            mVideoEOS = false;
572            mSkipRenderingAudioUntilMediaTimeUs = -1;
573            mSkipRenderingVideoUntilMediaTimeUs = -1;
574            mVideoLateByUs = 0;
575            mNumFramesTotal = 0;
576            mNumFramesDropped = 0;
577            mStarted = true;
578
579            /* instantiate decoders now for secure playback */
580            if (mSourceFlags & Source::FLAG_SECURE) {
581                if (mNativeWindow != NULL) {
582                    instantiateDecoder(false, &mVideoDecoder);
583                }
584
585                if (mAudioSink != NULL) {
586                    instantiateDecoder(true, &mAudioDecoder);
587                }
588            }
589
590            mSource->start();
591
592            uint32_t flags = 0;
593
594            if (mSource->isRealTime()) {
595                flags |= Renderer::FLAG_REAL_TIME;
596            }
597
598            sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
599            audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
600            if (mAudioSink != NULL) {
601                streamType = mAudioSink->getAudioStreamType();
602            }
603
604            sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
605
606            mOffloadAudio =
607                canOffloadStream(audioMeta, (videoFormat != NULL),
608                                 true /* is_streaming */, streamType);
609            if (mOffloadAudio) {
610                flags |= Renderer::FLAG_OFFLOAD_AUDIO;
611            }
612
613            mRenderer = new Renderer(
614                    mAudioSink,
615                    new AMessage(kWhatRendererNotify, id()),
616                    flags);
617
618            mRendererLooper = new ALooper;
619            mRendererLooper->setName("NuPlayerRenderer");
620            mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
621            mRendererLooper->registerHandler(mRenderer);
622
623            postScanSources();
624            break;
625        }
626
627        case kWhatScanSources:
628        {
629            int32_t generation;
630            CHECK(msg->findInt32("generation", &generation));
631            if (generation != mScanSourcesGeneration) {
632                // Drop obsolete msg.
633                break;
634            }
635
636            mScanSourcesPending = false;
637
638            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
639                 mAudioDecoder != NULL, mVideoDecoder != NULL);
640
641            bool mHadAnySourcesBefore =
642                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
643
644            // initialize video before audio because successful initialization of
645            // video may change deep buffer mode of audio.
646            if (mNativeWindow != NULL) {
647                instantiateDecoder(false, &mVideoDecoder);
648            }
649
650            if (mAudioSink != NULL) {
651                if (mOffloadAudio) {
652                    // open audio sink early under offload mode.
653                    sp<AMessage> format = mSource->getFormat(true /*audio*/);
654                    openAudioSink(format, true /*offloadOnly*/);
655                }
656                instantiateDecoder(true, &mAudioDecoder);
657            }
658
659            if (!mHadAnySourcesBefore
660                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
661                // This is the first time we've found anything playable.
662
663                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
664                    schedulePollDuration();
665                }
666            }
667
668            status_t err;
669            if ((err = mSource->feedMoreTSData()) != OK) {
670                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
671                    // We're not currently decoding anything (no audio or
672                    // video tracks found) and we just ran out of input data.
673
674                    if (err == ERROR_END_OF_STREAM) {
675                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
676                    } else {
677                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
678                    }
679                }
680                break;
681            }
682
683            if ((mAudioDecoder == NULL && mAudioSink != NULL)
684                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
685                msg->post(100000ll);
686                mScanSourcesPending = true;
687            }
688            break;
689        }
690
691        case kWhatVideoNotify:
692        case kWhatAudioNotify:
693        {
694            bool audio = msg->what() == kWhatAudioNotify;
695
696            int32_t currentDecoderGeneration =
697                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
698            int32_t requesterGeneration = currentDecoderGeneration - 1;
699            CHECK(msg->findInt32("generation", &requesterGeneration));
700
701            if (requesterGeneration != currentDecoderGeneration) {
702                ALOGV("got message from old %s decoder, generation(%d:%d)",
703                        audio ? "audio" : "video", requesterGeneration,
704                        currentDecoderGeneration);
705                sp<AMessage> reply;
706                if (!(msg->findMessage("reply", &reply))) {
707                    return;
708                }
709
710                reply->setInt32("err", INFO_DISCONTINUITY);
711                reply->post();
712                return;
713            }
714
715            int32_t what;
716            CHECK(msg->findInt32("what", &what));
717
718            if (what == Decoder::kWhatFillThisBuffer) {
719                status_t err = feedDecoderInputData(
720                        audio, msg);
721
722                if (err == -EWOULDBLOCK) {
723                    if (mSource->feedMoreTSData() == OK) {
724                        msg->post(10000ll);
725                    }
726                }
727            } else if (what == Decoder::kWhatEOS) {
728                int32_t err;
729                CHECK(msg->findInt32("err", &err));
730
731                if (err == ERROR_END_OF_STREAM) {
732                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
733                } else {
734                    ALOGV("got %s decoder EOS w/ error %d",
735                         audio ? "audio" : "video",
736                         err);
737                }
738
739                mRenderer->queueEOS(audio, err);
740            } else if (what == Decoder::kWhatFlushCompleted) {
741                bool needShutdown;
742
743                if (audio) {
744                    CHECK(IsFlushingState(mFlushingAudio, &needShutdown));
745                    mFlushingAudio = FLUSHED;
746                } else {
747                    CHECK(IsFlushingState(mFlushingVideo, &needShutdown));
748                    mFlushingVideo = FLUSHED;
749
750                    mVideoLateByUs = 0;
751                }
752
753                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
754
755                if (needShutdown) {
756                    ALOGV("initiating %s decoder shutdown",
757                         audio ? "audio" : "video");
758
759                    (audio ? mAudioDecoder : mVideoDecoder)->initiateShutdown();
760
761                    if (audio) {
762                        mFlushingAudio = SHUTTING_DOWN_DECODER;
763                    } else {
764                        mFlushingVideo = SHUTTING_DOWN_DECODER;
765                    }
766                }
767
768                finishFlushIfPossible();
769            } else if (what == Decoder::kWhatOutputFormatChanged) {
770                sp<AMessage> format;
771                CHECK(msg->findMessage("format", &format));
772
773                if (audio) {
774                    openAudioSink(format, false /*offloadOnly*/);
775                } else {
776                    // video
777                    sp<AMessage> inputFormat =
778                            mSource->getFormat(false /* audio */);
779
780                    updateVideoSize(inputFormat, format);
781                }
782            } else if (what == Decoder::kWhatShutdownCompleted) {
783                ALOGV("%s shutdown completed", audio ? "audio" : "video");
784                if (audio) {
785                    mAudioDecoder.clear();
786
787                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
788                    mFlushingAudio = SHUT_DOWN;
789                } else {
790                    mVideoDecoder.clear();
791
792                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
793                    mFlushingVideo = SHUT_DOWN;
794                }
795
796                finishFlushIfPossible();
797            } else if (what == Decoder::kWhatError) {
798                ALOGE("Received error from %s decoder, aborting playback.",
799                     audio ? "audio" : "video");
800
801                status_t err;
802                if (!msg->findInt32("err", &err)) {
803                    err = UNKNOWN_ERROR;
804                }
805                mRenderer->queueEOS(audio, err);
806            } else if (what == Decoder::kWhatDrainThisBuffer) {
807                renderBuffer(audio, msg);
808            } else {
809                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
810                      what,
811                      what >> 24,
812                      (what >> 16) & 0xff,
813                      (what >> 8) & 0xff,
814                      what & 0xff);
815            }
816
817            break;
818        }
819
820        case kWhatRendererNotify:
821        {
822            int32_t what;
823            CHECK(msg->findInt32("what", &what));
824
825            if (what == Renderer::kWhatEOS) {
826                int32_t audio;
827                CHECK(msg->findInt32("audio", &audio));
828
829                int32_t finalResult;
830                CHECK(msg->findInt32("finalResult", &finalResult));
831
832                if (audio) {
833                    mAudioEOS = true;
834                } else {
835                    mVideoEOS = true;
836                }
837
838                if (finalResult == ERROR_END_OF_STREAM) {
839                    ALOGV("reached %s EOS", audio ? "audio" : "video");
840                } else {
841                    ALOGE("%s track encountered an error (%d)",
842                         audio ? "audio" : "video", finalResult);
843
844                    notifyListener(
845                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
846                }
847
848                if ((mAudioEOS || mAudioDecoder == NULL)
849                        && (mVideoEOS || mVideoDecoder == NULL)) {
850                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
851                }
852            } else if (what == Renderer::kWhatPosition) {
853                int64_t positionUs;
854                CHECK(msg->findInt64("positionUs", &positionUs));
855
856                CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs));
857
858                if (mDriver != NULL) {
859                    sp<NuPlayerDriver> driver = mDriver.promote();
860                    if (driver != NULL) {
861                        driver->notifyPosition(positionUs);
862
863                        driver->notifyFrameStats(
864                                mNumFramesTotal, mNumFramesDropped);
865                    }
866                }
867            } else if (what == Renderer::kWhatFlushComplete) {
868                int32_t audio;
869                CHECK(msg->findInt32("audio", &audio));
870
871                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
872            } else if (what == Renderer::kWhatVideoRenderingStart) {
873                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
874            } else if (what == Renderer::kWhatMediaRenderingStart) {
875                ALOGV("media rendering started");
876                notifyListener(MEDIA_STARTED, 0, 0);
877            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
878                ALOGV("Tear down audio offload, fall back to s/w path");
879                int64_t positionUs;
880                CHECK(msg->findInt64("positionUs", &positionUs));
881                closeAudioSink();
882                mAudioDecoder.clear();
883                mRenderer->flush(true /* audio */);
884                if (mVideoDecoder != NULL) {
885                    mRenderer->flush(false /* audio */);
886                }
887                mRenderer->signalDisableOffloadAudio();
888                mOffloadAudio = false;
889
890                performSeek(positionUs);
891                instantiateDecoder(true /* audio */, &mAudioDecoder);
892            }
893            break;
894        }
895
896        case kWhatMoreDataQueued:
897        {
898            break;
899        }
900
901        case kWhatReset:
902        {
903            ALOGV("kWhatReset");
904
905            mDeferredActions.push_back(
906                    new ShutdownDecoderAction(
907                        true /* audio */, true /* video */));
908
909            mDeferredActions.push_back(
910                    new SimpleAction(&NuPlayer::performReset));
911
912            processDeferredActions();
913            break;
914        }
915
916        case kWhatSeek:
917        {
918            int64_t seekTimeUs;
919            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
920
921            ALOGV("kWhatSeek seekTimeUs=%lld us", seekTimeUs);
922
923            mDeferredActions.push_back(
924                    new SimpleAction(&NuPlayer::performDecoderFlush));
925
926            mDeferredActions.push_back(new SeekAction(seekTimeUs));
927
928            processDeferredActions();
929            break;
930        }
931
932        case kWhatPause:
933        {
934            CHECK(mRenderer != NULL);
935            mSource->pause();
936            mRenderer->pause();
937            break;
938        }
939
940        case kWhatResume:
941        {
942            CHECK(mRenderer != NULL);
943            mSource->resume();
944            mRenderer->resume();
945            break;
946        }
947
948        case kWhatSourceNotify:
949        {
950            onSourceNotify(msg);
951            break;
952        }
953
954        case kWhatClosedCaptionNotify:
955        {
956            onClosedCaptionNotify(msg);
957            break;
958        }
959
960        default:
961            TRESPASS();
962            break;
963    }
964}
965
966void NuPlayer::finishFlushIfPossible() {
967    if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED
968            && mFlushingAudio != SHUT_DOWN) {
969        return;
970    }
971
972    if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED
973            && mFlushingVideo != SHUT_DOWN) {
974        return;
975    }
976
977    ALOGV("both audio and video are flushed now.");
978
979    if (mTimeDiscontinuityPending) {
980        mRenderer->signalTimeDiscontinuity();
981        mTimeDiscontinuityPending = false;
982    }
983
984    if (mAudioDecoder != NULL && mFlushingAudio == FLUSHED) {
985        mAudioDecoder->signalResume();
986    }
987
988    if (mVideoDecoder != NULL && mFlushingVideo == FLUSHED) {
989        mVideoDecoder->signalResume();
990    }
991
992    mFlushingAudio = NONE;
993    mFlushingVideo = NONE;
994
995    processDeferredActions();
996}
997
998void NuPlayer::postScanSources() {
999    if (mScanSourcesPending) {
1000        return;
1001    }
1002
1003    sp<AMessage> msg = new AMessage(kWhatScanSources, id());
1004    msg->setInt32("generation", mScanSourcesGeneration);
1005    msg->post();
1006
1007    mScanSourcesPending = true;
1008}
1009
1010void NuPlayer::openAudioSink(const sp<AMessage> &format, bool offloadOnly) {
1011    ALOGV("openAudioSink: offloadOnly(%d) mOffloadAudio(%d)",
1012            offloadOnly, mOffloadAudio);
1013    bool audioSinkChanged = false;
1014
1015    int32_t numChannels;
1016    CHECK(format->findInt32("channel-count", &numChannels));
1017
1018    int32_t channelMask;
1019    if (!format->findInt32("channel-mask", &channelMask)) {
1020        // signal to the AudioSink to derive the mask from count.
1021        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1022    }
1023
1024    int32_t sampleRate;
1025    CHECK(format->findInt32("sample-rate", &sampleRate));
1026
1027    uint32_t flags;
1028    int64_t durationUs;
1029    // FIXME: we should handle the case where the video decoder
1030    // is created after we receive the format change indication.
1031    // Current code will just make that we select deep buffer
1032    // with video which should not be a problem as it should
1033    // not prevent from keeping A/V sync.
1034    if (mVideoDecoder == NULL &&
1035            mSource->getDuration(&durationUs) == OK &&
1036            durationUs
1037                > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
1038        flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1039    } else {
1040        flags = AUDIO_OUTPUT_FLAG_NONE;
1041    }
1042
1043    if (mOffloadAudio) {
1044        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1045        AString mime;
1046        CHECK(format->findString("mime", &mime));
1047        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1048
1049        if (err != OK) {
1050            ALOGE("Couldn't map mime \"%s\" to a valid "
1051                    "audio_format", mime.c_str());
1052            mOffloadAudio = false;
1053        } else {
1054            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1055                    mime.c_str(), audioFormat);
1056
1057            int avgBitRate = -1;
1058            format->findInt32("bit-rate", &avgBitRate);
1059
1060            int32_t aacProfile = -1;
1061            if (audioFormat == AUDIO_FORMAT_AAC
1062                    && format->findInt32("aac-profile", &aacProfile)) {
1063                // Redefine AAC format as per aac profile
1064                mapAACProfileToAudioFormat(
1065                        audioFormat,
1066                        aacProfile);
1067            }
1068
1069            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1070            offloadInfo.duration_us = -1;
1071            format->findInt64(
1072                    "durationUs", &offloadInfo.duration_us);
1073            offloadInfo.sample_rate = sampleRate;
1074            offloadInfo.channel_mask = channelMask;
1075            offloadInfo.format = audioFormat;
1076            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1077            offloadInfo.bit_rate = avgBitRate;
1078            offloadInfo.has_video = (mVideoDecoder != NULL);
1079            offloadInfo.is_streaming = true;
1080
1081            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1082                ALOGV("openAudioSink: no change in offload mode");
1083                return;  // no change from previous configuration, everything ok.
1084            }
1085            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1086            flags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1087            flags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1088            audioSinkChanged = true;
1089            mAudioSink->close();
1090            err = mAudioSink->open(
1091                    sampleRate,
1092                    numChannels,
1093                    (audio_channel_mask_t)channelMask,
1094                    audioFormat,
1095                    8 /* bufferCount */,
1096                    &NuPlayer::Renderer::AudioSinkCallback,
1097                    mRenderer.get(),
1098                    (audio_output_flags_t)flags,
1099                    &offloadInfo);
1100
1101            if (err == OK) {
1102                // If the playback is offloaded to h/w, we pass
1103                // the HAL some metadata information.
1104                // We don't want to do this for PCM because it
1105                // will be going through the AudioFlinger mixer
1106                // before reaching the hardware.
1107                sp<MetaData> audioMeta =
1108                        mSource->getFormatMeta(true /* audio */);
1109                sendMetaDataToHal(mAudioSink, audioMeta);
1110                mCurrentOffloadInfo = offloadInfo;
1111                err = mAudioSink->start();
1112                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1113            }
1114            if (err != OK) {
1115                // Clean up, fall back to non offload mode.
1116                mAudioSink->close();
1117                mRenderer->signalDisableOffloadAudio();
1118                mOffloadAudio = false;
1119                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1120                ALOGV("openAudioSink: offload failed");
1121            }
1122        }
1123    }
1124    if (!offloadOnly && !mOffloadAudio) {
1125        flags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1126        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1127
1128        audioSinkChanged = true;
1129        mAudioSink->close();
1130        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1131        CHECK_EQ(mAudioSink->open(
1132                    sampleRate,
1133                    numChannels,
1134                    (audio_channel_mask_t)channelMask,
1135                    AUDIO_FORMAT_PCM_16_BIT,
1136                    8 /* bufferCount */,
1137                    NULL,
1138                    NULL,
1139                    (audio_output_flags_t)flags),
1140                 (status_t)OK);
1141        mAudioSink->start();
1142    }
1143    if (audioSinkChanged) {
1144        mRenderer->signalAudioSinkChanged();
1145    }
1146}
1147
1148void NuPlayer::closeAudioSink() {
1149    mAudioSink->close();
1150    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1151}
1152
1153status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
1154    if (*decoder != NULL) {
1155        return OK;
1156    }
1157
1158    sp<AMessage> format = mSource->getFormat(audio);
1159
1160    if (format == NULL) {
1161        return -EWOULDBLOCK;
1162    }
1163
1164    if (!audio) {
1165        AString mime;
1166        CHECK(format->findString("mime", &mime));
1167        mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
1168
1169        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
1170        mCCDecoder = new CCDecoder(ccNotify);
1171
1172        if (mSourceFlags & Source::FLAG_SECURE) {
1173            format->setInt32("secure", true);
1174        }
1175    }
1176
1177    if (audio) {
1178        sp<AMessage> notify = new AMessage(kWhatAudioNotify, id());
1179        ++mAudioDecoderGeneration;
1180        notify->setInt32("generation", mAudioDecoderGeneration);
1181
1182        if (mOffloadAudio) {
1183            *decoder = new DecoderPassThrough(notify);
1184        } else {
1185            *decoder = new Decoder(notify);
1186        }
1187    } else {
1188        sp<AMessage> notify = new AMessage(kWhatVideoNotify, id());
1189        ++mVideoDecoderGeneration;
1190        notify->setInt32("generation", mVideoDecoderGeneration);
1191
1192        *decoder = new Decoder(notify, mNativeWindow);
1193    }
1194    (*decoder)->init();
1195    (*decoder)->configure(format);
1196
1197    // allocate buffers to decrypt widevine source buffers
1198    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1199        Vector<sp<ABuffer> > inputBufs;
1200        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1201
1202        Vector<MediaBuffer *> mediaBufs;
1203        for (size_t i = 0; i < inputBufs.size(); i++) {
1204            const sp<ABuffer> &buffer = inputBufs[i];
1205            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1206            mediaBufs.push(mbuf);
1207        }
1208
1209        status_t err = mSource->setBuffers(audio, mediaBufs);
1210        if (err != OK) {
1211            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1212                mediaBufs[i]->release();
1213            }
1214            mediaBufs.clear();
1215            ALOGE("Secure source didn't support secure mediaBufs.");
1216            return err;
1217        }
1218    }
1219    return OK;
1220}
1221
1222status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
1223    sp<AMessage> reply;
1224    CHECK(msg->findMessage("reply", &reply));
1225
1226    if ((audio && mFlushingAudio != NONE)
1227            || (!audio && mFlushingVideo != NONE)) {
1228        reply->setInt32("err", INFO_DISCONTINUITY);
1229        reply->post();
1230        return OK;
1231    }
1232
1233    sp<ABuffer> accessUnit;
1234
1235    bool dropAccessUnit;
1236    do {
1237        status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
1238
1239        if (err == -EWOULDBLOCK) {
1240            return err;
1241        } else if (err != OK) {
1242            if (err == INFO_DISCONTINUITY) {
1243                int32_t type;
1244                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
1245
1246                bool formatChange =
1247                    (audio &&
1248                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
1249                    || (!audio &&
1250                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
1251
1252                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
1253
1254                ALOGI("%s discontinuity (formatChange=%d, time=%d)",
1255                     audio ? "audio" : "video", formatChange, timeChange);
1256
1257                if (audio) {
1258                    mSkipRenderingAudioUntilMediaTimeUs = -1;
1259                } else {
1260                    mSkipRenderingVideoUntilMediaTimeUs = -1;
1261                }
1262
1263                if (timeChange) {
1264                    sp<AMessage> extra;
1265                    if (accessUnit->meta()->findMessage("extra", &extra)
1266                            && extra != NULL) {
1267                        int64_t resumeAtMediaTimeUs;
1268                        if (extra->findInt64(
1269                                    "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
1270                            ALOGI("suppressing rendering of %s until %lld us",
1271                                    audio ? "audio" : "video", resumeAtMediaTimeUs);
1272
1273                            if (audio) {
1274                                mSkipRenderingAudioUntilMediaTimeUs =
1275                                    resumeAtMediaTimeUs;
1276                            } else {
1277                                mSkipRenderingVideoUntilMediaTimeUs =
1278                                    resumeAtMediaTimeUs;
1279                            }
1280                        }
1281                    }
1282                }
1283
1284                mTimeDiscontinuityPending =
1285                    mTimeDiscontinuityPending || timeChange;
1286
1287                if (mFlushingAudio == NONE && mFlushingVideo == NONE) {
1288                    // And we'll resume scanning sources once we're done
1289                    // flushing.
1290                    mDeferredActions.push_front(
1291                            new SimpleAction(
1292                                &NuPlayer::performScanSources));
1293                }
1294
1295                if (formatChange || timeChange) {
1296
1297                    sp<AMessage> newFormat = mSource->getFormat(audio);
1298                    sp<Decoder> &decoder = audio ? mAudioDecoder : mVideoDecoder;
1299                    if (formatChange && !decoder->supportsSeamlessFormatChange(newFormat)) {
1300                        flushDecoder(audio, /* needShutdown = */ true);
1301                    } else {
1302                        flushDecoder(audio, /* needShutdown = */ false);
1303                        err = OK;
1304                    }
1305                } else {
1306                    // This stream is unaffected by the discontinuity
1307                    return -EWOULDBLOCK;
1308                }
1309            }
1310
1311            reply->setInt32("err", err);
1312            reply->post();
1313            return OK;
1314        }
1315
1316        if (!audio) {
1317            ++mNumFramesTotal;
1318        }
1319
1320        dropAccessUnit = false;
1321        if (!audio
1322                && !(mSourceFlags & Source::FLAG_SECURE)
1323                && mVideoLateByUs > 100000ll
1324                && mVideoIsAVC
1325                && !IsAVCReferenceFrame(accessUnit)) {
1326            dropAccessUnit = true;
1327            ++mNumFramesDropped;
1328        }
1329    } while (dropAccessUnit);
1330
1331    // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
1332
1333#if 0
1334    int64_t mediaTimeUs;
1335    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
1336    ALOGV("feeding %s input buffer at media time %.2f secs",
1337         audio ? "audio" : "video",
1338         mediaTimeUs / 1E6);
1339#endif
1340
1341    if (!audio) {
1342        mCCDecoder->decode(accessUnit);
1343    }
1344
1345    reply->setBuffer("buffer", accessUnit);
1346    reply->post();
1347
1348    return OK;
1349}
1350
1351void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
1352    // ALOGV("renderBuffer %s", audio ? "audio" : "video");
1353
1354    sp<AMessage> reply;
1355    CHECK(msg->findMessage("reply", &reply));
1356
1357    if ((audio && mFlushingAudio != NONE)
1358            || (!audio && mFlushingVideo != NONE)) {
1359        // We're currently attempting to flush the decoder, in order
1360        // to complete this, the decoder wants all its buffers back,
1361        // so we don't want any output buffers it sent us (from before
1362        // we initiated the flush) to be stuck in the renderer's queue.
1363
1364        ALOGV("we're still flushing the %s decoder, sending its output buffer"
1365             " right back.", audio ? "audio" : "video");
1366
1367        reply->post();
1368        return;
1369    }
1370
1371    sp<ABuffer> buffer;
1372    CHECK(msg->findBuffer("buffer", &buffer));
1373
1374    int64_t mediaTimeUs;
1375    CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));
1376
1377    int64_t &skipUntilMediaTimeUs =
1378        audio
1379            ? mSkipRenderingAudioUntilMediaTimeUs
1380            : mSkipRenderingVideoUntilMediaTimeUs;
1381
1382    if (skipUntilMediaTimeUs >= 0) {
1383
1384        if (mediaTimeUs < skipUntilMediaTimeUs) {
1385            ALOGV("dropping %s buffer at time %lld as requested.",
1386                 audio ? "audio" : "video",
1387                 mediaTimeUs);
1388
1389            reply->post();
1390            return;
1391        }
1392
1393        skipUntilMediaTimeUs = -1;
1394    }
1395
1396    if (!audio && mCCDecoder->isSelected()) {
1397        mCCDecoder->display(mediaTimeUs);
1398    }
1399
1400    mRenderer->queueBuffer(audio, buffer, reply);
1401}
1402
1403void NuPlayer::updateVideoSize(
1404        const sp<AMessage> &inputFormat,
1405        const sp<AMessage> &outputFormat) {
1406    if (inputFormat == NULL) {
1407        ALOGW("Unknown video size, reporting 0x0!");
1408        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1409        return;
1410    }
1411
1412    int32_t displayWidth, displayHeight;
1413    int32_t cropLeft, cropTop, cropRight, cropBottom;
1414
1415    if (outputFormat != NULL) {
1416        int32_t width, height;
1417        CHECK(outputFormat->findInt32("width", &width));
1418        CHECK(outputFormat->findInt32("height", &height));
1419
1420        int32_t cropLeft, cropTop, cropRight, cropBottom;
1421        CHECK(outputFormat->findRect(
1422                    "crop",
1423                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1424
1425        displayWidth = cropRight - cropLeft + 1;
1426        displayHeight = cropBottom - cropTop + 1;
1427
1428        ALOGV("Video output format changed to %d x %d "
1429             "(crop: %d x %d @ (%d, %d))",
1430             width, height,
1431             displayWidth,
1432             displayHeight,
1433             cropLeft, cropTop);
1434    } else {
1435        CHECK(inputFormat->findInt32("width", &displayWidth));
1436        CHECK(inputFormat->findInt32("height", &displayHeight));
1437
1438        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1439    }
1440
1441    // Take into account sample aspect ratio if necessary:
1442    int32_t sarWidth, sarHeight;
1443    if (inputFormat->findInt32("sar-width", &sarWidth)
1444            && inputFormat->findInt32("sar-height", &sarHeight)) {
1445        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1446
1447        displayWidth = (displayWidth * sarWidth) / sarHeight;
1448
1449        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1450    }
1451
1452    int32_t rotationDegrees;
1453    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1454        rotationDegrees = 0;
1455    }
1456
1457    if (rotationDegrees == 90 || rotationDegrees == 270) {
1458        int32_t tmp = displayWidth;
1459        displayWidth = displayHeight;
1460        displayHeight = tmp;
1461    }
1462
1463    notifyListener(
1464            MEDIA_SET_VIDEO_SIZE,
1465            displayWidth,
1466            displayHeight);
1467}
1468
1469void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1470    if (mDriver == NULL) {
1471        return;
1472    }
1473
1474    sp<NuPlayerDriver> driver = mDriver.promote();
1475
1476    if (driver == NULL) {
1477        return;
1478    }
1479
1480    driver->notifyListener(msg, ext1, ext2, in);
1481}
1482
1483void NuPlayer::flushDecoder(bool audio, bool needShutdown) {
1484    ALOGV("[%s] flushDecoder needShutdown=%d",
1485          audio ? "audio" : "video", needShutdown);
1486
1487    if ((audio && mAudioDecoder == NULL) || (!audio && mVideoDecoder == NULL)) {
1488        ALOGI("flushDecoder %s without decoder present",
1489             audio ? "audio" : "video");
1490    }
1491
1492    // Make sure we don't continue to scan sources until we finish flushing.
1493    ++mScanSourcesGeneration;
1494    mScanSourcesPending = false;
1495
1496    (audio ? mAudioDecoder : mVideoDecoder)->signalFlush();
1497    mRenderer->flush(audio);
1498
1499    FlushStatus newStatus =
1500        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1501
1502    if (audio) {
1503        ALOGE_IF(mFlushingAudio != NONE,
1504                "audio flushDecoder() is called in state %d", mFlushingAudio);
1505        mFlushingAudio = newStatus;
1506    } else {
1507        ALOGE_IF(mFlushingVideo != NONE,
1508                "video flushDecoder() is called in state %d", mFlushingVideo);
1509        mFlushingVideo = newStatus;
1510    }
1511}
1512
1513void NuPlayer::queueDecoderShutdown(
1514        bool audio, bool video, const sp<AMessage> &reply) {
1515    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1516
1517    mDeferredActions.push_back(
1518            new ShutdownDecoderAction(audio, video));
1519
1520    mDeferredActions.push_back(
1521            new SimpleAction(&NuPlayer::performScanSources));
1522
1523    mDeferredActions.push_back(new PostMessageAction(reply));
1524
1525    processDeferredActions();
1526}
1527
1528status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1529    mVideoScalingMode = mode;
1530    if (mNativeWindow != NULL) {
1531        status_t ret = native_window_set_scaling_mode(
1532                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1533        if (ret != OK) {
1534            ALOGE("Failed to set scaling mode (%d): %s",
1535                -ret, strerror(-ret));
1536            return ret;
1537        }
1538    }
1539    return OK;
1540}
1541
1542status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1543    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id());
1544    msg->setPointer("reply", reply);
1545
1546    sp<AMessage> response;
1547    status_t err = msg->postAndAwaitResponse(&response);
1548    return err;
1549}
1550
1551status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1552    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id());
1553    msg->setPointer("reply", reply);
1554    msg->setInt32("type", type);
1555
1556    sp<AMessage> response;
1557    status_t err = msg->postAndAwaitResponse(&response);
1558    if (err == OK && response != NULL) {
1559        CHECK(response->findInt32("err", &err));
1560    }
1561    return err;
1562}
1563
1564status_t NuPlayer::selectTrack(size_t trackIndex, bool select) {
1565    sp<AMessage> msg = new AMessage(kWhatSelectTrack, id());
1566    msg->setSize("trackIndex", trackIndex);
1567    msg->setInt32("select", select);
1568
1569    sp<AMessage> response;
1570    status_t err = msg->postAndAwaitResponse(&response);
1571
1572    if (err != OK) {
1573        return err;
1574    }
1575
1576    if (!response->findInt32("err", &err)) {
1577        err = OK;
1578    }
1579
1580    return err;
1581}
1582
1583void NuPlayer::schedulePollDuration() {
1584    sp<AMessage> msg = new AMessage(kWhatPollDuration, id());
1585    msg->setInt32("generation", mPollDurationGeneration);
1586    msg->post();
1587}
1588
1589void NuPlayer::cancelPollDuration() {
1590    ++mPollDurationGeneration;
1591}
1592
1593void NuPlayer::processDeferredActions() {
1594    while (!mDeferredActions.empty()) {
1595        // We won't execute any deferred actions until we're no longer in
1596        // an intermediate state, i.e. one more more decoders are currently
1597        // flushing or shutting down.
1598
1599        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1600            // We're currently flushing, postpone the reset until that's
1601            // completed.
1602
1603            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1604                  mFlushingAudio, mFlushingVideo);
1605
1606            break;
1607        }
1608
1609        sp<Action> action = *mDeferredActions.begin();
1610        mDeferredActions.erase(mDeferredActions.begin());
1611
1612        action->execute(this);
1613    }
1614}
1615
1616void NuPlayer::performSeek(int64_t seekTimeUs) {
1617    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs)",
1618          seekTimeUs,
1619          seekTimeUs / 1E6);
1620
1621    mSource->seekTo(seekTimeUs);
1622    ++mTimedTextGeneration;
1623
1624    if (mDriver != NULL) {
1625        sp<NuPlayerDriver> driver = mDriver.promote();
1626        if (driver != NULL) {
1627            driver->notifyPosition(seekTimeUs);
1628            driver->notifySeekComplete();
1629        }
1630    }
1631
1632    // everything's flushed, continue playback.
1633}
1634
1635void NuPlayer::performDecoderFlush() {
1636    ALOGV("performDecoderFlush");
1637
1638    if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
1639        return;
1640    }
1641
1642    mTimeDiscontinuityPending = true;
1643
1644    if (mAudioDecoder != NULL) {
1645        flushDecoder(true /* audio */, false /* needShutdown */);
1646    }
1647
1648    if (mVideoDecoder != NULL) {
1649        flushDecoder(false /* audio */, false /* needShutdown */);
1650    }
1651}
1652
1653void NuPlayer::performDecoderShutdown(bool audio, bool video) {
1654    ALOGV("performDecoderShutdown audio=%d, video=%d", audio, video);
1655
1656    if ((!audio || mAudioDecoder == NULL)
1657            && (!video || mVideoDecoder == NULL)) {
1658        return;
1659    }
1660
1661    mTimeDiscontinuityPending = true;
1662
1663    if (audio && mAudioDecoder != NULL) {
1664        flushDecoder(true /* audio */, true /* needShutdown */);
1665    }
1666
1667    if (video && mVideoDecoder != NULL) {
1668        flushDecoder(false /* audio */, true /* needShutdown */);
1669    }
1670}
1671
1672void NuPlayer::performReset() {
1673    ALOGV("performReset");
1674
1675    CHECK(mAudioDecoder == NULL);
1676    CHECK(mVideoDecoder == NULL);
1677
1678    cancelPollDuration();
1679
1680    ++mScanSourcesGeneration;
1681    mScanSourcesPending = false;
1682
1683    if (mRendererLooper != NULL) {
1684        if (mRenderer != NULL) {
1685            mRendererLooper->unregisterHandler(mRenderer->id());
1686        }
1687        mRendererLooper->stop();
1688        mRendererLooper.clear();
1689    }
1690    mRenderer.clear();
1691
1692    if (mSource != NULL) {
1693        mSource->stop();
1694
1695        mSource.clear();
1696    }
1697
1698    if (mDriver != NULL) {
1699        sp<NuPlayerDriver> driver = mDriver.promote();
1700        if (driver != NULL) {
1701            driver->notifyResetComplete();
1702        }
1703    }
1704
1705    mStarted = false;
1706}
1707
1708void NuPlayer::performScanSources() {
1709    ALOGV("performScanSources");
1710
1711    if (!mStarted) {
1712        return;
1713    }
1714
1715    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1716        postScanSources();
1717    }
1718}
1719
1720void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1721    ALOGV("performSetSurface");
1722
1723    mNativeWindow = wrapper;
1724
1725    // XXX - ignore error from setVideoScalingMode for now
1726    setVideoScalingMode(mVideoScalingMode);
1727}
1728
1729void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1730    int32_t what;
1731    CHECK(msg->findInt32("what", &what));
1732
1733    switch (what) {
1734        case Source::kWhatPrepared:
1735        {
1736            if (mSource == NULL) {
1737                // This is a stale notification from a source that was
1738                // asynchronously preparing when the client called reset().
1739                // We handled the reset, the source is gone.
1740                break;
1741            }
1742
1743            int32_t err;
1744            CHECK(msg->findInt32("err", &err));
1745
1746            sp<NuPlayerDriver> driver = mDriver.promote();
1747            if (driver != NULL) {
1748                // notify duration first, so that it's definitely set when
1749                // the app received the "prepare complete" callback.
1750                int64_t durationUs;
1751                if (mSource->getDuration(&durationUs) == OK) {
1752                    driver->notifyDuration(durationUs);
1753                }
1754                driver->notifyPrepareCompleted(err);
1755            }
1756
1757            break;
1758        }
1759
1760        case Source::kWhatFlagsChanged:
1761        {
1762            uint32_t flags;
1763            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1764
1765            sp<NuPlayerDriver> driver = mDriver.promote();
1766            if (driver != NULL) {
1767                driver->notifyFlagsChanged(flags);
1768            }
1769
1770            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1771                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
1772                cancelPollDuration();
1773            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1774                    && (flags & Source::FLAG_DYNAMIC_DURATION)
1775                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
1776                schedulePollDuration();
1777            }
1778
1779            mSourceFlags = flags;
1780            break;
1781        }
1782
1783        case Source::kWhatVideoSizeChanged:
1784        {
1785            sp<AMessage> format;
1786            CHECK(msg->findMessage("format", &format));
1787
1788            updateVideoSize(format);
1789            break;
1790        }
1791
1792        case Source::kWhatBufferingStart:
1793        {
1794            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
1795            break;
1796        }
1797
1798        case Source::kWhatBufferingEnd:
1799        {
1800            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
1801            break;
1802        }
1803
1804        case Source::kWhatSubtitleData:
1805        {
1806            sp<ABuffer> buffer;
1807            CHECK(msg->findBuffer("buffer", &buffer));
1808
1809            sendSubtitleData(buffer, 0 /* baseIndex */);
1810            break;
1811        }
1812
1813        case Source::kWhatTimedTextData:
1814        {
1815            int32_t generation;
1816            if (msg->findInt32("generation", &generation)
1817                    && generation != mTimedTextGeneration) {
1818                break;
1819            }
1820
1821            sp<ABuffer> buffer;
1822            CHECK(msg->findBuffer("buffer", &buffer));
1823
1824            sp<NuPlayerDriver> driver = mDriver.promote();
1825            if (driver == NULL) {
1826                break;
1827            }
1828
1829            int posMs;
1830            int64_t timeUs, posUs;
1831            driver->getCurrentPosition(&posMs);
1832            posUs = posMs * 1000;
1833            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1834
1835            if (posUs < timeUs) {
1836                if (!msg->findInt32("generation", &generation)) {
1837                    msg->setInt32("generation", mTimedTextGeneration);
1838                }
1839                msg->post(timeUs - posUs);
1840            } else {
1841                sendTimedTextData(buffer);
1842            }
1843            break;
1844        }
1845
1846        case Source::kWhatQueueDecoderShutdown:
1847        {
1848            int32_t audio, video;
1849            CHECK(msg->findInt32("audio", &audio));
1850            CHECK(msg->findInt32("video", &video));
1851
1852            sp<AMessage> reply;
1853            CHECK(msg->findMessage("reply", &reply));
1854
1855            queueDecoderShutdown(audio, video, reply);
1856            break;
1857        }
1858
1859        default:
1860            TRESPASS();
1861    }
1862}
1863
1864void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
1865    int32_t what;
1866    CHECK(msg->findInt32("what", &what));
1867
1868    switch (what) {
1869        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
1870        {
1871            sp<ABuffer> buffer;
1872            CHECK(msg->findBuffer("buffer", &buffer));
1873
1874            size_t inbandTracks = 0;
1875            if (mSource != NULL) {
1876                inbandTracks = mSource->getTrackCount();
1877            }
1878
1879            sendSubtitleData(buffer, inbandTracks);
1880            break;
1881        }
1882
1883        case NuPlayer::CCDecoder::kWhatTrackAdded:
1884        {
1885            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
1886
1887            break;
1888        }
1889
1890        default:
1891            TRESPASS();
1892    }
1893
1894
1895}
1896
1897void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
1898    int32_t trackIndex;
1899    int64_t timeUs, durationUs;
1900    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
1901    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1902    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
1903
1904    Parcel in;
1905    in.writeInt32(trackIndex + baseIndex);
1906    in.writeInt64(timeUs);
1907    in.writeInt64(durationUs);
1908    in.writeInt32(buffer->size());
1909    in.writeInt32(buffer->size());
1910    in.write(buffer->data(), buffer->size());
1911
1912    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
1913}
1914
1915void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
1916    const void *data;
1917    size_t size = 0;
1918    int64_t timeUs;
1919    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
1920
1921    AString mime;
1922    CHECK(buffer->meta()->findString("mime", &mime));
1923    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
1924
1925    data = buffer->data();
1926    size = buffer->size();
1927
1928    Parcel parcel;
1929    if (size > 0) {
1930        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1931        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
1932        TextDescriptions::getParcelOfDescriptions(
1933                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
1934    }
1935
1936    if ((parcel.dataSize() > 0)) {
1937        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
1938    } else {  // send an empty timed text
1939        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
1940    }
1941}
1942////////////////////////////////////////////////////////////////////////////////
1943
1944sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
1945    sp<MetaData> meta = getFormatMeta(audio);
1946
1947    if (meta == NULL) {
1948        return NULL;
1949    }
1950
1951    sp<AMessage> msg = new AMessage;
1952
1953    if(convertMetaDataToMessage(meta, &msg) == OK) {
1954        return msg;
1955    }
1956    return NULL;
1957}
1958
1959void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
1960    sp<AMessage> notify = dupNotify();
1961    notify->setInt32("what", kWhatFlagsChanged);
1962    notify->setInt32("flags", flags);
1963    notify->post();
1964}
1965
1966void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
1967    sp<AMessage> notify = dupNotify();
1968    notify->setInt32("what", kWhatVideoSizeChanged);
1969    notify->setMessage("format", format);
1970    notify->post();
1971}
1972
1973void NuPlayer::Source::notifyPrepared(status_t err) {
1974    sp<AMessage> notify = dupNotify();
1975    notify->setInt32("what", kWhatPrepared);
1976    notify->setInt32("err", err);
1977    notify->post();
1978}
1979
1980void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
1981    TRESPASS();
1982}
1983
1984}  // namespace android
1985