NuPlayer.cpp revision 88703c34fb4a9db1ff51495879f9775474c8ce89 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerDecoder.h"
25#include "NuPlayerDecoderPassThrough.h"
26#include "NuPlayerDriver.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29#include "RTSPSource.h"
30#include "StreamingSource.h"
31#include "GenericSource.h"
32#include "TextDescriptions.h"
33
34#include "ATSParser.h"
35
36#include <media/stagefright/foundation/hexdump.h>
37#include <media/stagefright/foundation/ABuffer.h>
38#include <media/stagefright/foundation/ADebug.h>
39#include <media/stagefright/foundation/AMessage.h>
40#include <media/stagefright/MediaBuffer.h>
41#include <media/stagefright/MediaDefs.h>
42#include <media/stagefright/MediaErrors.h>
43#include <media/stagefright/MetaData.h>
44#include <gui/IGraphicBufferProducer.h>
45
46#include "avc_utils.h"
47
48#include "ESDS.h"
49#include <media/stagefright/Utils.h>
50
51namespace android {
52
53struct NuPlayer::Action : public RefBase {
54    Action() {}
55
56    virtual void execute(NuPlayer *player) = 0;
57
58private:
59    DISALLOW_EVIL_CONSTRUCTORS(Action);
60};
61
62struct NuPlayer::SeekAction : public Action {
63    SeekAction(int64_t seekTimeUs)
64        : mSeekTimeUs(seekTimeUs) {
65    }
66
67    virtual void execute(NuPlayer *player) {
68        player->performSeek(mSeekTimeUs);
69    }
70
71private:
72    int64_t mSeekTimeUs;
73
74    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
75};
76
77struct NuPlayer::SetSurfaceAction : public Action {
78    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
79        : mWrapper(wrapper) {
80    }
81
82    virtual void execute(NuPlayer *player) {
83        player->performSetSurface(mWrapper);
84    }
85
86private:
87    sp<NativeWindowWrapper> mWrapper;
88
89    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
90};
91
92struct NuPlayer::ShutdownDecoderAction : public Action {
93    ShutdownDecoderAction(bool audio, bool video)
94        : mAudio(audio),
95          mVideo(video) {
96    }
97
98    virtual void execute(NuPlayer *player) {
99        player->performDecoderShutdown(mAudio, mVideo);
100    }
101
102private:
103    bool mAudio;
104    bool mVideo;
105
106    DISALLOW_EVIL_CONSTRUCTORS(ShutdownDecoderAction);
107};
108
109struct NuPlayer::PostMessageAction : public Action {
110    PostMessageAction(const sp<AMessage> &msg)
111        : mMessage(msg) {
112    }
113
114    virtual void execute(NuPlayer *) {
115        mMessage->post();
116    }
117
118private:
119    sp<AMessage> mMessage;
120
121    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
122};
123
124// Use this if there's no state necessary to save in order to execute
125// the action.
126struct NuPlayer::SimpleAction : public Action {
127    typedef void (NuPlayer::*ActionFunc)();
128
129    SimpleAction(ActionFunc func)
130        : mFunc(func) {
131    }
132
133    virtual void execute(NuPlayer *player) {
134        (player->*mFunc)();
135    }
136
137private:
138    ActionFunc mFunc;
139
140    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
141};
142
143////////////////////////////////////////////////////////////////////////////////
144
145NuPlayer::NuPlayer()
146    : mUIDValid(false),
147      mSourceFlags(0),
148      mVideoIsAVC(false),
149      mOffloadAudio(false),
150      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
151      mAudioDecoderGeneration(0),
152      mVideoDecoderGeneration(0),
153      mAudioEOS(false),
154      mVideoEOS(false),
155      mScanSourcesPending(false),
156      mScanSourcesGeneration(0),
157      mPollDurationGeneration(0),
158      mTimedTextGeneration(0),
159      mTimeDiscontinuityPending(false),
160      mFlushingAudio(NONE),
161      mFlushingVideo(NONE),
162      mSkipRenderingAudioUntilMediaTimeUs(-1ll),
163      mSkipRenderingVideoUntilMediaTimeUs(-1ll),
164      mVideoLateByUs(0ll),
165      mNumFramesTotal(0ll),
166      mNumFramesDropped(0ll),
167      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
168      mStarted(false) {
169}
170
171NuPlayer::~NuPlayer() {
172}
173
174void NuPlayer::setUID(uid_t uid) {
175    mUIDValid = true;
176    mUID = uid;
177}
178
179void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
180    mDriver = driver;
181}
182
183void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
184    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
185
186    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
187
188    msg->setObject("source", new StreamingSource(notify, source));
189    msg->post();
190}
191
192static bool IsHTTPLiveURL(const char *url) {
193    if (!strncasecmp("http://", url, 7)
194            || !strncasecmp("https://", url, 8)
195            || !strncasecmp("file://", url, 7)) {
196        size_t len = strlen(url);
197        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
198            return true;
199        }
200
201        if (strstr(url,"m3u8")) {
202            return true;
203        }
204    }
205
206    return false;
207}
208
209void NuPlayer::setDataSourceAsync(
210        const sp<IMediaHTTPService> &httpService,
211        const char *url,
212        const KeyedVector<String8, String8> *headers) {
213
214    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
215    size_t len = strlen(url);
216
217    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
218
219    sp<Source> source;
220    if (IsHTTPLiveURL(url)) {
221        source = new HTTPLiveSource(notify, httpService, url, headers);
222    } else if (!strncasecmp(url, "rtsp://", 7)) {
223        source = new RTSPSource(
224                notify, httpService, url, headers, mUIDValid, mUID);
225    } else if ((!strncasecmp(url, "http://", 7)
226                || !strncasecmp(url, "https://", 8))
227                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
228                    || strstr(url, ".sdp?"))) {
229        source = new RTSPSource(
230                notify, httpService, url, headers, mUIDValid, mUID, true);
231    } else {
232        sp<GenericSource> genericSource =
233                new GenericSource(notify, mUIDValid, mUID);
234        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
235        // The correct flags will be updated in Source::kWhatFlagsChanged
236        // handler when  GenericSource is prepared.
237
238        status_t err = genericSource->setDataSource(httpService, url, headers);
239
240        if (err == OK) {
241            source = genericSource;
242        } else {
243            ALOGE("Failed to set data source!");
244        }
245    }
246    msg->setObject("source", source);
247    msg->post();
248}
249
250void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
251    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
252
253    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
254
255    sp<GenericSource> source =
256            new GenericSource(notify, mUIDValid, mUID);
257
258    status_t err = source->setDataSource(fd, offset, length);
259
260    if (err != OK) {
261        ALOGE("Failed to set data source!");
262        source = NULL;
263    }
264
265    msg->setObject("source", source);
266    msg->post();
267}
268
269void NuPlayer::prepareAsync() {
270    (new AMessage(kWhatPrepare, id()))->post();
271}
272
273void NuPlayer::setVideoSurfaceTextureAsync(
274        const sp<IGraphicBufferProducer> &bufferProducer) {
275    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id());
276
277    if (bufferProducer == NULL) {
278        msg->setObject("native-window", NULL);
279    } else {
280        msg->setObject(
281                "native-window",
282                new NativeWindowWrapper(
283                    new Surface(bufferProducer)));
284    }
285
286    msg->post();
287}
288
289void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
290    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id());
291    msg->setObject("sink", sink);
292    msg->post();
293}
294
295void NuPlayer::start() {
296    (new AMessage(kWhatStart, id()))->post();
297}
298
299void NuPlayer::pause() {
300    (new AMessage(kWhatPause, id()))->post();
301}
302
303void NuPlayer::resume() {
304    (new AMessage(kWhatResume, id()))->post();
305}
306
307void NuPlayer::resetAsync() {
308    (new AMessage(kWhatReset, id()))->post();
309}
310
311void NuPlayer::seekToAsync(int64_t seekTimeUs) {
312    sp<AMessage> msg = new AMessage(kWhatSeek, id());
313    msg->setInt64("seekTimeUs", seekTimeUs);
314    msg->post();
315}
316
317// static
318bool NuPlayer::IsFlushingState(FlushStatus state, bool *needShutdown) {
319    switch (state) {
320        case FLUSHING_DECODER:
321            if (needShutdown != NULL) {
322                *needShutdown = false;
323            }
324            return true;
325
326        case FLUSHING_DECODER_SHUTDOWN:
327            if (needShutdown != NULL) {
328                *needShutdown = true;
329            }
330            return true;
331
332        default:
333            return false;
334    }
335}
336
337void NuPlayer::writeTrackInfo(
338        Parcel* reply, const sp<AMessage> format) const {
339    int32_t trackType;
340    CHECK(format->findInt32("type", &trackType));
341
342    AString lang;
343    CHECK(format->findString("language", &lang));
344
345    reply->writeInt32(2); // write something non-zero
346    reply->writeInt32(trackType);
347    reply->writeString16(String16(lang.c_str()));
348
349    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
350        AString mime;
351        CHECK(format->findString("mime", &mime));
352
353        int32_t isAuto, isDefault, isForced;
354        CHECK(format->findInt32("auto", &isAuto));
355        CHECK(format->findInt32("default", &isDefault));
356        CHECK(format->findInt32("forced", &isForced));
357
358        reply->writeString16(String16(mime.c_str()));
359        reply->writeInt32(isAuto);
360        reply->writeInt32(isDefault);
361        reply->writeInt32(isForced);
362    }
363}
364
365void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
366    switch (msg->what()) {
367        case kWhatSetDataSource:
368        {
369            ALOGV("kWhatSetDataSource");
370
371            CHECK(mSource == NULL);
372
373            status_t err = OK;
374            sp<RefBase> obj;
375            CHECK(msg->findObject("source", &obj));
376            if (obj != NULL) {
377                mSource = static_cast<Source *>(obj.get());
378            } else {
379                err = UNKNOWN_ERROR;
380            }
381
382            CHECK(mDriver != NULL);
383            sp<NuPlayerDriver> driver = mDriver.promote();
384            if (driver != NULL) {
385                driver->notifySetDataSourceCompleted(err);
386            }
387            break;
388        }
389
390        case kWhatPrepare:
391        {
392            mSource->prepareAsync();
393            break;
394        }
395
396        case kWhatGetTrackInfo:
397        {
398            uint32_t replyID;
399            CHECK(msg->senderAwaitsResponse(&replyID));
400
401            Parcel* reply;
402            CHECK(msg->findPointer("reply", (void**)&reply));
403
404            size_t inbandTracks = 0;
405            if (mSource != NULL) {
406                inbandTracks = mSource->getTrackCount();
407            }
408
409            size_t ccTracks = 0;
410            if (mCCDecoder != NULL) {
411                ccTracks = mCCDecoder->getTrackCount();
412            }
413
414            // total track count
415            reply->writeInt32(inbandTracks + ccTracks);
416
417            // write inband tracks
418            for (size_t i = 0; i < inbandTracks; ++i) {
419                writeTrackInfo(reply, mSource->getTrackInfo(i));
420            }
421
422            // write CC track
423            for (size_t i = 0; i < ccTracks; ++i) {
424                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
425            }
426
427            sp<AMessage> response = new AMessage;
428            response->postReply(replyID);
429            break;
430        }
431
432        case kWhatGetSelectedTrack:
433        {
434            status_t err = INVALID_OPERATION;
435            if (mSource != NULL) {
436                err = OK;
437
438                int32_t type32;
439                CHECK(msg->findInt32("type", (int32_t*)&type32));
440                media_track_type type = (media_track_type)type32;
441                ssize_t selectedTrack = mSource->getSelectedTrack(type);
442
443                Parcel* reply;
444                CHECK(msg->findPointer("reply", (void**)&reply));
445                reply->writeInt32(selectedTrack);
446            }
447
448            sp<AMessage> response = new AMessage;
449            response->setInt32("err", err);
450
451            uint32_t replyID;
452            CHECK(msg->senderAwaitsResponse(&replyID));
453            response->postReply(replyID);
454            break;
455        }
456
457        case kWhatSelectTrack:
458        {
459            uint32_t replyID;
460            CHECK(msg->senderAwaitsResponse(&replyID));
461
462            size_t trackIndex;
463            int32_t select;
464            CHECK(msg->findSize("trackIndex", &trackIndex));
465            CHECK(msg->findInt32("select", &select));
466
467            status_t err = INVALID_OPERATION;
468
469            size_t inbandTracks = 0;
470            if (mSource != NULL) {
471                inbandTracks = mSource->getTrackCount();
472            }
473            size_t ccTracks = 0;
474            if (mCCDecoder != NULL) {
475                ccTracks = mCCDecoder->getTrackCount();
476            }
477
478            if (trackIndex < inbandTracks) {
479                err = mSource->selectTrack(trackIndex, select);
480
481                if (!select && err == OK) {
482                    int32_t type;
483                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
484                    if (info != NULL
485                            && info->findInt32("type", &type)
486                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
487                        ++mTimedTextGeneration;
488                    }
489                }
490            } else {
491                trackIndex -= inbandTracks;
492
493                if (trackIndex < ccTracks) {
494                    err = mCCDecoder->selectTrack(trackIndex, select);
495                }
496            }
497
498            sp<AMessage> response = new AMessage;
499            response->setInt32("err", err);
500
501            response->postReply(replyID);
502            break;
503        }
504
505        case kWhatPollDuration:
506        {
507            int32_t generation;
508            CHECK(msg->findInt32("generation", &generation));
509
510            if (generation != mPollDurationGeneration) {
511                // stale
512                break;
513            }
514
515            int64_t durationUs;
516            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
517                sp<NuPlayerDriver> driver = mDriver.promote();
518                if (driver != NULL) {
519                    driver->notifyDuration(durationUs);
520                }
521            }
522
523            msg->post(1000000ll);  // poll again in a second.
524            break;
525        }
526
527        case kWhatSetVideoNativeWindow:
528        {
529            ALOGV("kWhatSetVideoNativeWindow");
530
531            mDeferredActions.push_back(
532                    new ShutdownDecoderAction(
533                        false /* audio */, true /* video */));
534
535            sp<RefBase> obj;
536            CHECK(msg->findObject("native-window", &obj));
537
538            mDeferredActions.push_back(
539                    new SetSurfaceAction(
540                        static_cast<NativeWindowWrapper *>(obj.get())));
541
542            if (obj != NULL) {
543                // If there is a new surface texture, instantiate decoders
544                // again if possible.
545                mDeferredActions.push_back(
546                        new SimpleAction(&NuPlayer::performScanSources));
547            }
548
549            processDeferredActions();
550            break;
551        }
552
553        case kWhatSetAudioSink:
554        {
555            ALOGV("kWhatSetAudioSink");
556
557            sp<RefBase> obj;
558            CHECK(msg->findObject("sink", &obj));
559
560            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
561            break;
562        }
563
564        case kWhatStart:
565        {
566            ALOGV("kWhatStart");
567
568            mVideoIsAVC = false;
569            mOffloadAudio = false;
570            mAudioEOS = false;
571            mVideoEOS = false;
572            mSkipRenderingAudioUntilMediaTimeUs = -1;
573            mSkipRenderingVideoUntilMediaTimeUs = -1;
574            mVideoLateByUs = 0;
575            mNumFramesTotal = 0;
576            mNumFramesDropped = 0;
577            mStarted = true;
578
579            /* instantiate decoders now for secure playback */
580            if (mSourceFlags & Source::FLAG_SECURE) {
581                if (mNativeWindow != NULL) {
582                    instantiateDecoder(false, &mVideoDecoder);
583                }
584
585                if (mAudioSink != NULL) {
586                    instantiateDecoder(true, &mAudioDecoder);
587                }
588            }
589
590            mSource->start();
591
592            uint32_t flags = 0;
593
594            if (mSource->isRealTime()) {
595                flags |= Renderer::FLAG_REAL_TIME;
596            }
597
598            sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
599            audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
600            if (mAudioSink != NULL) {
601                streamType = mAudioSink->getAudioStreamType();
602            }
603
604            sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
605
606            mOffloadAudio =
607                canOffloadStream(audioMeta, (videoFormat != NULL),
608                                 true /* is_streaming */, streamType);
609            if (mOffloadAudio) {
610                flags |= Renderer::FLAG_OFFLOAD_AUDIO;
611            }
612
613            mRenderer = new Renderer(
614                    mAudioSink,
615                    new AMessage(kWhatRendererNotify, id()),
616                    flags);
617
618            mRendererLooper = new ALooper;
619            mRendererLooper->setName("NuPlayerRenderer");
620            mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
621            mRendererLooper->registerHandler(mRenderer);
622
623            postScanSources();
624            break;
625        }
626
627        case kWhatScanSources:
628        {
629            int32_t generation;
630            CHECK(msg->findInt32("generation", &generation));
631            if (generation != mScanSourcesGeneration) {
632                // Drop obsolete msg.
633                break;
634            }
635
636            mScanSourcesPending = false;
637
638            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
639                 mAudioDecoder != NULL, mVideoDecoder != NULL);
640
641            bool mHadAnySourcesBefore =
642                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
643
644            // initialize video before audio because successful initialization of
645            // video may change deep buffer mode of audio.
646            if (mNativeWindow != NULL) {
647                instantiateDecoder(false, &mVideoDecoder);
648            }
649
650            if (mAudioSink != NULL) {
651                if (mOffloadAudio) {
652                    // open audio sink early under offload mode.
653                    sp<AMessage> format = mSource->getFormat(true /*audio*/);
654                    openAudioSink(format, true /*offloadOnly*/);
655                }
656                instantiateDecoder(true, &mAudioDecoder);
657            }
658
659            if (!mHadAnySourcesBefore
660                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
661                // This is the first time we've found anything playable.
662
663                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
664                    schedulePollDuration();
665                }
666            }
667
668            status_t err;
669            if ((err = mSource->feedMoreTSData()) != OK) {
670                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
671                    // We're not currently decoding anything (no audio or
672                    // video tracks found) and we just ran out of input data.
673
674                    if (err == ERROR_END_OF_STREAM) {
675                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
676                    } else {
677                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
678                    }
679                }
680                break;
681            }
682
683            if ((mAudioDecoder == NULL && mAudioSink != NULL)
684                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
685                msg->post(100000ll);
686                mScanSourcesPending = true;
687            }
688            break;
689        }
690
691        case kWhatVideoNotify:
692        case kWhatAudioNotify:
693        {
694            bool audio = msg->what() == kWhatAudioNotify;
695
696            int32_t currentDecoderGeneration =
697                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
698            int32_t requesterGeneration = currentDecoderGeneration - 1;
699            CHECK(msg->findInt32("generation", &requesterGeneration));
700
701            if (requesterGeneration != currentDecoderGeneration) {
702                ALOGV("got message from old %s decoder, generation(%d:%d)",
703                        audio ? "audio" : "video", requesterGeneration,
704                        currentDecoderGeneration);
705                sp<AMessage> reply;
706                if (!(msg->findMessage("reply", &reply))) {
707                    return;
708                }
709
710                reply->setInt32("err", INFO_DISCONTINUITY);
711                reply->post();
712                return;
713            }
714
715            int32_t what;
716            CHECK(msg->findInt32("what", &what));
717
718            if (what == Decoder::kWhatFillThisBuffer) {
719                status_t err = feedDecoderInputData(
720                        audio, msg);
721
722                if (err == -EWOULDBLOCK) {
723                    if (mSource->feedMoreTSData() == OK) {
724                        msg->post(10000ll);
725                    }
726                }
727            } else if (what == Decoder::kWhatEOS) {
728                int32_t err;
729                CHECK(msg->findInt32("err", &err));
730
731                if (err == ERROR_END_OF_STREAM) {
732                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
733                } else {
734                    ALOGV("got %s decoder EOS w/ error %d",
735                         audio ? "audio" : "video",
736                         err);
737                }
738
739                mRenderer->queueEOS(audio, err);
740            } else if (what == Decoder::kWhatFlushCompleted) {
741                bool needShutdown;
742
743                if (audio) {
744                    CHECK(IsFlushingState(mFlushingAudio, &needShutdown));
745                    mFlushingAudio = FLUSHED;
746                } else {
747                    CHECK(IsFlushingState(mFlushingVideo, &needShutdown));
748                    mFlushingVideo = FLUSHED;
749
750                    mVideoLateByUs = 0;
751                }
752
753                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
754
755                if (needShutdown) {
756                    ALOGV("initiating %s decoder shutdown",
757                         audio ? "audio" : "video");
758
759                    (audio ? mAudioDecoder : mVideoDecoder)->initiateShutdown();
760
761                    if (audio) {
762                        mFlushingAudio = SHUTTING_DOWN_DECODER;
763                    } else {
764                        mFlushingVideo = SHUTTING_DOWN_DECODER;
765                    }
766                }
767
768                finishFlushIfPossible();
769            } else if (what == Decoder::kWhatOutputFormatChanged) {
770                sp<AMessage> format;
771                CHECK(msg->findMessage("format", &format));
772
773                if (audio) {
774                    openAudioSink(format, false /*offloadOnly*/);
775                } else {
776                    // video
777                    sp<AMessage> inputFormat =
778                            mSource->getFormat(false /* audio */);
779
780                    updateVideoSize(inputFormat, format);
781                }
782            } else if (what == Decoder::kWhatShutdownCompleted) {
783                ALOGV("%s shutdown completed", audio ? "audio" : "video");
784                if (audio) {
785                    mAudioDecoder.clear();
786
787                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
788                    mFlushingAudio = SHUT_DOWN;
789                } else {
790                    mVideoDecoder.clear();
791
792                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
793                    mFlushingVideo = SHUT_DOWN;
794                }
795
796                finishFlushIfPossible();
797            } else if (what == Decoder::kWhatError) {
798                ALOGE("Received error from %s decoder, aborting playback.",
799                     audio ? "audio" : "video");
800
801                status_t err;
802                if (!msg->findInt32("err", &err)) {
803                    err = UNKNOWN_ERROR;
804                }
805                mRenderer->queueEOS(audio, err);
806            } else if (what == Decoder::kWhatDrainThisBuffer) {
807                renderBuffer(audio, msg);
808            } else {
809                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
810                      what,
811                      what >> 24,
812                      (what >> 16) & 0xff,
813                      (what >> 8) & 0xff,
814                      what & 0xff);
815            }
816
817            break;
818        }
819
820        case kWhatRendererNotify:
821        {
822            int32_t what;
823            CHECK(msg->findInt32("what", &what));
824
825            if (what == Renderer::kWhatEOS) {
826                int32_t audio;
827                CHECK(msg->findInt32("audio", &audio));
828
829                int32_t finalResult;
830                CHECK(msg->findInt32("finalResult", &finalResult));
831
832                if (audio) {
833                    mAudioEOS = true;
834                } else {
835                    mVideoEOS = true;
836                }
837
838                if (finalResult == ERROR_END_OF_STREAM) {
839                    ALOGV("reached %s EOS", audio ? "audio" : "video");
840                } else {
841                    ALOGE("%s track encountered an error (%d)",
842                         audio ? "audio" : "video", finalResult);
843
844                    notifyListener(
845                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
846                }
847
848                if ((mAudioEOS || mAudioDecoder == NULL)
849                        && (mVideoEOS || mVideoDecoder == NULL)) {
850                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
851                }
852            } else if (what == Renderer::kWhatPosition) {
853                int64_t positionUs;
854                CHECK(msg->findInt64("positionUs", &positionUs));
855
856                CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs));
857
858                if (mDriver != NULL) {
859                    sp<NuPlayerDriver> driver = mDriver.promote();
860                    if (driver != NULL) {
861                        driver->notifyPosition(positionUs);
862
863                        driver->notifyFrameStats(
864                                mNumFramesTotal, mNumFramesDropped);
865                    }
866                }
867            } else if (what == Renderer::kWhatFlushComplete) {
868                int32_t audio;
869                CHECK(msg->findInt32("audio", &audio));
870
871                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
872            } else if (what == Renderer::kWhatVideoRenderingStart) {
873                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
874            } else if (what == Renderer::kWhatMediaRenderingStart) {
875                ALOGV("media rendering started");
876                notifyListener(MEDIA_STARTED, 0, 0);
877            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
878                ALOGV("Tear down audio offload, fall back to s/w path");
879                int64_t positionUs;
880                CHECK(msg->findInt64("positionUs", &positionUs));
881                closeAudioSink();
882                mAudioDecoder.clear();
883                mRenderer->flush(true /* audio */);
884                if (mVideoDecoder != NULL) {
885                    mRenderer->flush(false /* audio */);
886                }
887                mRenderer->signalDisableOffloadAudio();
888                mOffloadAudio = false;
889
890                performSeek(positionUs);
891                instantiateDecoder(true /* audio */, &mAudioDecoder);
892            }
893            break;
894        }
895
896        case kWhatMoreDataQueued:
897        {
898            break;
899        }
900
901        case kWhatReset:
902        {
903            ALOGV("kWhatReset");
904
905            mDeferredActions.push_back(
906                    new ShutdownDecoderAction(
907                        true /* audio */, true /* video */));
908
909            mDeferredActions.push_back(
910                    new SimpleAction(&NuPlayer::performReset));
911
912            processDeferredActions();
913            break;
914        }
915
916        case kWhatSeek:
917        {
918            int64_t seekTimeUs;
919            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
920
921            ALOGV("kWhatSeek seekTimeUs=%lld us", seekTimeUs);
922
923            mDeferredActions.push_back(
924                    new SimpleAction(&NuPlayer::performDecoderFlush));
925
926            mDeferredActions.push_back(new SeekAction(seekTimeUs));
927
928            processDeferredActions();
929            break;
930        }
931
932        case kWhatPause:
933        {
934            CHECK(mRenderer != NULL);
935            mSource->pause();
936            mRenderer->pause();
937            break;
938        }
939
940        case kWhatResume:
941        {
942            CHECK(mRenderer != NULL);
943            mSource->resume();
944            mRenderer->resume();
945            break;
946        }
947
948        case kWhatSourceNotify:
949        {
950            onSourceNotify(msg);
951            break;
952        }
953
954        case kWhatClosedCaptionNotify:
955        {
956            onClosedCaptionNotify(msg);
957            break;
958        }
959
960        default:
961            TRESPASS();
962            break;
963    }
964}
965
966void NuPlayer::finishFlushIfPossible() {
967    if (mFlushingAudio != FLUSHED && mFlushingAudio != SHUT_DOWN) {
968        return;
969    }
970
971    if (mFlushingVideo != FLUSHED && mFlushingVideo != SHUT_DOWN) {
972        return;
973    }
974
975    ALOGV("both audio and video are flushed now.");
976
977    if (mTimeDiscontinuityPending) {
978        mRenderer->signalTimeDiscontinuity();
979        mTimeDiscontinuityPending = false;
980    }
981
982    if (mAudioDecoder != NULL) {
983        mAudioDecoder->signalResume();
984    }
985
986    if (mVideoDecoder != NULL) {
987        mVideoDecoder->signalResume();
988    }
989
990    mFlushingAudio = NONE;
991    mFlushingVideo = NONE;
992
993    processDeferredActions();
994}
995
996void NuPlayer::postScanSources() {
997    if (mScanSourcesPending) {
998        return;
999    }
1000
1001    sp<AMessage> msg = new AMessage(kWhatScanSources, id());
1002    msg->setInt32("generation", mScanSourcesGeneration);
1003    msg->post();
1004
1005    mScanSourcesPending = true;
1006}
1007
1008void NuPlayer::openAudioSink(const sp<AMessage> &format, bool offloadOnly) {
1009    ALOGV("openAudioSink: offloadOnly(%d) mOffloadAudio(%d)",
1010            offloadOnly, mOffloadAudio);
1011    bool audioSinkChanged = false;
1012
1013    int32_t numChannels;
1014    CHECK(format->findInt32("channel-count", &numChannels));
1015
1016    int32_t channelMask;
1017    if (!format->findInt32("channel-mask", &channelMask)) {
1018        // signal to the AudioSink to derive the mask from count.
1019        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1020    }
1021
1022    int32_t sampleRate;
1023    CHECK(format->findInt32("sample-rate", &sampleRate));
1024
1025    uint32_t flags;
1026    int64_t durationUs;
1027    // FIXME: we should handle the case where the video decoder
1028    // is created after we receive the format change indication.
1029    // Current code will just make that we select deep buffer
1030    // with video which should not be a problem as it should
1031    // not prevent from keeping A/V sync.
1032    if (mVideoDecoder == NULL &&
1033            mSource->getDuration(&durationUs) == OK &&
1034            durationUs
1035                > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
1036        flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1037    } else {
1038        flags = AUDIO_OUTPUT_FLAG_NONE;
1039    }
1040
1041    if (mOffloadAudio) {
1042        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1043        AString mime;
1044        CHECK(format->findString("mime", &mime));
1045        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1046
1047        if (err != OK) {
1048            ALOGE("Couldn't map mime \"%s\" to a valid "
1049                    "audio_format", mime.c_str());
1050            mOffloadAudio = false;
1051        } else {
1052            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1053                    mime.c_str(), audioFormat);
1054
1055            int avgBitRate = -1;
1056            format->findInt32("bit-rate", &avgBitRate);
1057
1058            int32_t aacProfile = -1;
1059            if (audioFormat == AUDIO_FORMAT_AAC
1060                    && format->findInt32("aac-profile", &aacProfile)) {
1061                // Redefine AAC format as per aac profile
1062                mapAACProfileToAudioFormat(
1063                        audioFormat,
1064                        aacProfile);
1065            }
1066
1067            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1068            offloadInfo.duration_us = -1;
1069            format->findInt64(
1070                    "durationUs", &offloadInfo.duration_us);
1071            offloadInfo.sample_rate = sampleRate;
1072            offloadInfo.channel_mask = channelMask;
1073            offloadInfo.format = audioFormat;
1074            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1075            offloadInfo.bit_rate = avgBitRate;
1076            offloadInfo.has_video = (mVideoDecoder != NULL);
1077            offloadInfo.is_streaming = true;
1078
1079            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1080                ALOGV("openAudioSink: no change in offload mode");
1081                return;  // no change from previous configuration, everything ok.
1082            }
1083            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1084            flags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1085            audioSinkChanged = true;
1086            mAudioSink->close();
1087            err = mAudioSink->open(
1088                    sampleRate,
1089                    numChannels,
1090                    (audio_channel_mask_t)channelMask,
1091                    audioFormat,
1092                    8 /* bufferCount */,
1093                    &NuPlayer::Renderer::AudioSinkCallback,
1094                    mRenderer.get(),
1095                    (audio_output_flags_t)flags,
1096                    &offloadInfo);
1097
1098            if (err == OK) {
1099                // If the playback is offloaded to h/w, we pass
1100                // the HAL some metadata information.
1101                // We don't want to do this for PCM because it
1102                // will be going through the AudioFlinger mixer
1103                // before reaching the hardware.
1104                sp<MetaData> audioMeta =
1105                        mSource->getFormatMeta(true /* audio */);
1106                sendMetaDataToHal(mAudioSink, audioMeta);
1107                mCurrentOffloadInfo = offloadInfo;
1108                err = mAudioSink->start();
1109                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1110            }
1111            if (err != OK) {
1112                // Clean up, fall back to non offload mode.
1113                mAudioSink->close();
1114                mRenderer->signalDisableOffloadAudio();
1115                mOffloadAudio = false;
1116                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1117                ALOGV("openAudioSink: offload failed");
1118            }
1119        }
1120    }
1121    if (!offloadOnly && !mOffloadAudio) {
1122        flags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1123        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1124
1125        audioSinkChanged = true;
1126        mAudioSink->close();
1127        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1128        CHECK_EQ(mAudioSink->open(
1129                    sampleRate,
1130                    numChannels,
1131                    (audio_channel_mask_t)channelMask,
1132                    AUDIO_FORMAT_PCM_16_BIT,
1133                    8 /* bufferCount */,
1134                    NULL,
1135                    NULL,
1136                    (audio_output_flags_t)flags),
1137                 (status_t)OK);
1138        mAudioSink->start();
1139    }
1140    if (audioSinkChanged) {
1141        mRenderer->signalAudioSinkChanged();
1142    }
1143}
1144
1145void NuPlayer::closeAudioSink() {
1146    mAudioSink->close();
1147    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1148}
1149
1150status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
1151    if (*decoder != NULL) {
1152        return OK;
1153    }
1154
1155    sp<AMessage> format = mSource->getFormat(audio);
1156
1157    if (format == NULL) {
1158        return -EWOULDBLOCK;
1159    }
1160
1161    if (!audio) {
1162        AString mime;
1163        CHECK(format->findString("mime", &mime));
1164        mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
1165
1166        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
1167        mCCDecoder = new CCDecoder(ccNotify);
1168
1169        if (mSourceFlags & Source::FLAG_SECURE) {
1170            format->setInt32("secure", true);
1171        }
1172    }
1173
1174    if (audio) {
1175        sp<AMessage> notify = new AMessage(kWhatAudioNotify, id());
1176        ++mAudioDecoderGeneration;
1177        notify->setInt32("generation", mAudioDecoderGeneration);
1178
1179        if (mOffloadAudio) {
1180            *decoder = new DecoderPassThrough(notify);
1181        } else {
1182            *decoder = new Decoder(notify);
1183        }
1184    } else {
1185        sp<AMessage> notify = new AMessage(kWhatVideoNotify, id());
1186        ++mVideoDecoderGeneration;
1187        notify->setInt32("generation", mVideoDecoderGeneration);
1188
1189        *decoder = new Decoder(notify, mNativeWindow);
1190    }
1191    (*decoder)->init();
1192    (*decoder)->configure(format);
1193
1194    // allocate buffers to decrypt widevine source buffers
1195    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1196        Vector<sp<ABuffer> > inputBufs;
1197        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1198
1199        Vector<MediaBuffer *> mediaBufs;
1200        for (size_t i = 0; i < inputBufs.size(); i++) {
1201            const sp<ABuffer> &buffer = inputBufs[i];
1202            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1203            mediaBufs.push(mbuf);
1204        }
1205
1206        status_t err = mSource->setBuffers(audio, mediaBufs);
1207        if (err != OK) {
1208            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1209                mediaBufs[i]->release();
1210            }
1211            mediaBufs.clear();
1212            ALOGE("Secure source didn't support secure mediaBufs.");
1213            return err;
1214        }
1215    }
1216    return OK;
1217}
1218
1219status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
1220    sp<AMessage> reply;
1221    CHECK(msg->findMessage("reply", &reply));
1222
1223    if ((audio && IsFlushingState(mFlushingAudio))
1224            || (!audio && IsFlushingState(mFlushingVideo))) {
1225        reply->setInt32("err", INFO_DISCONTINUITY);
1226        reply->post();
1227        return OK;
1228    }
1229
1230    sp<ABuffer> accessUnit;
1231
1232    bool dropAccessUnit;
1233    do {
1234        status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
1235
1236        if (err == -EWOULDBLOCK) {
1237            return err;
1238        } else if (err != OK) {
1239            if (err == INFO_DISCONTINUITY) {
1240                int32_t type;
1241                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
1242
1243                bool formatChange =
1244                    (audio &&
1245                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
1246                    || (!audio &&
1247                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
1248
1249                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
1250
1251                ALOGI("%s discontinuity (formatChange=%d, time=%d)",
1252                     audio ? "audio" : "video", formatChange, timeChange);
1253
1254                if (audio) {
1255                    mSkipRenderingAudioUntilMediaTimeUs = -1;
1256                } else {
1257                    mSkipRenderingVideoUntilMediaTimeUs = -1;
1258                }
1259
1260                if (timeChange) {
1261                    sp<AMessage> extra;
1262                    if (accessUnit->meta()->findMessage("extra", &extra)
1263                            && extra != NULL) {
1264                        int64_t resumeAtMediaTimeUs;
1265                        if (extra->findInt64(
1266                                    "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
1267                            ALOGI("suppressing rendering of %s until %lld us",
1268                                    audio ? "audio" : "video", resumeAtMediaTimeUs);
1269
1270                            if (audio) {
1271                                mSkipRenderingAudioUntilMediaTimeUs =
1272                                    resumeAtMediaTimeUs;
1273                            } else {
1274                                mSkipRenderingVideoUntilMediaTimeUs =
1275                                    resumeAtMediaTimeUs;
1276                            }
1277                        }
1278                    }
1279                }
1280
1281                mTimeDiscontinuityPending =
1282                    mTimeDiscontinuityPending || timeChange;
1283
1284                if (mFlushingAudio == NONE && mFlushingVideo == NONE) {
1285                    // And we'll resume scanning sources once we're done
1286                    // flushing.
1287                    mDeferredActions.push_front(
1288                            new SimpleAction(
1289                                &NuPlayer::performScanSources));
1290                }
1291
1292                if (formatChange || timeChange) {
1293
1294                    sp<AMessage> newFormat = mSource->getFormat(audio);
1295                    sp<Decoder> &decoder = audio ? mAudioDecoder : mVideoDecoder;
1296                    if (formatChange && !decoder->supportsSeamlessFormatChange(newFormat)) {
1297                        flushDecoder(audio, /* needShutdown = */ true);
1298                    } else {
1299                        flushDecoder(audio, /* needShutdown = */ false);
1300                        err = OK;
1301                    }
1302                } else {
1303                    // This stream is unaffected by the discontinuity
1304
1305                    if (audio) {
1306                        mFlushingAudio = FLUSHED;
1307                    } else {
1308                        mFlushingVideo = FLUSHED;
1309                    }
1310
1311                    finishFlushIfPossible();
1312
1313                    return -EWOULDBLOCK;
1314                }
1315            }
1316
1317            reply->setInt32("err", err);
1318            reply->post();
1319            return OK;
1320        }
1321
1322        if (!audio) {
1323            ++mNumFramesTotal;
1324        }
1325
1326        dropAccessUnit = false;
1327        if (!audio
1328                && !(mSourceFlags & Source::FLAG_SECURE)
1329                && mVideoLateByUs > 100000ll
1330                && mVideoIsAVC
1331                && !IsAVCReferenceFrame(accessUnit)) {
1332            dropAccessUnit = true;
1333            ++mNumFramesDropped;
1334        }
1335    } while (dropAccessUnit);
1336
1337    // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
1338
1339#if 0
1340    int64_t mediaTimeUs;
1341    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
1342    ALOGV("feeding %s input buffer at media time %.2f secs",
1343         audio ? "audio" : "video",
1344         mediaTimeUs / 1E6);
1345#endif
1346
1347    if (!audio) {
1348        mCCDecoder->decode(accessUnit);
1349    }
1350
1351    reply->setBuffer("buffer", accessUnit);
1352    reply->post();
1353
1354    return OK;
1355}
1356
1357void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
1358    // ALOGV("renderBuffer %s", audio ? "audio" : "video");
1359
1360    sp<AMessage> reply;
1361    CHECK(msg->findMessage("reply", &reply));
1362
1363    if (IsFlushingState(audio ? mFlushingAudio : mFlushingVideo)) {
1364        // We're currently attempting to flush the decoder, in order
1365        // to complete this, the decoder wants all its buffers back,
1366        // so we don't want any output buffers it sent us (from before
1367        // we initiated the flush) to be stuck in the renderer's queue.
1368
1369        ALOGV("we're still flushing the %s decoder, sending its output buffer"
1370             " right back.", audio ? "audio" : "video");
1371
1372        reply->post();
1373        return;
1374    }
1375
1376    sp<ABuffer> buffer;
1377    CHECK(msg->findBuffer("buffer", &buffer));
1378
1379    int64_t mediaTimeUs;
1380    CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));
1381
1382    int64_t &skipUntilMediaTimeUs =
1383        audio
1384            ? mSkipRenderingAudioUntilMediaTimeUs
1385            : mSkipRenderingVideoUntilMediaTimeUs;
1386
1387    if (skipUntilMediaTimeUs >= 0) {
1388
1389        if (mediaTimeUs < skipUntilMediaTimeUs) {
1390            ALOGV("dropping %s buffer at time %lld as requested.",
1391                 audio ? "audio" : "video",
1392                 mediaTimeUs);
1393
1394            reply->post();
1395            return;
1396        }
1397
1398        skipUntilMediaTimeUs = -1;
1399    }
1400
1401    if (!audio && mCCDecoder->isSelected()) {
1402        mCCDecoder->display(mediaTimeUs);
1403    }
1404
1405    mRenderer->queueBuffer(audio, buffer, reply);
1406}
1407
1408void NuPlayer::updateVideoSize(
1409        const sp<AMessage> &inputFormat,
1410        const sp<AMessage> &outputFormat) {
1411    if (inputFormat == NULL) {
1412        ALOGW("Unknown video size, reporting 0x0!");
1413        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1414        return;
1415    }
1416
1417    int32_t displayWidth, displayHeight;
1418    int32_t cropLeft, cropTop, cropRight, cropBottom;
1419
1420    if (outputFormat != NULL) {
1421        int32_t width, height;
1422        CHECK(outputFormat->findInt32("width", &width));
1423        CHECK(outputFormat->findInt32("height", &height));
1424
1425        int32_t cropLeft, cropTop, cropRight, cropBottom;
1426        CHECK(outputFormat->findRect(
1427                    "crop",
1428                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1429
1430        displayWidth = cropRight - cropLeft + 1;
1431        displayHeight = cropBottom - cropTop + 1;
1432
1433        ALOGV("Video output format changed to %d x %d "
1434             "(crop: %d x %d @ (%d, %d))",
1435             width, height,
1436             displayWidth,
1437             displayHeight,
1438             cropLeft, cropTop);
1439    } else {
1440        CHECK(inputFormat->findInt32("width", &displayWidth));
1441        CHECK(inputFormat->findInt32("height", &displayHeight));
1442
1443        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1444    }
1445
1446    // Take into account sample aspect ratio if necessary:
1447    int32_t sarWidth, sarHeight;
1448    if (inputFormat->findInt32("sar-width", &sarWidth)
1449            && inputFormat->findInt32("sar-height", &sarHeight)) {
1450        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1451
1452        displayWidth = (displayWidth * sarWidth) / sarHeight;
1453
1454        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1455    }
1456
1457    int32_t rotationDegrees;
1458    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1459        rotationDegrees = 0;
1460    }
1461
1462    if (rotationDegrees == 90 || rotationDegrees == 270) {
1463        int32_t tmp = displayWidth;
1464        displayWidth = displayHeight;
1465        displayHeight = tmp;
1466    }
1467
1468    notifyListener(
1469            MEDIA_SET_VIDEO_SIZE,
1470            displayWidth,
1471            displayHeight);
1472}
1473
1474void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1475    if (mDriver == NULL) {
1476        return;
1477    }
1478
1479    sp<NuPlayerDriver> driver = mDriver.promote();
1480
1481    if (driver == NULL) {
1482        return;
1483    }
1484
1485    driver->notifyListener(msg, ext1, ext2, in);
1486}
1487
1488void NuPlayer::flushDecoder(bool audio, bool needShutdown) {
1489    ALOGV("[%s] flushDecoder needShutdown=%d",
1490          audio ? "audio" : "video", needShutdown);
1491
1492    if ((audio && mAudioDecoder == NULL) || (!audio && mVideoDecoder == NULL)) {
1493        ALOGI("flushDecoder %s without decoder present",
1494             audio ? "audio" : "video");
1495    }
1496
1497    // Make sure we don't continue to scan sources until we finish flushing.
1498    ++mScanSourcesGeneration;
1499    mScanSourcesPending = false;
1500
1501    (audio ? mAudioDecoder : mVideoDecoder)->signalFlush();
1502    mRenderer->flush(audio);
1503
1504    FlushStatus newStatus =
1505        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1506
1507    if (audio) {
1508        CHECK(mFlushingAudio == NONE
1509                || mFlushingAudio == AWAITING_DISCONTINUITY);
1510
1511        mFlushingAudio = newStatus;
1512
1513        if (mFlushingVideo == NONE) {
1514            mFlushingVideo = (mVideoDecoder != NULL)
1515                ? AWAITING_DISCONTINUITY
1516                : FLUSHED;
1517        }
1518    } else {
1519        CHECK(mFlushingVideo == NONE
1520                || mFlushingVideo == AWAITING_DISCONTINUITY);
1521
1522        mFlushingVideo = newStatus;
1523
1524        if (mFlushingAudio == NONE) {
1525            mFlushingAudio = (mAudioDecoder != NULL)
1526                ? AWAITING_DISCONTINUITY
1527                : FLUSHED;
1528        }
1529    }
1530}
1531
1532void NuPlayer::queueDecoderShutdown(
1533        bool audio, bool video, const sp<AMessage> &reply) {
1534    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1535
1536    mDeferredActions.push_back(
1537            new ShutdownDecoderAction(audio, video));
1538
1539    mDeferredActions.push_back(
1540            new SimpleAction(&NuPlayer::performScanSources));
1541
1542    mDeferredActions.push_back(new PostMessageAction(reply));
1543
1544    processDeferredActions();
1545}
1546
1547status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1548    mVideoScalingMode = mode;
1549    if (mNativeWindow != NULL) {
1550        status_t ret = native_window_set_scaling_mode(
1551                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1552        if (ret != OK) {
1553            ALOGE("Failed to set scaling mode (%d): %s",
1554                -ret, strerror(-ret));
1555            return ret;
1556        }
1557    }
1558    return OK;
1559}
1560
1561status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1562    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id());
1563    msg->setPointer("reply", reply);
1564
1565    sp<AMessage> response;
1566    status_t err = msg->postAndAwaitResponse(&response);
1567    return err;
1568}
1569
1570status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1571    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id());
1572    msg->setPointer("reply", reply);
1573    msg->setInt32("type", type);
1574
1575    sp<AMessage> response;
1576    status_t err = msg->postAndAwaitResponse(&response);
1577    if (err == OK && response != NULL) {
1578        CHECK(response->findInt32("err", &err));
1579    }
1580    return err;
1581}
1582
1583status_t NuPlayer::selectTrack(size_t trackIndex, bool select) {
1584    sp<AMessage> msg = new AMessage(kWhatSelectTrack, id());
1585    msg->setSize("trackIndex", trackIndex);
1586    msg->setInt32("select", select);
1587
1588    sp<AMessage> response;
1589    status_t err = msg->postAndAwaitResponse(&response);
1590
1591    if (err != OK) {
1592        return err;
1593    }
1594
1595    if (!response->findInt32("err", &err)) {
1596        err = OK;
1597    }
1598
1599    return err;
1600}
1601
1602void NuPlayer::schedulePollDuration() {
1603    sp<AMessage> msg = new AMessage(kWhatPollDuration, id());
1604    msg->setInt32("generation", mPollDurationGeneration);
1605    msg->post();
1606}
1607
1608void NuPlayer::cancelPollDuration() {
1609    ++mPollDurationGeneration;
1610}
1611
1612void NuPlayer::processDeferredActions() {
1613    while (!mDeferredActions.empty()) {
1614        // We won't execute any deferred actions until we're no longer in
1615        // an intermediate state, i.e. one more more decoders are currently
1616        // flushing or shutting down.
1617
1618        if (mRenderer != NULL) {
1619            // There's an edge case where the renderer owns all output
1620            // buffers and is paused, therefore the decoder will not read
1621            // more input data and will never encounter the matching
1622            // discontinuity. To avoid this, we resume the renderer.
1623
1624            if (mFlushingAudio == AWAITING_DISCONTINUITY
1625                    || mFlushingVideo == AWAITING_DISCONTINUITY) {
1626                mRenderer->resume();
1627            }
1628        }
1629
1630        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1631            // We're currently flushing, postpone the reset until that's
1632            // completed.
1633
1634            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1635                  mFlushingAudio, mFlushingVideo);
1636
1637            break;
1638        }
1639
1640        sp<Action> action = *mDeferredActions.begin();
1641        mDeferredActions.erase(mDeferredActions.begin());
1642
1643        action->execute(this);
1644    }
1645}
1646
1647void NuPlayer::performSeek(int64_t seekTimeUs) {
1648    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs)",
1649          seekTimeUs,
1650          seekTimeUs / 1E6);
1651
1652    mSource->seekTo(seekTimeUs);
1653    ++mTimedTextGeneration;
1654
1655    if (mDriver != NULL) {
1656        sp<NuPlayerDriver> driver = mDriver.promote();
1657        if (driver != NULL) {
1658            driver->notifyPosition(seekTimeUs);
1659            driver->notifySeekComplete();
1660        }
1661    }
1662
1663    // everything's flushed, continue playback.
1664}
1665
1666void NuPlayer::performDecoderFlush() {
1667    ALOGV("performDecoderFlush");
1668
1669    if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
1670        return;
1671    }
1672
1673    mTimeDiscontinuityPending = true;
1674
1675    if (mAudioDecoder != NULL) {
1676        flushDecoder(true /* audio */, false /* needShutdown */);
1677    }
1678
1679    if (mVideoDecoder != NULL) {
1680        flushDecoder(false /* audio */, false /* needShutdown */);
1681    }
1682}
1683
1684void NuPlayer::performDecoderShutdown(bool audio, bool video) {
1685    ALOGV("performDecoderShutdown audio=%d, video=%d", audio, video);
1686
1687    if ((!audio || mAudioDecoder == NULL)
1688            && (!video || mVideoDecoder == NULL)) {
1689        return;
1690    }
1691
1692    mTimeDiscontinuityPending = true;
1693
1694    if (mFlushingAudio == NONE && (!audio || mAudioDecoder == NULL)) {
1695        mFlushingAudio = FLUSHED;
1696    }
1697
1698    if (mFlushingVideo == NONE && (!video || mVideoDecoder == NULL)) {
1699        mFlushingVideo = FLUSHED;
1700    }
1701
1702    if (audio && mAudioDecoder != NULL) {
1703        flushDecoder(true /* audio */, true /* needShutdown */);
1704    }
1705
1706    if (video && mVideoDecoder != NULL) {
1707        flushDecoder(false /* audio */, true /* needShutdown */);
1708    }
1709}
1710
1711void NuPlayer::performReset() {
1712    ALOGV("performReset");
1713
1714    CHECK(mAudioDecoder == NULL);
1715    CHECK(mVideoDecoder == NULL);
1716
1717    cancelPollDuration();
1718
1719    ++mScanSourcesGeneration;
1720    mScanSourcesPending = false;
1721
1722    if (mRendererLooper != NULL) {
1723        if (mRenderer != NULL) {
1724            mRendererLooper->unregisterHandler(mRenderer->id());
1725        }
1726        mRendererLooper->stop();
1727        mRendererLooper.clear();
1728    }
1729    mRenderer.clear();
1730
1731    if (mSource != NULL) {
1732        mSource->stop();
1733
1734        mSource.clear();
1735    }
1736
1737    if (mDriver != NULL) {
1738        sp<NuPlayerDriver> driver = mDriver.promote();
1739        if (driver != NULL) {
1740            driver->notifyResetComplete();
1741        }
1742    }
1743
1744    mStarted = false;
1745}
1746
1747void NuPlayer::performScanSources() {
1748    ALOGV("performScanSources");
1749
1750    if (!mStarted) {
1751        return;
1752    }
1753
1754    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1755        postScanSources();
1756    }
1757}
1758
1759void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1760    ALOGV("performSetSurface");
1761
1762    mNativeWindow = wrapper;
1763
1764    // XXX - ignore error from setVideoScalingMode for now
1765    setVideoScalingMode(mVideoScalingMode);
1766}
1767
1768void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1769    int32_t what;
1770    CHECK(msg->findInt32("what", &what));
1771
1772    switch (what) {
1773        case Source::kWhatPrepared:
1774        {
1775            if (mSource == NULL) {
1776                // This is a stale notification from a source that was
1777                // asynchronously preparing when the client called reset().
1778                // We handled the reset, the source is gone.
1779                break;
1780            }
1781
1782            int32_t err;
1783            CHECK(msg->findInt32("err", &err));
1784
1785            sp<NuPlayerDriver> driver = mDriver.promote();
1786            if (driver != NULL) {
1787                // notify duration first, so that it's definitely set when
1788                // the app received the "prepare complete" callback.
1789                int64_t durationUs;
1790                if (mSource->getDuration(&durationUs) == OK) {
1791                    driver->notifyDuration(durationUs);
1792                }
1793                driver->notifyPrepareCompleted(err);
1794            }
1795
1796            break;
1797        }
1798
1799        case Source::kWhatFlagsChanged:
1800        {
1801            uint32_t flags;
1802            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1803
1804            sp<NuPlayerDriver> driver = mDriver.promote();
1805            if (driver != NULL) {
1806                driver->notifyFlagsChanged(flags);
1807            }
1808
1809            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1810                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
1811                cancelPollDuration();
1812            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1813                    && (flags & Source::FLAG_DYNAMIC_DURATION)
1814                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
1815                schedulePollDuration();
1816            }
1817
1818            mSourceFlags = flags;
1819            break;
1820        }
1821
1822        case Source::kWhatVideoSizeChanged:
1823        {
1824            sp<AMessage> format;
1825            CHECK(msg->findMessage("format", &format));
1826
1827            updateVideoSize(format);
1828            break;
1829        }
1830
1831        case Source::kWhatBufferingStart:
1832        {
1833            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
1834            break;
1835        }
1836
1837        case Source::kWhatBufferingEnd:
1838        {
1839            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
1840            break;
1841        }
1842
1843        case Source::kWhatSubtitleData:
1844        {
1845            sp<ABuffer> buffer;
1846            CHECK(msg->findBuffer("buffer", &buffer));
1847
1848            sendSubtitleData(buffer, 0 /* baseIndex */);
1849            break;
1850        }
1851
1852        case Source::kWhatTimedTextData:
1853        {
1854            int32_t generation;
1855            if (msg->findInt32("generation", &generation)
1856                    && generation != mTimedTextGeneration) {
1857                break;
1858            }
1859
1860            sp<ABuffer> buffer;
1861            CHECK(msg->findBuffer("buffer", &buffer));
1862
1863            sp<NuPlayerDriver> driver = mDriver.promote();
1864            if (driver == NULL) {
1865                break;
1866            }
1867
1868            int posMs;
1869            int64_t timeUs, posUs;
1870            driver->getCurrentPosition(&posMs);
1871            posUs = posMs * 1000;
1872            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1873
1874            if (posUs < timeUs) {
1875                if (!msg->findInt32("generation", &generation)) {
1876                    msg->setInt32("generation", mTimedTextGeneration);
1877                }
1878                msg->post(timeUs - posUs);
1879            } else {
1880                sendTimedTextData(buffer);
1881            }
1882            break;
1883        }
1884
1885        case Source::kWhatQueueDecoderShutdown:
1886        {
1887            int32_t audio, video;
1888            CHECK(msg->findInt32("audio", &audio));
1889            CHECK(msg->findInt32("video", &video));
1890
1891            sp<AMessage> reply;
1892            CHECK(msg->findMessage("reply", &reply));
1893
1894            queueDecoderShutdown(audio, video, reply);
1895            break;
1896        }
1897
1898        default:
1899            TRESPASS();
1900    }
1901}
1902
1903void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
1904    int32_t what;
1905    CHECK(msg->findInt32("what", &what));
1906
1907    switch (what) {
1908        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
1909        {
1910            sp<ABuffer> buffer;
1911            CHECK(msg->findBuffer("buffer", &buffer));
1912
1913            size_t inbandTracks = 0;
1914            if (mSource != NULL) {
1915                inbandTracks = mSource->getTrackCount();
1916            }
1917
1918            sendSubtitleData(buffer, inbandTracks);
1919            break;
1920        }
1921
1922        case NuPlayer::CCDecoder::kWhatTrackAdded:
1923        {
1924            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
1925
1926            break;
1927        }
1928
1929        default:
1930            TRESPASS();
1931    }
1932
1933
1934}
1935
1936void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
1937    int32_t trackIndex;
1938    int64_t timeUs, durationUs;
1939    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
1940    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1941    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
1942
1943    Parcel in;
1944    in.writeInt32(trackIndex + baseIndex);
1945    in.writeInt64(timeUs);
1946    in.writeInt64(durationUs);
1947    in.writeInt32(buffer->size());
1948    in.writeInt32(buffer->size());
1949    in.write(buffer->data(), buffer->size());
1950
1951    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
1952}
1953
1954void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
1955    const void *data;
1956    size_t size = 0;
1957    int64_t timeUs;
1958    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
1959
1960    AString mime;
1961    CHECK(buffer->meta()->findString("mime", &mime));
1962    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
1963
1964    data = buffer->data();
1965    size = buffer->size();
1966
1967    Parcel parcel;
1968    if (size > 0) {
1969        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1970        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
1971        TextDescriptions::getParcelOfDescriptions(
1972                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
1973    }
1974
1975    if ((parcel.dataSize() > 0)) {
1976        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
1977    } else {  // send an empty timed text
1978        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
1979    }
1980}
1981////////////////////////////////////////////////////////////////////////////////
1982
1983sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
1984    sp<MetaData> meta = getFormatMeta(audio);
1985
1986    if (meta == NULL) {
1987        return NULL;
1988    }
1989
1990    sp<AMessage> msg = new AMessage;
1991
1992    if(convertMetaDataToMessage(meta, &msg) == OK) {
1993        return msg;
1994    }
1995    return NULL;
1996}
1997
1998void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
1999    sp<AMessage> notify = dupNotify();
2000    notify->setInt32("what", kWhatFlagsChanged);
2001    notify->setInt32("flags", flags);
2002    notify->post();
2003}
2004
2005void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
2006    sp<AMessage> notify = dupNotify();
2007    notify->setInt32("what", kWhatVideoSizeChanged);
2008    notify->setMessage("format", format);
2009    notify->post();
2010}
2011
2012void NuPlayer::Source::notifyPrepared(status_t err) {
2013    sp<AMessage> notify = dupNotify();
2014    notify->setInt32("what", kWhatPrepared);
2015    notify->setInt32("err", err);
2016    notify->post();
2017}
2018
2019void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
2020    TRESPASS();
2021}
2022
2023}  // namespace android
2024