NuPlayer.cpp revision 9c03a40367c149526c31ddf14a518ba2036195a5 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayer"
19#include <utils/Log.h>
20
21#include "NuPlayer.h"
22
23#include "HTTPLiveSource.h"
24#include "NuPlayerDecoder.h"
25#include "NuPlayerDecoderPassThrough.h"
26#include "NuPlayerDriver.h"
27#include "NuPlayerRenderer.h"
28#include "NuPlayerSource.h"
29#include "RTSPSource.h"
30#include "StreamingSource.h"
31#include "GenericSource.h"
32#include "TextDescriptions.h"
33
34#include "ATSParser.h"
35
36#include <media/stagefright/foundation/hexdump.h>
37#include <media/stagefright/foundation/ABuffer.h>
38#include <media/stagefright/foundation/ADebug.h>
39#include <media/stagefright/foundation/AMessage.h>
40#include <media/stagefright/MediaBuffer.h>
41#include <media/stagefright/MediaDefs.h>
42#include <media/stagefright/MediaErrors.h>
43#include <media/stagefright/MetaData.h>
44#include <gui/IGraphicBufferProducer.h>
45
46#include "avc_utils.h"
47
48#include "ESDS.h"
49#include <media/stagefright/Utils.h>
50
51namespace android {
52
53struct NuPlayer::Action : public RefBase {
54    Action() {}
55
56    virtual void execute(NuPlayer *player) = 0;
57
58private:
59    DISALLOW_EVIL_CONSTRUCTORS(Action);
60};
61
62struct NuPlayer::SeekAction : public Action {
63    SeekAction(int64_t seekTimeUs)
64        : mSeekTimeUs(seekTimeUs) {
65    }
66
67    virtual void execute(NuPlayer *player) {
68        player->performSeek(mSeekTimeUs);
69    }
70
71private:
72    int64_t mSeekTimeUs;
73
74    DISALLOW_EVIL_CONSTRUCTORS(SeekAction);
75};
76
77struct NuPlayer::SetSurfaceAction : public Action {
78    SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper)
79        : mWrapper(wrapper) {
80    }
81
82    virtual void execute(NuPlayer *player) {
83        player->performSetSurface(mWrapper);
84    }
85
86private:
87    sp<NativeWindowWrapper> mWrapper;
88
89    DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction);
90};
91
92struct NuPlayer::ShutdownDecoderAction : public Action {
93    ShutdownDecoderAction(bool audio, bool video)
94        : mAudio(audio),
95          mVideo(video) {
96    }
97
98    virtual void execute(NuPlayer *player) {
99        player->performDecoderShutdown(mAudio, mVideo);
100    }
101
102private:
103    bool mAudio;
104    bool mVideo;
105
106    DISALLOW_EVIL_CONSTRUCTORS(ShutdownDecoderAction);
107};
108
109struct NuPlayer::PostMessageAction : public Action {
110    PostMessageAction(const sp<AMessage> &msg)
111        : mMessage(msg) {
112    }
113
114    virtual void execute(NuPlayer *) {
115        mMessage->post();
116    }
117
118private:
119    sp<AMessage> mMessage;
120
121    DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction);
122};
123
124// Use this if there's no state necessary to save in order to execute
125// the action.
126struct NuPlayer::SimpleAction : public Action {
127    typedef void (NuPlayer::*ActionFunc)();
128
129    SimpleAction(ActionFunc func)
130        : mFunc(func) {
131    }
132
133    virtual void execute(NuPlayer *player) {
134        (player->*mFunc)();
135    }
136
137private:
138    ActionFunc mFunc;
139
140    DISALLOW_EVIL_CONSTRUCTORS(SimpleAction);
141};
142
143////////////////////////////////////////////////////////////////////////////////
144
145NuPlayer::NuPlayer()
146    : mUIDValid(false),
147      mSourceFlags(0),
148      mVideoIsAVC(false),
149      mOffloadAudio(false),
150      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
151      mAudioDecoderGeneration(0),
152      mVideoDecoderGeneration(0),
153      mAudioEOS(false),
154      mVideoEOS(false),
155      mScanSourcesPending(false),
156      mScanSourcesGeneration(0),
157      mPollDurationGeneration(0),
158      mTimedTextGeneration(0),
159      mTimeDiscontinuityPending(false),
160      mFlushingAudio(NONE),
161      mFlushingVideo(NONE),
162      mSkipRenderingAudioUntilMediaTimeUs(-1ll),
163      mSkipRenderingVideoUntilMediaTimeUs(-1ll),
164      mVideoLateByUs(0ll),
165      mNumFramesTotal(0ll),
166      mNumFramesDropped(0ll),
167      mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
168      mStarted(false) {
169}
170
171NuPlayer::~NuPlayer() {
172}
173
174void NuPlayer::setUID(uid_t uid) {
175    mUIDValid = true;
176    mUID = uid;
177}
178
179void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) {
180    mDriver = driver;
181}
182
183void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) {
184    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
185
186    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
187
188    msg->setObject("source", new StreamingSource(notify, source));
189    msg->post();
190}
191
192static bool IsHTTPLiveURL(const char *url) {
193    if (!strncasecmp("http://", url, 7)
194            || !strncasecmp("https://", url, 8)
195            || !strncasecmp("file://", url, 7)) {
196        size_t len = strlen(url);
197        if (len >= 5 && !strcasecmp(".m3u8", &url[len - 5])) {
198            return true;
199        }
200
201        if (strstr(url,"m3u8")) {
202            return true;
203        }
204    }
205
206    return false;
207}
208
209void NuPlayer::setDataSourceAsync(
210        const sp<IMediaHTTPService> &httpService,
211        const char *url,
212        const KeyedVector<String8, String8> *headers) {
213
214    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
215    size_t len = strlen(url);
216
217    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
218
219    sp<Source> source;
220    if (IsHTTPLiveURL(url)) {
221        source = new HTTPLiveSource(notify, httpService, url, headers);
222    } else if (!strncasecmp(url, "rtsp://", 7)) {
223        source = new RTSPSource(
224                notify, httpService, url, headers, mUIDValid, mUID);
225    } else if ((!strncasecmp(url, "http://", 7)
226                || !strncasecmp(url, "https://", 8))
227                    && ((len >= 4 && !strcasecmp(".sdp", &url[len - 4]))
228                    || strstr(url, ".sdp?"))) {
229        source = new RTSPSource(
230                notify, httpService, url, headers, mUIDValid, mUID, true);
231    } else {
232        sp<GenericSource> genericSource =
233                new GenericSource(notify, mUIDValid, mUID);
234        // Don't set FLAG_SECURE on mSourceFlags here for widevine.
235        // The correct flags will be updated in Source::kWhatFlagsChanged
236        // handler when  GenericSource is prepared.
237
238        status_t err = genericSource->setDataSource(httpService, url, headers);
239
240        if (err == OK) {
241            source = genericSource;
242        } else {
243            ALOGE("Failed to set data source!");
244        }
245    }
246    msg->setObject("source", source);
247    msg->post();
248}
249
250void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
251    sp<AMessage> msg = new AMessage(kWhatSetDataSource, id());
252
253    sp<AMessage> notify = new AMessage(kWhatSourceNotify, id());
254
255    sp<GenericSource> source =
256            new GenericSource(notify, mUIDValid, mUID);
257
258    status_t err = source->setDataSource(fd, offset, length);
259
260    if (err != OK) {
261        ALOGE("Failed to set data source!");
262        source = NULL;
263    }
264
265    msg->setObject("source", source);
266    msg->post();
267}
268
269void NuPlayer::prepareAsync() {
270    (new AMessage(kWhatPrepare, id()))->post();
271}
272
273void NuPlayer::setVideoSurfaceTextureAsync(
274        const sp<IGraphicBufferProducer> &bufferProducer) {
275    sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id());
276
277    if (bufferProducer == NULL) {
278        msg->setObject("native-window", NULL);
279    } else {
280        msg->setObject(
281                "native-window",
282                new NativeWindowWrapper(
283                    new Surface(bufferProducer, true /* controlledByApp */)));
284    }
285
286    msg->post();
287}
288
289void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) {
290    sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id());
291    msg->setObject("sink", sink);
292    msg->post();
293}
294
295void NuPlayer::start() {
296    (new AMessage(kWhatStart, id()))->post();
297}
298
299void NuPlayer::pause() {
300    (new AMessage(kWhatPause, id()))->post();
301}
302
303void NuPlayer::resume() {
304    (new AMessage(kWhatResume, id()))->post();
305}
306
307void NuPlayer::resetAsync() {
308    (new AMessage(kWhatReset, id()))->post();
309}
310
311void NuPlayer::seekToAsync(int64_t seekTimeUs) {
312    sp<AMessage> msg = new AMessage(kWhatSeek, id());
313    msg->setInt64("seekTimeUs", seekTimeUs);
314    msg->post();
315}
316
317// static
318bool NuPlayer::IsFlushingState(FlushStatus state, bool *needShutdown) {
319    switch (state) {
320        case FLUSHING_DECODER:
321            if (needShutdown != NULL) {
322                *needShutdown = false;
323            }
324            return true;
325
326        case FLUSHING_DECODER_SHUTDOWN:
327            if (needShutdown != NULL) {
328                *needShutdown = true;
329            }
330            return true;
331
332        default:
333            return false;
334    }
335}
336
337void NuPlayer::writeTrackInfo(
338        Parcel* reply, const sp<AMessage> format) const {
339    int32_t trackType;
340    CHECK(format->findInt32("type", &trackType));
341
342    AString lang;
343    CHECK(format->findString("language", &lang));
344
345    reply->writeInt32(2); // write something non-zero
346    reply->writeInt32(trackType);
347    reply->writeString16(String16(lang.c_str()));
348
349    if (trackType == MEDIA_TRACK_TYPE_SUBTITLE) {
350        AString mime;
351        CHECK(format->findString("mime", &mime));
352
353        int32_t isAuto, isDefault, isForced;
354        CHECK(format->findInt32("auto", &isAuto));
355        CHECK(format->findInt32("default", &isDefault));
356        CHECK(format->findInt32("forced", &isForced));
357
358        reply->writeString16(String16(mime.c_str()));
359        reply->writeInt32(isAuto);
360        reply->writeInt32(isDefault);
361        reply->writeInt32(isForced);
362    }
363}
364
365void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
366    switch (msg->what()) {
367        case kWhatSetDataSource:
368        {
369            ALOGV("kWhatSetDataSource");
370
371            CHECK(mSource == NULL);
372
373            status_t err = OK;
374            sp<RefBase> obj;
375            CHECK(msg->findObject("source", &obj));
376            if (obj != NULL) {
377                mSource = static_cast<Source *>(obj.get());
378            } else {
379                err = UNKNOWN_ERROR;
380            }
381
382            CHECK(mDriver != NULL);
383            sp<NuPlayerDriver> driver = mDriver.promote();
384            if (driver != NULL) {
385                driver->notifySetDataSourceCompleted(err);
386            }
387            break;
388        }
389
390        case kWhatPrepare:
391        {
392            mSource->prepareAsync();
393            break;
394        }
395
396        case kWhatGetTrackInfo:
397        {
398            uint32_t replyID;
399            CHECK(msg->senderAwaitsResponse(&replyID));
400
401            Parcel* reply;
402            CHECK(msg->findPointer("reply", (void**)&reply));
403
404            size_t inbandTracks = 0;
405            if (mSource != NULL) {
406                inbandTracks = mSource->getTrackCount();
407            }
408
409            size_t ccTracks = 0;
410            if (mCCDecoder != NULL) {
411                ccTracks = mCCDecoder->getTrackCount();
412            }
413
414            // total track count
415            reply->writeInt32(inbandTracks + ccTracks);
416
417            // write inband tracks
418            for (size_t i = 0; i < inbandTracks; ++i) {
419                writeTrackInfo(reply, mSource->getTrackInfo(i));
420            }
421
422            // write CC track
423            for (size_t i = 0; i < ccTracks; ++i) {
424                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
425            }
426
427            sp<AMessage> response = new AMessage;
428            response->postReply(replyID);
429            break;
430        }
431
432        case kWhatGetSelectedTrack:
433        {
434            status_t err = INVALID_OPERATION;
435            if (mSource != NULL) {
436                err = OK;
437
438                int32_t type32;
439                CHECK(msg->findInt32("type", (int32_t*)&type32));
440                media_track_type type = (media_track_type)type32;
441                ssize_t selectedTrack = mSource->getSelectedTrack(type);
442
443                Parcel* reply;
444                CHECK(msg->findPointer("reply", (void**)&reply));
445                reply->writeInt32(selectedTrack);
446            }
447
448            sp<AMessage> response = new AMessage;
449            response->setInt32("err", err);
450
451            uint32_t replyID;
452            CHECK(msg->senderAwaitsResponse(&replyID));
453            response->postReply(replyID);
454            break;
455        }
456
457        case kWhatSelectTrack:
458        {
459            uint32_t replyID;
460            CHECK(msg->senderAwaitsResponse(&replyID));
461
462            size_t trackIndex;
463            int32_t select;
464            CHECK(msg->findSize("trackIndex", &trackIndex));
465            CHECK(msg->findInt32("select", &select));
466
467            status_t err = INVALID_OPERATION;
468
469            size_t inbandTracks = 0;
470            if (mSource != NULL) {
471                inbandTracks = mSource->getTrackCount();
472            }
473            size_t ccTracks = 0;
474            if (mCCDecoder != NULL) {
475                ccTracks = mCCDecoder->getTrackCount();
476            }
477
478            if (trackIndex < inbandTracks) {
479                err = mSource->selectTrack(trackIndex, select);
480
481                if (!select && err == OK) {
482                    int32_t type;
483                    sp<AMessage> info = mSource->getTrackInfo(trackIndex);
484                    if (info != NULL
485                            && info->findInt32("type", &type)
486                            && type == MEDIA_TRACK_TYPE_TIMEDTEXT) {
487                        ++mTimedTextGeneration;
488                    }
489                }
490            } else {
491                trackIndex -= inbandTracks;
492
493                if (trackIndex < ccTracks) {
494                    err = mCCDecoder->selectTrack(trackIndex, select);
495                }
496            }
497
498            sp<AMessage> response = new AMessage;
499            response->setInt32("err", err);
500
501            response->postReply(replyID);
502            break;
503        }
504
505        case kWhatPollDuration:
506        {
507            int32_t generation;
508            CHECK(msg->findInt32("generation", &generation));
509
510            if (generation != mPollDurationGeneration) {
511                // stale
512                break;
513            }
514
515            int64_t durationUs;
516            if (mDriver != NULL && mSource->getDuration(&durationUs) == OK) {
517                sp<NuPlayerDriver> driver = mDriver.promote();
518                if (driver != NULL) {
519                    driver->notifyDuration(durationUs);
520                }
521            }
522
523            msg->post(1000000ll);  // poll again in a second.
524            break;
525        }
526
527        case kWhatSetVideoNativeWindow:
528        {
529            ALOGV("kWhatSetVideoNativeWindow");
530
531            mDeferredActions.push_back(
532                    new ShutdownDecoderAction(
533                        false /* audio */, true /* video */));
534
535            sp<RefBase> obj;
536            CHECK(msg->findObject("native-window", &obj));
537
538            mDeferredActions.push_back(
539                    new SetSurfaceAction(
540                        static_cast<NativeWindowWrapper *>(obj.get())));
541
542            if (obj != NULL) {
543                // If there is a new surface texture, instantiate decoders
544                // again if possible.
545                mDeferredActions.push_back(
546                        new SimpleAction(&NuPlayer::performScanSources));
547            }
548
549            processDeferredActions();
550            break;
551        }
552
553        case kWhatSetAudioSink:
554        {
555            ALOGV("kWhatSetAudioSink");
556
557            sp<RefBase> obj;
558            CHECK(msg->findObject("sink", &obj));
559
560            mAudioSink = static_cast<MediaPlayerBase::AudioSink *>(obj.get());
561            break;
562        }
563
564        case kWhatStart:
565        {
566            ALOGV("kWhatStart");
567
568            mVideoIsAVC = false;
569            mOffloadAudio = false;
570            mAudioEOS = false;
571            mVideoEOS = false;
572            mSkipRenderingAudioUntilMediaTimeUs = -1;
573            mSkipRenderingVideoUntilMediaTimeUs = -1;
574            mVideoLateByUs = 0;
575            mNumFramesTotal = 0;
576            mNumFramesDropped = 0;
577            mStarted = true;
578
579            /* instantiate decoders now for secure playback */
580            if (mSourceFlags & Source::FLAG_SECURE) {
581                if (mNativeWindow != NULL) {
582                    instantiateDecoder(false, &mVideoDecoder);
583                }
584
585                if (mAudioSink != NULL) {
586                    instantiateDecoder(true, &mAudioDecoder);
587                }
588            }
589
590            mSource->start();
591
592            uint32_t flags = 0;
593
594            if (mSource->isRealTime()) {
595                flags |= Renderer::FLAG_REAL_TIME;
596            }
597
598            sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
599            audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
600            if (mAudioSink != NULL) {
601                streamType = mAudioSink->getAudioStreamType();
602            }
603
604            sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
605
606            mOffloadAudio =
607                canOffloadStream(audioMeta, (videoFormat != NULL),
608                                 true /* is_streaming */, streamType);
609            if (mOffloadAudio) {
610                flags |= Renderer::FLAG_OFFLOAD_AUDIO;
611            }
612
613            mRenderer = new Renderer(
614                    mAudioSink,
615                    new AMessage(kWhatRendererNotify, id()),
616                    flags);
617
618            mRendererLooper = new ALooper;
619            mRendererLooper->setName("NuPlayerRenderer");
620            mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
621            mRendererLooper->registerHandler(mRenderer);
622
623            postScanSources();
624            break;
625        }
626
627        case kWhatScanSources:
628        {
629            int32_t generation;
630            CHECK(msg->findInt32("generation", &generation));
631            if (generation != mScanSourcesGeneration) {
632                // Drop obsolete msg.
633                break;
634            }
635
636            mScanSourcesPending = false;
637
638            ALOGV("scanning sources haveAudio=%d, haveVideo=%d",
639                 mAudioDecoder != NULL, mVideoDecoder != NULL);
640
641            bool mHadAnySourcesBefore =
642                (mAudioDecoder != NULL) || (mVideoDecoder != NULL);
643
644            // initialize video before audio because successful initialization of
645            // video may change deep buffer mode of audio.
646            if (mNativeWindow != NULL) {
647                instantiateDecoder(false, &mVideoDecoder);
648            }
649
650            if (mAudioSink != NULL) {
651                if (mOffloadAudio) {
652                    // open audio sink early under offload mode.
653                    sp<AMessage> format = mSource->getFormat(true /*audio*/);
654                    openAudioSink(format, true /*offloadOnly*/);
655                }
656                instantiateDecoder(true, &mAudioDecoder);
657            }
658
659            if (!mHadAnySourcesBefore
660                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
661                // This is the first time we've found anything playable.
662
663                if (mSourceFlags & Source::FLAG_DYNAMIC_DURATION) {
664                    schedulePollDuration();
665                }
666            }
667
668            status_t err;
669            if ((err = mSource->feedMoreTSData()) != OK) {
670                if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
671                    // We're not currently decoding anything (no audio or
672                    // video tracks found) and we just ran out of input data.
673
674                    if (err == ERROR_END_OF_STREAM) {
675                        notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
676                    } else {
677                        notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
678                    }
679                }
680                break;
681            }
682
683            if ((mAudioDecoder == NULL && mAudioSink != NULL)
684                    || (mVideoDecoder == NULL && mNativeWindow != NULL)) {
685                msg->post(100000ll);
686                mScanSourcesPending = true;
687            }
688            break;
689        }
690
691        case kWhatVideoNotify:
692        case kWhatAudioNotify:
693        {
694            bool audio = msg->what() == kWhatAudioNotify;
695
696            int32_t currentDecoderGeneration =
697                (audio? mAudioDecoderGeneration : mVideoDecoderGeneration);
698            int32_t requesterGeneration = currentDecoderGeneration - 1;
699            CHECK(msg->findInt32("generation", &requesterGeneration));
700
701            if (requesterGeneration != currentDecoderGeneration) {
702                ALOGV("got message from old %s decoder, generation(%d:%d)",
703                        audio ? "audio" : "video", requesterGeneration,
704                        currentDecoderGeneration);
705                sp<AMessage> reply;
706                if (!(msg->findMessage("reply", &reply))) {
707                    return;
708                }
709
710                reply->setInt32("err", INFO_DISCONTINUITY);
711                reply->post();
712                return;
713            }
714
715            int32_t what;
716            CHECK(msg->findInt32("what", &what));
717
718            if (what == Decoder::kWhatFillThisBuffer) {
719                status_t err = feedDecoderInputData(
720                        audio, msg);
721
722                if (err == -EWOULDBLOCK) {
723                    if (mSource->feedMoreTSData() == OK) {
724                        msg->post(10000ll);
725                    }
726                }
727            } else if (what == Decoder::kWhatEOS) {
728                int32_t err;
729                CHECK(msg->findInt32("err", &err));
730
731                if (err == ERROR_END_OF_STREAM) {
732                    ALOGV("got %s decoder EOS", audio ? "audio" : "video");
733                } else {
734                    ALOGV("got %s decoder EOS w/ error %d",
735                         audio ? "audio" : "video",
736                         err);
737                }
738
739                mRenderer->queueEOS(audio, err);
740            } else if (what == Decoder::kWhatFlushCompleted) {
741                bool needShutdown;
742
743                if (audio) {
744                    CHECK(IsFlushingState(mFlushingAudio, &needShutdown));
745                    mFlushingAudio = FLUSHED;
746                } else {
747                    CHECK(IsFlushingState(mFlushingVideo, &needShutdown));
748                    mFlushingVideo = FLUSHED;
749
750                    mVideoLateByUs = 0;
751                }
752
753                ALOGV("decoder %s flush completed", audio ? "audio" : "video");
754
755                if (needShutdown) {
756                    ALOGV("initiating %s decoder shutdown",
757                         audio ? "audio" : "video");
758
759                    getDecoder(audio)->initiateShutdown();
760
761                    if (audio) {
762                        mFlushingAudio = SHUTTING_DOWN_DECODER;
763                    } else {
764                        mFlushingVideo = SHUTTING_DOWN_DECODER;
765                    }
766                }
767
768                finishFlushIfPossible();
769            } else if (what == Decoder::kWhatOutputFormatChanged) {
770                sp<AMessage> format;
771                CHECK(msg->findMessage("format", &format));
772
773                if (audio) {
774                    openAudioSink(format, false /*offloadOnly*/);
775                } else {
776                    // video
777                    sp<AMessage> inputFormat =
778                            mSource->getFormat(false /* audio */);
779
780                    updateVideoSize(inputFormat, format);
781                }
782            } else if (what == Decoder::kWhatShutdownCompleted) {
783                ALOGV("%s shutdown completed", audio ? "audio" : "video");
784                if (audio) {
785                    mAudioDecoder.clear();
786
787                    CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER);
788                    mFlushingAudio = SHUT_DOWN;
789                } else {
790                    mVideoDecoder.clear();
791
792                    CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER);
793                    mFlushingVideo = SHUT_DOWN;
794                }
795
796                finishFlushIfPossible();
797            } else if (what == Decoder::kWhatError) {
798                ALOGE("Received error from %s decoder, aborting playback.",
799                     audio ? "audio" : "video");
800
801                status_t err;
802                if (!msg->findInt32("err", &err)) {
803                    err = UNKNOWN_ERROR;
804                }
805                mRenderer->queueEOS(audio, err);
806                if (audio && mFlushingAudio != NONE) {
807                    mAudioDecoder.clear();
808                    mFlushingAudio = SHUT_DOWN;
809                } else if (!audio && mFlushingVideo != NONE){
810                    mVideoDecoder.clear();
811                    mFlushingVideo = SHUT_DOWN;
812                }
813                finishFlushIfPossible();
814            } else if (what == Decoder::kWhatDrainThisBuffer) {
815                renderBuffer(audio, msg);
816            } else {
817                ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
818                      what,
819                      what >> 24,
820                      (what >> 16) & 0xff,
821                      (what >> 8) & 0xff,
822                      what & 0xff);
823            }
824
825            break;
826        }
827
828        case kWhatRendererNotify:
829        {
830            int32_t what;
831            CHECK(msg->findInt32("what", &what));
832
833            if (what == Renderer::kWhatEOS) {
834                int32_t audio;
835                CHECK(msg->findInt32("audio", &audio));
836
837                int32_t finalResult;
838                CHECK(msg->findInt32("finalResult", &finalResult));
839
840                if (audio) {
841                    mAudioEOS = true;
842                } else {
843                    mVideoEOS = true;
844                }
845
846                if (finalResult == ERROR_END_OF_STREAM) {
847                    ALOGV("reached %s EOS", audio ? "audio" : "video");
848                } else {
849                    ALOGE("%s track encountered an error (%d)",
850                         audio ? "audio" : "video", finalResult);
851
852                    notifyListener(
853                            MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, finalResult);
854                }
855
856                if ((mAudioEOS || mAudioDecoder == NULL)
857                        && (mVideoEOS || mVideoDecoder == NULL)) {
858                    notifyListener(MEDIA_PLAYBACK_COMPLETE, 0, 0);
859                }
860            } else if (what == Renderer::kWhatPosition) {
861                int64_t positionUs;
862                CHECK(msg->findInt64("positionUs", &positionUs));
863
864                CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs));
865
866                if (mDriver != NULL) {
867                    sp<NuPlayerDriver> driver = mDriver.promote();
868                    if (driver != NULL) {
869                        driver->notifyPosition(positionUs);
870
871                        driver->notifyFrameStats(
872                                mNumFramesTotal, mNumFramesDropped);
873                    }
874                }
875            } else if (what == Renderer::kWhatFlushComplete) {
876                int32_t audio;
877                CHECK(msg->findInt32("audio", &audio));
878
879                ALOGV("renderer %s flush completed.", audio ? "audio" : "video");
880            } else if (what == Renderer::kWhatVideoRenderingStart) {
881                notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0);
882            } else if (what == Renderer::kWhatMediaRenderingStart) {
883                ALOGV("media rendering started");
884                notifyListener(MEDIA_STARTED, 0, 0);
885            } else if (what == Renderer::kWhatAudioOffloadTearDown) {
886                ALOGV("Tear down audio offload, fall back to s/w path");
887                int64_t positionUs;
888                CHECK(msg->findInt64("positionUs", &positionUs));
889                closeAudioSink();
890                mAudioDecoder.clear();
891                mRenderer->flush(true /* audio */);
892                if (mVideoDecoder != NULL) {
893                    mRenderer->flush(false /* audio */);
894                }
895                mRenderer->signalDisableOffloadAudio();
896                mOffloadAudio = false;
897
898                performSeek(positionUs);
899                instantiateDecoder(true /* audio */, &mAudioDecoder);
900            }
901            break;
902        }
903
904        case kWhatMoreDataQueued:
905        {
906            break;
907        }
908
909        case kWhatReset:
910        {
911            ALOGV("kWhatReset");
912
913            mDeferredActions.push_back(
914                    new ShutdownDecoderAction(
915                        true /* audio */, true /* video */));
916
917            mDeferredActions.push_back(
918                    new SimpleAction(&NuPlayer::performReset));
919
920            processDeferredActions();
921            break;
922        }
923
924        case kWhatSeek:
925        {
926            int64_t seekTimeUs;
927            CHECK(msg->findInt64("seekTimeUs", &seekTimeUs));
928
929            ALOGV("kWhatSeek seekTimeUs=%lld us", seekTimeUs);
930
931            mDeferredActions.push_back(
932                    new SimpleAction(&NuPlayer::performDecoderFlush));
933
934            mDeferredActions.push_back(new SeekAction(seekTimeUs));
935
936            processDeferredActions();
937            break;
938        }
939
940        case kWhatPause:
941        {
942            CHECK(mRenderer != NULL);
943            mSource->pause();
944            mRenderer->pause();
945            break;
946        }
947
948        case kWhatResume:
949        {
950            CHECK(mRenderer != NULL);
951            mSource->resume();
952            mRenderer->resume();
953            break;
954        }
955
956        case kWhatSourceNotify:
957        {
958            onSourceNotify(msg);
959            break;
960        }
961
962        case kWhatClosedCaptionNotify:
963        {
964            onClosedCaptionNotify(msg);
965            break;
966        }
967
968        default:
969            TRESPASS();
970            break;
971    }
972}
973
974void NuPlayer::finishFlushIfPossible() {
975    if (mFlushingAudio != NONE && mFlushingAudio != FLUSHED
976            && mFlushingAudio != SHUT_DOWN) {
977        return;
978    }
979
980    if (mFlushingVideo != NONE && mFlushingVideo != FLUSHED
981            && mFlushingVideo != SHUT_DOWN) {
982        return;
983    }
984
985    ALOGV("both audio and video are flushed now.");
986
987    if (mTimeDiscontinuityPending) {
988        mRenderer->signalTimeDiscontinuity();
989        mTimeDiscontinuityPending = false;
990    }
991
992    if (mAudioDecoder != NULL && mFlushingAudio == FLUSHED) {
993        mAudioDecoder->signalResume();
994    }
995
996    if (mVideoDecoder != NULL && mFlushingVideo == FLUSHED) {
997        mVideoDecoder->signalResume();
998    }
999
1000    mFlushingAudio = NONE;
1001    mFlushingVideo = NONE;
1002
1003    processDeferredActions();
1004}
1005
1006void NuPlayer::postScanSources() {
1007    if (mScanSourcesPending) {
1008        return;
1009    }
1010
1011    sp<AMessage> msg = new AMessage(kWhatScanSources, id());
1012    msg->setInt32("generation", mScanSourcesGeneration);
1013    msg->post();
1014
1015    mScanSourcesPending = true;
1016}
1017
1018void NuPlayer::openAudioSink(const sp<AMessage> &format, bool offloadOnly) {
1019    ALOGV("openAudioSink: offloadOnly(%d) mOffloadAudio(%d)",
1020            offloadOnly, mOffloadAudio);
1021    bool audioSinkChanged = false;
1022
1023    int32_t numChannels;
1024    CHECK(format->findInt32("channel-count", &numChannels));
1025
1026    int32_t channelMask;
1027    if (!format->findInt32("channel-mask", &channelMask)) {
1028        // signal to the AudioSink to derive the mask from count.
1029        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1030    }
1031
1032    int32_t sampleRate;
1033    CHECK(format->findInt32("sample-rate", &sampleRate));
1034
1035    uint32_t flags;
1036    int64_t durationUs;
1037    // FIXME: we should handle the case where the video decoder
1038    // is created after we receive the format change indication.
1039    // Current code will just make that we select deep buffer
1040    // with video which should not be a problem as it should
1041    // not prevent from keeping A/V sync.
1042    if (mVideoDecoder == NULL &&
1043            mSource->getDuration(&durationUs) == OK &&
1044            durationUs
1045                > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
1046        flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1047    } else {
1048        flags = AUDIO_OUTPUT_FLAG_NONE;
1049    }
1050
1051    if (mOffloadAudio) {
1052        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1053        AString mime;
1054        CHECK(format->findString("mime", &mime));
1055        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1056
1057        if (err != OK) {
1058            ALOGE("Couldn't map mime \"%s\" to a valid "
1059                    "audio_format", mime.c_str());
1060            mOffloadAudio = false;
1061        } else {
1062            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1063                    mime.c_str(), audioFormat);
1064
1065            int avgBitRate = -1;
1066            format->findInt32("bit-rate", &avgBitRate);
1067
1068            int32_t aacProfile = -1;
1069            if (audioFormat == AUDIO_FORMAT_AAC
1070                    && format->findInt32("aac-profile", &aacProfile)) {
1071                // Redefine AAC format as per aac profile
1072                mapAACProfileToAudioFormat(
1073                        audioFormat,
1074                        aacProfile);
1075            }
1076
1077            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1078            offloadInfo.duration_us = -1;
1079            format->findInt64(
1080                    "durationUs", &offloadInfo.duration_us);
1081            offloadInfo.sample_rate = sampleRate;
1082            offloadInfo.channel_mask = channelMask;
1083            offloadInfo.format = audioFormat;
1084            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1085            offloadInfo.bit_rate = avgBitRate;
1086            offloadInfo.has_video = (mVideoDecoder != NULL);
1087            offloadInfo.is_streaming = true;
1088
1089            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1090                ALOGV("openAudioSink: no change in offload mode");
1091                return;  // no change from previous configuration, everything ok.
1092            }
1093            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1094            flags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1095            flags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1096            audioSinkChanged = true;
1097            mAudioSink->close();
1098            err = mAudioSink->open(
1099                    sampleRate,
1100                    numChannels,
1101                    (audio_channel_mask_t)channelMask,
1102                    audioFormat,
1103                    8 /* bufferCount */,
1104                    &NuPlayer::Renderer::AudioSinkCallback,
1105                    mRenderer.get(),
1106                    (audio_output_flags_t)flags,
1107                    &offloadInfo);
1108
1109            if (err == OK) {
1110                // If the playback is offloaded to h/w, we pass
1111                // the HAL some metadata information.
1112                // We don't want to do this for PCM because it
1113                // will be going through the AudioFlinger mixer
1114                // before reaching the hardware.
1115                sp<MetaData> audioMeta =
1116                        mSource->getFormatMeta(true /* audio */);
1117                sendMetaDataToHal(mAudioSink, audioMeta);
1118                mCurrentOffloadInfo = offloadInfo;
1119                err = mAudioSink->start();
1120                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1121            }
1122            if (err != OK) {
1123                // Clean up, fall back to non offload mode.
1124                mAudioSink->close();
1125                mRenderer->signalDisableOffloadAudio();
1126                mOffloadAudio = false;
1127                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1128                ALOGV("openAudioSink: offload failed");
1129            }
1130        }
1131    }
1132    if (!offloadOnly && !mOffloadAudio) {
1133        flags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1134        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1135
1136        audioSinkChanged = true;
1137        mAudioSink->close();
1138        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1139        CHECK_EQ(mAudioSink->open(
1140                    sampleRate,
1141                    numChannels,
1142                    (audio_channel_mask_t)channelMask,
1143                    AUDIO_FORMAT_PCM_16_BIT,
1144                    8 /* bufferCount */,
1145                    NULL,
1146                    NULL,
1147                    (audio_output_flags_t)flags),
1148                 (status_t)OK);
1149        mAudioSink->start();
1150    }
1151    if (audioSinkChanged) {
1152        mRenderer->signalAudioSinkChanged();
1153    }
1154}
1155
1156void NuPlayer::closeAudioSink() {
1157    mAudioSink->close();
1158    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1159}
1160
1161status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
1162    if (*decoder != NULL) {
1163        return OK;
1164    }
1165
1166    sp<AMessage> format = mSource->getFormat(audio);
1167
1168    if (format == NULL) {
1169        return -EWOULDBLOCK;
1170    }
1171
1172    if (!audio) {
1173        AString mime;
1174        CHECK(format->findString("mime", &mime));
1175        mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
1176
1177        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
1178        mCCDecoder = new CCDecoder(ccNotify);
1179
1180        if (mSourceFlags & Source::FLAG_SECURE) {
1181            format->setInt32("secure", true);
1182        }
1183    }
1184
1185    if (audio) {
1186        sp<AMessage> notify = new AMessage(kWhatAudioNotify, id());
1187        ++mAudioDecoderGeneration;
1188        notify->setInt32("generation", mAudioDecoderGeneration);
1189
1190        if (mOffloadAudio) {
1191            *decoder = new DecoderPassThrough(notify);
1192        } else {
1193            *decoder = new Decoder(notify);
1194        }
1195    } else {
1196        sp<AMessage> notify = new AMessage(kWhatVideoNotify, id());
1197        ++mVideoDecoderGeneration;
1198        notify->setInt32("generation", mVideoDecoderGeneration);
1199
1200        *decoder = new Decoder(notify, mNativeWindow);
1201    }
1202    (*decoder)->init();
1203    (*decoder)->configure(format);
1204
1205    // allocate buffers to decrypt widevine source buffers
1206    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
1207        Vector<sp<ABuffer> > inputBufs;
1208        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);
1209
1210        Vector<MediaBuffer *> mediaBufs;
1211        for (size_t i = 0; i < inputBufs.size(); i++) {
1212            const sp<ABuffer> &buffer = inputBufs[i];
1213            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
1214            mediaBufs.push(mbuf);
1215        }
1216
1217        status_t err = mSource->setBuffers(audio, mediaBufs);
1218        if (err != OK) {
1219            for (size_t i = 0; i < mediaBufs.size(); ++i) {
1220                mediaBufs[i]->release();
1221            }
1222            mediaBufs.clear();
1223            ALOGE("Secure source didn't support secure mediaBufs.");
1224            return err;
1225        }
1226    }
1227    return OK;
1228}
1229
1230status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
1231    sp<AMessage> reply;
1232    CHECK(msg->findMessage("reply", &reply));
1233
1234    if ((audio && mFlushingAudio != NONE)
1235            || (!audio && mFlushingVideo != NONE)) {
1236        reply->setInt32("err", INFO_DISCONTINUITY);
1237        reply->post();
1238        return OK;
1239    }
1240
1241    sp<ABuffer> accessUnit;
1242
1243    bool dropAccessUnit;
1244    do {
1245        status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
1246
1247        if (err == -EWOULDBLOCK) {
1248            return err;
1249        } else if (err != OK) {
1250            if (err == INFO_DISCONTINUITY) {
1251                int32_t type;
1252                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
1253
1254                bool formatChange =
1255                    (audio &&
1256                     (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
1257                    || (!audio &&
1258                            (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
1259
1260                bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
1261
1262                ALOGI("%s discontinuity (formatChange=%d, time=%d)",
1263                     audio ? "audio" : "video", formatChange, timeChange);
1264
1265                if (audio) {
1266                    mSkipRenderingAudioUntilMediaTimeUs = -1;
1267                } else {
1268                    mSkipRenderingVideoUntilMediaTimeUs = -1;
1269                }
1270
1271                if (timeChange) {
1272                    sp<AMessage> extra;
1273                    if (accessUnit->meta()->findMessage("extra", &extra)
1274                            && extra != NULL) {
1275                        int64_t resumeAtMediaTimeUs;
1276                        if (extra->findInt64(
1277                                    "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
1278                            ALOGI("suppressing rendering of %s until %lld us",
1279                                    audio ? "audio" : "video", resumeAtMediaTimeUs);
1280
1281                            if (audio) {
1282                                mSkipRenderingAudioUntilMediaTimeUs =
1283                                    resumeAtMediaTimeUs;
1284                            } else {
1285                                mSkipRenderingVideoUntilMediaTimeUs =
1286                                    resumeAtMediaTimeUs;
1287                            }
1288                        }
1289                    }
1290                }
1291
1292                mTimeDiscontinuityPending =
1293                    mTimeDiscontinuityPending || timeChange;
1294
1295                bool seamlessFormatChange = false;
1296                sp<AMessage> newFormat = mSource->getFormat(audio);
1297                if (formatChange) {
1298                    seamlessFormatChange =
1299                        getDecoder(audio)->supportsSeamlessFormatChange(newFormat);
1300                    // treat seamless format change separately
1301                    formatChange = !seamlessFormatChange;
1302                }
1303                bool shutdownOrFlush = formatChange || timeChange;
1304
1305                // We want to queue up scan-sources only once per discontinuity.
1306                // We control this by doing it only if neither audio nor video are
1307                // flushing or shutting down.  (After handling 1st discontinuity, one
1308                // of the flushing states will not be NONE.)
1309                // No need to scan sources if this discontinuity does not result
1310                // in a flush or shutdown, as the flushing state will stay NONE.
1311                if (mFlushingAudio == NONE && mFlushingVideo == NONE &&
1312                        shutdownOrFlush) {
1313                    // And we'll resume scanning sources once we're done
1314                    // flushing.
1315                    mDeferredActions.push_front(
1316                            new SimpleAction(
1317                                &NuPlayer::performScanSources));
1318                }
1319
1320                if (formatChange /* not seamless */) {
1321                    // must change decoder
1322                    flushDecoder(audio, /* needShutdown = */ true);
1323                } else if (timeChange) {
1324                    // need to flush
1325                    flushDecoder(audio, /* needShutdown = */ false, newFormat);
1326                    err = OK;
1327                } else if (seamlessFormatChange) {
1328                    // reuse existing decoder and don't flush
1329                    updateDecoderFormatWithoutFlush(audio, newFormat);
1330                    err = OK;
1331                } else {
1332                    // This stream is unaffected by the discontinuity
1333                    return -EWOULDBLOCK;
1334                }
1335            }
1336
1337            reply->setInt32("err", err);
1338            reply->post();
1339            return OK;
1340        }
1341
1342        if (!audio) {
1343            ++mNumFramesTotal;
1344        }
1345
1346        dropAccessUnit = false;
1347        if (!audio
1348                && !(mSourceFlags & Source::FLAG_SECURE)
1349                && mVideoLateByUs > 100000ll
1350                && mVideoIsAVC
1351                && !IsAVCReferenceFrame(accessUnit)) {
1352            dropAccessUnit = true;
1353            ++mNumFramesDropped;
1354        }
1355    } while (dropAccessUnit);
1356
1357    // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
1358
1359#if 0
1360    int64_t mediaTimeUs;
1361    CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
1362    ALOGV("feeding %s input buffer at media time %.2f secs",
1363         audio ? "audio" : "video",
1364         mediaTimeUs / 1E6);
1365#endif
1366
1367    if (!audio) {
1368        mCCDecoder->decode(accessUnit);
1369    }
1370
1371    reply->setBuffer("buffer", accessUnit);
1372    reply->post();
1373
1374    return OK;
1375}
1376
1377void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
1378    // ALOGV("renderBuffer %s", audio ? "audio" : "video");
1379
1380    sp<AMessage> reply;
1381    CHECK(msg->findMessage("reply", &reply));
1382
1383    if ((audio && mFlushingAudio != NONE)
1384            || (!audio && mFlushingVideo != NONE)) {
1385        // We're currently attempting to flush the decoder, in order
1386        // to complete this, the decoder wants all its buffers back,
1387        // so we don't want any output buffers it sent us (from before
1388        // we initiated the flush) to be stuck in the renderer's queue.
1389
1390        ALOGV("we're still flushing the %s decoder, sending its output buffer"
1391             " right back.", audio ? "audio" : "video");
1392
1393        reply->post();
1394        return;
1395    }
1396
1397    sp<ABuffer> buffer;
1398    CHECK(msg->findBuffer("buffer", &buffer));
1399
1400    int64_t mediaTimeUs;
1401    CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));
1402
1403    int64_t &skipUntilMediaTimeUs =
1404        audio
1405            ? mSkipRenderingAudioUntilMediaTimeUs
1406            : mSkipRenderingVideoUntilMediaTimeUs;
1407
1408    if (skipUntilMediaTimeUs >= 0) {
1409
1410        if (mediaTimeUs < skipUntilMediaTimeUs) {
1411            ALOGV("dropping %s buffer at time %lld as requested.",
1412                 audio ? "audio" : "video",
1413                 mediaTimeUs);
1414
1415            reply->post();
1416            return;
1417        }
1418
1419        skipUntilMediaTimeUs = -1;
1420    }
1421
1422    if (!audio && mCCDecoder->isSelected()) {
1423        mCCDecoder->display(mediaTimeUs);
1424    }
1425
1426    mRenderer->queueBuffer(audio, buffer, reply);
1427}
1428
1429void NuPlayer::updateVideoSize(
1430        const sp<AMessage> &inputFormat,
1431        const sp<AMessage> &outputFormat) {
1432    if (inputFormat == NULL) {
1433        ALOGW("Unknown video size, reporting 0x0!");
1434        notifyListener(MEDIA_SET_VIDEO_SIZE, 0, 0);
1435        return;
1436    }
1437
1438    int32_t displayWidth, displayHeight;
1439    int32_t cropLeft, cropTop, cropRight, cropBottom;
1440
1441    if (outputFormat != NULL) {
1442        int32_t width, height;
1443        CHECK(outputFormat->findInt32("width", &width));
1444        CHECK(outputFormat->findInt32("height", &height));
1445
1446        int32_t cropLeft, cropTop, cropRight, cropBottom;
1447        CHECK(outputFormat->findRect(
1448                    "crop",
1449                    &cropLeft, &cropTop, &cropRight, &cropBottom));
1450
1451        displayWidth = cropRight - cropLeft + 1;
1452        displayHeight = cropBottom - cropTop + 1;
1453
1454        ALOGV("Video output format changed to %d x %d "
1455             "(crop: %d x %d @ (%d, %d))",
1456             width, height,
1457             displayWidth,
1458             displayHeight,
1459             cropLeft, cropTop);
1460    } else {
1461        CHECK(inputFormat->findInt32("width", &displayWidth));
1462        CHECK(inputFormat->findInt32("height", &displayHeight));
1463
1464        ALOGV("Video input format %d x %d", displayWidth, displayHeight);
1465    }
1466
1467    // Take into account sample aspect ratio if necessary:
1468    int32_t sarWidth, sarHeight;
1469    if (inputFormat->findInt32("sar-width", &sarWidth)
1470            && inputFormat->findInt32("sar-height", &sarHeight)) {
1471        ALOGV("Sample aspect ratio %d : %d", sarWidth, sarHeight);
1472
1473        displayWidth = (displayWidth * sarWidth) / sarHeight;
1474
1475        ALOGV("display dimensions %d x %d", displayWidth, displayHeight);
1476    }
1477
1478    int32_t rotationDegrees;
1479    if (!inputFormat->findInt32("rotation-degrees", &rotationDegrees)) {
1480        rotationDegrees = 0;
1481    }
1482
1483    if (rotationDegrees == 90 || rotationDegrees == 270) {
1484        int32_t tmp = displayWidth;
1485        displayWidth = displayHeight;
1486        displayHeight = tmp;
1487    }
1488
1489    notifyListener(
1490            MEDIA_SET_VIDEO_SIZE,
1491            displayWidth,
1492            displayHeight);
1493}
1494
1495void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) {
1496    if (mDriver == NULL) {
1497        return;
1498    }
1499
1500    sp<NuPlayerDriver> driver = mDriver.promote();
1501
1502    if (driver == NULL) {
1503        return;
1504    }
1505
1506    driver->notifyListener(msg, ext1, ext2, in);
1507}
1508
1509void NuPlayer::flushDecoder(
1510        bool audio, bool needShutdown, const sp<AMessage> &newFormat) {
1511    ALOGV("[%s] flushDecoder needShutdown=%d",
1512          audio ? "audio" : "video", needShutdown);
1513
1514    const sp<Decoder> &decoder = getDecoder(audio);
1515    if (decoder == NULL) {
1516        ALOGI("flushDecoder %s without decoder present",
1517             audio ? "audio" : "video");
1518        return;
1519    }
1520
1521    // Make sure we don't continue to scan sources until we finish flushing.
1522    ++mScanSourcesGeneration;
1523    mScanSourcesPending = false;
1524
1525    decoder->signalFlush(newFormat);
1526    mRenderer->flush(audio);
1527
1528    FlushStatus newStatus =
1529        needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;
1530
1531    if (audio) {
1532        ALOGE_IF(mFlushingAudio != NONE,
1533                "audio flushDecoder() is called in state %d", mFlushingAudio);
1534        mFlushingAudio = newStatus;
1535    } else {
1536        ALOGE_IF(mFlushingVideo != NONE,
1537                "video flushDecoder() is called in state %d", mFlushingVideo);
1538        mFlushingVideo = newStatus;
1539    }
1540}
1541
1542void NuPlayer::updateDecoderFormatWithoutFlush(
1543        bool audio, const sp<AMessage> &format) {
1544    ALOGV("[%s] updateDecoderFormatWithoutFlush", audio ? "audio" : "video");
1545
1546    const sp<Decoder> &decoder = getDecoder(audio);
1547    if (decoder == NULL) {
1548        ALOGI("updateDecoderFormatWithoutFlush %s without decoder present",
1549             audio ? "audio" : "video");
1550        return;
1551    }
1552
1553    decoder->signalUpdateFormat(format);
1554}
1555
1556void NuPlayer::queueDecoderShutdown(
1557        bool audio, bool video, const sp<AMessage> &reply) {
1558    ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video);
1559
1560    mDeferredActions.push_back(
1561            new ShutdownDecoderAction(audio, video));
1562
1563    mDeferredActions.push_back(
1564            new SimpleAction(&NuPlayer::performScanSources));
1565
1566    mDeferredActions.push_back(new PostMessageAction(reply));
1567
1568    processDeferredActions();
1569}
1570
1571status_t NuPlayer::setVideoScalingMode(int32_t mode) {
1572    mVideoScalingMode = mode;
1573    if (mNativeWindow != NULL) {
1574        status_t ret = native_window_set_scaling_mode(
1575                mNativeWindow->getNativeWindow().get(), mVideoScalingMode);
1576        if (ret != OK) {
1577            ALOGE("Failed to set scaling mode (%d): %s",
1578                -ret, strerror(-ret));
1579            return ret;
1580        }
1581    }
1582    return OK;
1583}
1584
1585status_t NuPlayer::getTrackInfo(Parcel* reply) const {
1586    sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id());
1587    msg->setPointer("reply", reply);
1588
1589    sp<AMessage> response;
1590    status_t err = msg->postAndAwaitResponse(&response);
1591    return err;
1592}
1593
1594status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const {
1595    sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id());
1596    msg->setPointer("reply", reply);
1597    msg->setInt32("type", type);
1598
1599    sp<AMessage> response;
1600    status_t err = msg->postAndAwaitResponse(&response);
1601    if (err == OK && response != NULL) {
1602        CHECK(response->findInt32("err", &err));
1603    }
1604    return err;
1605}
1606
1607status_t NuPlayer::selectTrack(size_t trackIndex, bool select) {
1608    sp<AMessage> msg = new AMessage(kWhatSelectTrack, id());
1609    msg->setSize("trackIndex", trackIndex);
1610    msg->setInt32("select", select);
1611
1612    sp<AMessage> response;
1613    status_t err = msg->postAndAwaitResponse(&response);
1614
1615    if (err != OK) {
1616        return err;
1617    }
1618
1619    if (!response->findInt32("err", &err)) {
1620        err = OK;
1621    }
1622
1623    return err;
1624}
1625
1626void NuPlayer::schedulePollDuration() {
1627    sp<AMessage> msg = new AMessage(kWhatPollDuration, id());
1628    msg->setInt32("generation", mPollDurationGeneration);
1629    msg->post();
1630}
1631
1632void NuPlayer::cancelPollDuration() {
1633    ++mPollDurationGeneration;
1634}
1635
1636void NuPlayer::processDeferredActions() {
1637    while (!mDeferredActions.empty()) {
1638        // We won't execute any deferred actions until we're no longer in
1639        // an intermediate state, i.e. one more more decoders are currently
1640        // flushing or shutting down.
1641
1642        if (mFlushingAudio != NONE || mFlushingVideo != NONE) {
1643            // We're currently flushing, postpone the reset until that's
1644            // completed.
1645
1646            ALOGV("postponing action mFlushingAudio=%d, mFlushingVideo=%d",
1647                  mFlushingAudio, mFlushingVideo);
1648
1649            break;
1650        }
1651
1652        sp<Action> action = *mDeferredActions.begin();
1653        mDeferredActions.erase(mDeferredActions.begin());
1654
1655        action->execute(this);
1656    }
1657}
1658
1659void NuPlayer::performSeek(int64_t seekTimeUs) {
1660    ALOGV("performSeek seekTimeUs=%lld us (%.2f secs)",
1661          seekTimeUs,
1662          seekTimeUs / 1E6);
1663
1664    mSource->seekTo(seekTimeUs);
1665    ++mTimedTextGeneration;
1666
1667    if (mDriver != NULL) {
1668        sp<NuPlayerDriver> driver = mDriver.promote();
1669        if (driver != NULL) {
1670            driver->notifyPosition(seekTimeUs);
1671            driver->notifySeekComplete();
1672        }
1673    }
1674
1675    // everything's flushed, continue playback.
1676}
1677
1678void NuPlayer::performDecoderFlush() {
1679    ALOGV("performDecoderFlush");
1680
1681    if (mAudioDecoder == NULL && mVideoDecoder == NULL) {
1682        return;
1683    }
1684
1685    mTimeDiscontinuityPending = true;
1686
1687    if (mAudioDecoder != NULL) {
1688        flushDecoder(true /* audio */, false /* needShutdown */);
1689    }
1690
1691    if (mVideoDecoder != NULL) {
1692        flushDecoder(false /* audio */, false /* needShutdown */);
1693    }
1694}
1695
1696void NuPlayer::performDecoderShutdown(bool audio, bool video) {
1697    ALOGV("performDecoderShutdown audio=%d, video=%d", audio, video);
1698
1699    if ((!audio || mAudioDecoder == NULL)
1700            && (!video || mVideoDecoder == NULL)) {
1701        return;
1702    }
1703
1704    mTimeDiscontinuityPending = true;
1705
1706    if (audio && mAudioDecoder != NULL) {
1707        flushDecoder(true /* audio */, true /* needShutdown */);
1708    }
1709
1710    if (video && mVideoDecoder != NULL) {
1711        flushDecoder(false /* audio */, true /* needShutdown */);
1712    }
1713}
1714
1715void NuPlayer::performReset() {
1716    ALOGV("performReset");
1717
1718    CHECK(mAudioDecoder == NULL);
1719    CHECK(mVideoDecoder == NULL);
1720
1721    cancelPollDuration();
1722
1723    ++mScanSourcesGeneration;
1724    mScanSourcesPending = false;
1725
1726    if (mRendererLooper != NULL) {
1727        if (mRenderer != NULL) {
1728            mRendererLooper->unregisterHandler(mRenderer->id());
1729        }
1730        mRendererLooper->stop();
1731        mRendererLooper.clear();
1732    }
1733    mRenderer.clear();
1734
1735    if (mSource != NULL) {
1736        mSource->stop();
1737
1738        mSource.clear();
1739    }
1740
1741    if (mDriver != NULL) {
1742        sp<NuPlayerDriver> driver = mDriver.promote();
1743        if (driver != NULL) {
1744            driver->notifyResetComplete();
1745        }
1746    }
1747
1748    mStarted = false;
1749}
1750
1751void NuPlayer::performScanSources() {
1752    ALOGV("performScanSources");
1753
1754    if (!mStarted) {
1755        return;
1756    }
1757
1758    if (mAudioDecoder == NULL || mVideoDecoder == NULL) {
1759        postScanSources();
1760    }
1761}
1762
1763void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) {
1764    ALOGV("performSetSurface");
1765
1766    mNativeWindow = wrapper;
1767
1768    // XXX - ignore error from setVideoScalingMode for now
1769    setVideoScalingMode(mVideoScalingMode);
1770
1771    if (mDriver != NULL) {
1772        sp<NuPlayerDriver> driver = mDriver.promote();
1773        if (driver != NULL) {
1774            driver->notifySetSurfaceComplete();
1775        }
1776    }
1777}
1778
1779void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
1780    int32_t what;
1781    CHECK(msg->findInt32("what", &what));
1782
1783    switch (what) {
1784        case Source::kWhatPrepared:
1785        {
1786            if (mSource == NULL) {
1787                // This is a stale notification from a source that was
1788                // asynchronously preparing when the client called reset().
1789                // We handled the reset, the source is gone.
1790                break;
1791            }
1792
1793            int32_t err;
1794            CHECK(msg->findInt32("err", &err));
1795
1796            sp<NuPlayerDriver> driver = mDriver.promote();
1797            if (driver != NULL) {
1798                // notify duration first, so that it's definitely set when
1799                // the app received the "prepare complete" callback.
1800                int64_t durationUs;
1801                if (mSource->getDuration(&durationUs) == OK) {
1802                    driver->notifyDuration(durationUs);
1803                }
1804                driver->notifyPrepareCompleted(err);
1805            }
1806
1807            break;
1808        }
1809
1810        case Source::kWhatFlagsChanged:
1811        {
1812            uint32_t flags;
1813            CHECK(msg->findInt32("flags", (int32_t *)&flags));
1814
1815            sp<NuPlayerDriver> driver = mDriver.promote();
1816            if (driver != NULL) {
1817                driver->notifyFlagsChanged(flags);
1818            }
1819
1820            if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1821                    && (!(flags & Source::FLAG_DYNAMIC_DURATION))) {
1822                cancelPollDuration();
1823            } else if (!(mSourceFlags & Source::FLAG_DYNAMIC_DURATION)
1824                    && (flags & Source::FLAG_DYNAMIC_DURATION)
1825                    && (mAudioDecoder != NULL || mVideoDecoder != NULL)) {
1826                schedulePollDuration();
1827            }
1828
1829            mSourceFlags = flags;
1830            break;
1831        }
1832
1833        case Source::kWhatVideoSizeChanged:
1834        {
1835            sp<AMessage> format;
1836            CHECK(msg->findMessage("format", &format));
1837
1838            updateVideoSize(format);
1839            break;
1840        }
1841
1842        case Source::kWhatBufferingUpdate:
1843        {
1844            int32_t percentage;
1845            CHECK(msg->findInt32("percentage", &percentage));
1846
1847            notifyListener(MEDIA_BUFFERING_UPDATE, percentage, 0);
1848            break;
1849        }
1850
1851        case Source::kWhatBufferingStart:
1852        {
1853            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_START, 0);
1854            break;
1855        }
1856
1857        case Source::kWhatBufferingEnd:
1858        {
1859            notifyListener(MEDIA_INFO, MEDIA_INFO_BUFFERING_END, 0);
1860            break;
1861        }
1862
1863        case Source::kWhatSubtitleData:
1864        {
1865            sp<ABuffer> buffer;
1866            CHECK(msg->findBuffer("buffer", &buffer));
1867
1868            sendSubtitleData(buffer, 0 /* baseIndex */);
1869            break;
1870        }
1871
1872        case Source::kWhatTimedTextData:
1873        {
1874            int32_t generation;
1875            if (msg->findInt32("generation", &generation)
1876                    && generation != mTimedTextGeneration) {
1877                break;
1878            }
1879
1880            sp<ABuffer> buffer;
1881            CHECK(msg->findBuffer("buffer", &buffer));
1882
1883            sp<NuPlayerDriver> driver = mDriver.promote();
1884            if (driver == NULL) {
1885                break;
1886            }
1887
1888            int posMs;
1889            int64_t timeUs, posUs;
1890            driver->getCurrentPosition(&posMs);
1891            posUs = posMs * 1000;
1892            CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1893
1894            if (posUs < timeUs) {
1895                if (!msg->findInt32("generation", &generation)) {
1896                    msg->setInt32("generation", mTimedTextGeneration);
1897                }
1898                msg->post(timeUs - posUs);
1899            } else {
1900                sendTimedTextData(buffer);
1901            }
1902            break;
1903        }
1904
1905        case Source::kWhatQueueDecoderShutdown:
1906        {
1907            int32_t audio, video;
1908            CHECK(msg->findInt32("audio", &audio));
1909            CHECK(msg->findInt32("video", &video));
1910
1911            sp<AMessage> reply;
1912            CHECK(msg->findMessage("reply", &reply));
1913
1914            queueDecoderShutdown(audio, video, reply);
1915            break;
1916        }
1917
1918        default:
1919            TRESPASS();
1920    }
1921}
1922
1923void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
1924    int32_t what;
1925    CHECK(msg->findInt32("what", &what));
1926
1927    switch (what) {
1928        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
1929        {
1930            sp<ABuffer> buffer;
1931            CHECK(msg->findBuffer("buffer", &buffer));
1932
1933            size_t inbandTracks = 0;
1934            if (mSource != NULL) {
1935                inbandTracks = mSource->getTrackCount();
1936            }
1937
1938            sendSubtitleData(buffer, inbandTracks);
1939            break;
1940        }
1941
1942        case NuPlayer::CCDecoder::kWhatTrackAdded:
1943        {
1944            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
1945
1946            break;
1947        }
1948
1949        default:
1950            TRESPASS();
1951    }
1952
1953
1954}
1955
1956void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
1957    int32_t trackIndex;
1958    int64_t timeUs, durationUs;
1959    CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex));
1960    CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1961    CHECK(buffer->meta()->findInt64("durationUs", &durationUs));
1962
1963    Parcel in;
1964    in.writeInt32(trackIndex + baseIndex);
1965    in.writeInt64(timeUs);
1966    in.writeInt64(durationUs);
1967    in.writeInt32(buffer->size());
1968    in.writeInt32(buffer->size());
1969    in.write(buffer->data(), buffer->size());
1970
1971    notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in);
1972}
1973
1974void NuPlayer::sendTimedTextData(const sp<ABuffer> &buffer) {
1975    const void *data;
1976    size_t size = 0;
1977    int64_t timeUs;
1978    int32_t flag = TextDescriptions::LOCAL_DESCRIPTIONS;
1979
1980    AString mime;
1981    CHECK(buffer->meta()->findString("mime", &mime));
1982    CHECK(strcasecmp(mime.c_str(), MEDIA_MIMETYPE_TEXT_3GPP) == 0);
1983
1984    data = buffer->data();
1985    size = buffer->size();
1986
1987    Parcel parcel;
1988    if (size > 0) {
1989        CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1990        flag |= TextDescriptions::IN_BAND_TEXT_3GPP;
1991        TextDescriptions::getParcelOfDescriptions(
1992                (const uint8_t *)data, size, flag, timeUs / 1000, &parcel);
1993    }
1994
1995    if ((parcel.dataSize() > 0)) {
1996        notifyListener(MEDIA_TIMED_TEXT, 0, 0, &parcel);
1997    } else {  // send an empty timed text
1998        notifyListener(MEDIA_TIMED_TEXT, 0, 0);
1999    }
2000}
2001////////////////////////////////////////////////////////////////////////////////
2002
2003sp<AMessage> NuPlayer::Source::getFormat(bool audio) {
2004    sp<MetaData> meta = getFormatMeta(audio);
2005
2006    if (meta == NULL) {
2007        return NULL;
2008    }
2009
2010    sp<AMessage> msg = new AMessage;
2011
2012    if(convertMetaDataToMessage(meta, &msg) == OK) {
2013        return msg;
2014    }
2015    return NULL;
2016}
2017
2018void NuPlayer::Source::notifyFlagsChanged(uint32_t flags) {
2019    sp<AMessage> notify = dupNotify();
2020    notify->setInt32("what", kWhatFlagsChanged);
2021    notify->setInt32("flags", flags);
2022    notify->post();
2023}
2024
2025void NuPlayer::Source::notifyVideoSizeChanged(const sp<AMessage> &format) {
2026    sp<AMessage> notify = dupNotify();
2027    notify->setInt32("what", kWhatVideoSizeChanged);
2028    notify->setMessage("format", format);
2029    notify->post();
2030}
2031
2032void NuPlayer::Source::notifyPrepared(status_t err) {
2033    sp<AMessage> notify = dupNotify();
2034    notify->setInt32("what", kWhatPrepared);
2035    notify->setInt32("err", err);
2036    notify->post();
2037}
2038
2039void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) {
2040    TRESPASS();
2041}
2042
2043}  // namespace android
2044