NuPlayerRenderer.cpp revision 12b6265f1d4be368957f91104d5210cf604ac4cc 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22#include <cutils/properties.h>
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26#include <media/stagefright/foundation/AUtils.h>
27#include <media/stagefright/foundation/AWakeLock.h>
28#include <media/stagefright/MediaClock.h>
29#include <media/stagefright/MediaErrors.h>
30#include <media/stagefright/MetaData.h>
31#include <media/stagefright/Utils.h>
32#include <media/stagefright/VideoFrameScheduler.h>
33
34#include <inttypes.h>
35
36namespace android {
37
38/*
39 * Example of common configuration settings in shell script form
40
41   #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager
42   adb shell setprop audio.offload.disable 1
43
44   #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager
45   adb shell setprop audio.offload.video 1
46
47   #Use audio callbacks for PCM data
48   adb shell setprop media.stagefright.audio.cbk 1
49
50   #Use deep buffer for PCM data with video (it is generally enabled for audio-only)
51   adb shell setprop media.stagefright.audio.deep 1
52
53   #Set size of buffers for pcm audio sink in msec (example: 1000 msec)
54   adb shell setprop media.stagefright.audio.sink 1000
55
56 * These configurations take effect for the next track played (not the current track).
57 */
58
59static inline bool getUseAudioCallbackSetting() {
60    return property_get_bool("media.stagefright.audio.cbk", false /* default_value */);
61}
62
63static inline int32_t getAudioSinkPcmMsSetting() {
64    return property_get_int32(
65            "media.stagefright.audio.sink", 500 /* default_value */);
66}
67
68// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
69// is closed to allow the audio DSP to power down.
70static const int64_t kOffloadPauseMaxUs = 10000000ll;
71
72// Maximum allowed delay from AudioSink, 1.5 seconds.
73static const int64_t kMaxAllowedAudioSinkDelayUs = 1500000ll;
74
75// static
76const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
77        AUDIO_CHANNEL_NONE,
78        AUDIO_OUTPUT_FLAG_NONE,
79        AUDIO_FORMAT_INVALID,
80        0, // mNumChannels
81        0 // mSampleRate
82};
83
84// static
85const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
86
87NuPlayer::Renderer::Renderer(
88        const sp<MediaPlayerBase::AudioSink> &sink,
89        const sp<AMessage> &notify,
90        uint32_t flags)
91    : mAudioSink(sink),
92      mNotify(notify),
93      mFlags(flags),
94      mNumFramesWritten(0),
95      mDrainAudioQueuePending(false),
96      mDrainVideoQueuePending(false),
97      mAudioQueueGeneration(0),
98      mVideoQueueGeneration(0),
99      mAudioDrainGeneration(0),
100      mVideoDrainGeneration(0),
101      mAudioEOSGeneration(0),
102      mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
103      mAudioFirstAnchorTimeMediaUs(-1),
104      mAnchorTimeMediaUs(-1),
105      mAnchorNumFramesWritten(-1),
106      mVideoLateByUs(0ll),
107      mHasAudio(false),
108      mHasVideo(false),
109      mNotifyCompleteAudio(false),
110      mNotifyCompleteVideo(false),
111      mSyncQueues(false),
112      mPaused(false),
113      mPauseDrainAudioAllowedUs(0),
114      mVideoSampleReceived(false),
115      mVideoRenderingStarted(false),
116      mVideoRenderingStartGeneration(0),
117      mAudioRenderingStartGeneration(0),
118      mRenderingDataDelivered(false),
119      mLastAudioMediaTimeUs(-1),
120      mAudioOffloadPauseTimeoutGeneration(0),
121      mAudioTornDown(false),
122      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
123      mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
124      mTotalBuffersQueued(0),
125      mLastAudioBufferDrained(0),
126      mUseAudioCallback(false),
127      mWakeLock(new AWakeLock()) {
128    mMediaClock = new MediaClock;
129    mPlaybackRate = mPlaybackSettings.mSpeed;
130    mMediaClock->setPlaybackRate(mPlaybackRate);
131}
132
133NuPlayer::Renderer::~Renderer() {
134    if (offloadingAudio()) {
135        mAudioSink->stop();
136        mAudioSink->flush();
137        mAudioSink->close();
138    }
139}
140
141void NuPlayer::Renderer::queueBuffer(
142        bool audio,
143        const sp<ABuffer> &buffer,
144        const sp<AMessage> &notifyConsumed) {
145    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
146    msg->setInt32("queueGeneration", getQueueGeneration(audio));
147    msg->setInt32("audio", static_cast<int32_t>(audio));
148    msg->setBuffer("buffer", buffer);
149    msg->setMessage("notifyConsumed", notifyConsumed);
150    msg->post();
151}
152
153void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
154    CHECK_NE(finalResult, (status_t)OK);
155
156    sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
157    msg->setInt32("queueGeneration", getQueueGeneration(audio));
158    msg->setInt32("audio", static_cast<int32_t>(audio));
159    msg->setInt32("finalResult", finalResult);
160    msg->post();
161}
162
163status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) {
164    sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
165    writeToAMessage(msg, rate);
166    sp<AMessage> response;
167    status_t err = msg->postAndAwaitResponse(&response);
168    if (err == OK && response != NULL) {
169        CHECK(response->findInt32("err", &err));
170    }
171    return err;
172}
173
174status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) {
175    if (rate.mSpeed == 0.f) {
176        onPause();
177        // don't call audiosink's setPlaybackRate if pausing, as pitch does not
178        // have to correspond to the any non-0 speed (e.g old speed). Keep
179        // settings nonetheless, using the old speed, in case audiosink changes.
180        AudioPlaybackRate newRate = rate;
181        newRate.mSpeed = mPlaybackSettings.mSpeed;
182        mPlaybackSettings = newRate;
183        return OK;
184    }
185
186    if (mAudioSink != NULL && mAudioSink->ready()) {
187        status_t err = mAudioSink->setPlaybackRate(rate);
188        if (err != OK) {
189            return err;
190        }
191    }
192    mPlaybackSettings = rate;
193    mPlaybackRate = rate.mSpeed;
194    mMediaClock->setPlaybackRate(mPlaybackRate);
195    return OK;
196}
197
198status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
199    sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
200    sp<AMessage> response;
201    status_t err = msg->postAndAwaitResponse(&response);
202    if (err == OK && response != NULL) {
203        CHECK(response->findInt32("err", &err));
204        if (err == OK) {
205            readFromAMessage(response, rate);
206        }
207    }
208    return err;
209}
210
211status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
212    if (mAudioSink != NULL && mAudioSink->ready()) {
213        status_t err = mAudioSink->getPlaybackRate(rate);
214        if (err == OK) {
215            if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) {
216                ALOGW("correcting mismatch in internal/external playback rate");
217            }
218            // get playback settings used by audiosink, as it may be
219            // slightly off due to audiosink not taking small changes.
220            mPlaybackSettings = *rate;
221            if (mPaused) {
222                rate->mSpeed = 0.f;
223            }
224        }
225        return err;
226    }
227    *rate = mPlaybackSettings;
228    return OK;
229}
230
231status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
232    sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
233    writeToAMessage(msg, sync, videoFpsHint);
234    sp<AMessage> response;
235    status_t err = msg->postAndAwaitResponse(&response);
236    if (err == OK && response != NULL) {
237        CHECK(response->findInt32("err", &err));
238    }
239    return err;
240}
241
242status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) {
243    if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
244        return BAD_VALUE;
245    }
246    // TODO: support sync sources
247    return INVALID_OPERATION;
248}
249
250status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) {
251    sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
252    sp<AMessage> response;
253    status_t err = msg->postAndAwaitResponse(&response);
254    if (err == OK && response != NULL) {
255        CHECK(response->findInt32("err", &err));
256        if (err == OK) {
257            readFromAMessage(response, sync, videoFps);
258        }
259    }
260    return err;
261}
262
263status_t NuPlayer::Renderer::onGetSyncSettings(
264        AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
265    *sync = mSyncSettings;
266    *videoFps = -1.f;
267    return OK;
268}
269
270void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
271    {
272        Mutex::Autolock autoLock(mLock);
273        if (audio) {
274            mNotifyCompleteAudio |= notifyComplete;
275            clearAudioFirstAnchorTime_l();
276            ++mAudioQueueGeneration;
277            ++mAudioDrainGeneration;
278        } else {
279            mNotifyCompleteVideo |= notifyComplete;
280            ++mVideoQueueGeneration;
281            ++mVideoDrainGeneration;
282        }
283
284        clearAnchorTime_l();
285        mVideoLateByUs = 0;
286        mSyncQueues = false;
287    }
288
289    sp<AMessage> msg = new AMessage(kWhatFlush, this);
290    msg->setInt32("audio", static_cast<int32_t>(audio));
291    msg->post();
292}
293
294void NuPlayer::Renderer::signalTimeDiscontinuity() {
295}
296
297void NuPlayer::Renderer::signalDisableOffloadAudio() {
298    (new AMessage(kWhatDisableOffloadAudio, this))->post();
299}
300
301void NuPlayer::Renderer::signalEnableOffloadAudio() {
302    (new AMessage(kWhatEnableOffloadAudio, this))->post();
303}
304
305void NuPlayer::Renderer::pause() {
306    (new AMessage(kWhatPause, this))->post();
307}
308
309void NuPlayer::Renderer::resume() {
310    (new AMessage(kWhatResume, this))->post();
311}
312
313void NuPlayer::Renderer::setVideoFrameRate(float fps) {
314    sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
315    msg->setFloat("frame-rate", fps);
316    msg->post();
317}
318
319// Called on any threads without mLock acquired.
320status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
321    status_t result = mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
322    if (result == OK) {
323        return result;
324    }
325
326    // MediaClock has not started yet. Try to start it if possible.
327    {
328        Mutex::Autolock autoLock(mLock);
329        if (mAudioFirstAnchorTimeMediaUs == -1) {
330            return result;
331        }
332
333        AudioTimestamp ts;
334        status_t res = mAudioSink->getTimestamp(ts);
335        if (res != OK) {
336            return result;
337        }
338
339        // AudioSink has rendered some frames.
340        int64_t nowUs = ALooper::GetNowUs();
341        int64_t nowMediaUs = mAudioSink->getPlayedOutDurationUs(nowUs)
342                + mAudioFirstAnchorTimeMediaUs;
343        mMediaClock->updateAnchor(nowMediaUs, nowUs, -1);
344    }
345
346    return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
347}
348
349void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
350    mAudioFirstAnchorTimeMediaUs = -1;
351    mMediaClock->setStartingTimeMedia(-1);
352}
353
354void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
355    if (mAudioFirstAnchorTimeMediaUs == -1) {
356        mAudioFirstAnchorTimeMediaUs = mediaUs;
357        mMediaClock->setStartingTimeMedia(mediaUs);
358    }
359}
360
361void NuPlayer::Renderer::clearAnchorTime_l() {
362    mMediaClock->clearAnchor();
363    mAnchorTimeMediaUs = -1;
364    mAnchorNumFramesWritten = -1;
365}
366
367void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
368    Mutex::Autolock autoLock(mLock);
369    mVideoLateByUs = lateUs;
370}
371
372int64_t NuPlayer::Renderer::getVideoLateByUs() {
373    Mutex::Autolock autoLock(mLock);
374    return mVideoLateByUs;
375}
376
377status_t NuPlayer::Renderer::openAudioSink(
378        const sp<AMessage> &format,
379        bool offloadOnly,
380        bool hasVideo,
381        uint32_t flags,
382        bool *isOffloaded) {
383    sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
384    msg->setMessage("format", format);
385    msg->setInt32("offload-only", offloadOnly);
386    msg->setInt32("has-video", hasVideo);
387    msg->setInt32("flags", flags);
388
389    sp<AMessage> response;
390    msg->postAndAwaitResponse(&response);
391
392    int32_t err;
393    if (!response->findInt32("err", &err)) {
394        err = INVALID_OPERATION;
395    } else if (err == OK && isOffloaded != NULL) {
396        int32_t offload;
397        CHECK(response->findInt32("offload", &offload));
398        *isOffloaded = (offload != 0);
399    }
400    return err;
401}
402
403void NuPlayer::Renderer::closeAudioSink() {
404    sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
405
406    sp<AMessage> response;
407    msg->postAndAwaitResponse(&response);
408}
409
410void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
411    switch (msg->what()) {
412        case kWhatOpenAudioSink:
413        {
414            sp<AMessage> format;
415            CHECK(msg->findMessage("format", &format));
416
417            int32_t offloadOnly;
418            CHECK(msg->findInt32("offload-only", &offloadOnly));
419
420            int32_t hasVideo;
421            CHECK(msg->findInt32("has-video", &hasVideo));
422
423            uint32_t flags;
424            CHECK(msg->findInt32("flags", (int32_t *)&flags));
425
426            status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
427
428            sp<AMessage> response = new AMessage;
429            response->setInt32("err", err);
430            response->setInt32("offload", offloadingAudio());
431
432            sp<AReplyToken> replyID;
433            CHECK(msg->senderAwaitsResponse(&replyID));
434            response->postReply(replyID);
435
436            break;
437        }
438
439        case kWhatCloseAudioSink:
440        {
441            sp<AReplyToken> replyID;
442            CHECK(msg->senderAwaitsResponse(&replyID));
443
444            onCloseAudioSink();
445
446            sp<AMessage> response = new AMessage;
447            response->postReply(replyID);
448            break;
449        }
450
451        case kWhatStopAudioSink:
452        {
453            mAudioSink->stop();
454            break;
455        }
456
457        case kWhatDrainAudioQueue:
458        {
459            mDrainAudioQueuePending = false;
460
461            int32_t generation;
462            CHECK(msg->findInt32("drainGeneration", &generation));
463            if (generation != getDrainGeneration(true /* audio */)) {
464                break;
465            }
466
467            if (onDrainAudioQueue()) {
468                uint32_t numFramesPlayed;
469                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
470                         (status_t)OK);
471
472                uint32_t numFramesPendingPlayout =
473                    mNumFramesWritten - numFramesPlayed;
474
475                // This is how long the audio sink will have data to
476                // play back.
477                int64_t delayUs =
478                    mAudioSink->msecsPerFrame()
479                        * numFramesPendingPlayout * 1000ll;
480                if (mPlaybackRate > 1.0f) {
481                    delayUs /= mPlaybackRate;
482                }
483
484                // Let's give it more data after about half that time
485                // has elapsed.
486                Mutex::Autolock autoLock(mLock);
487                postDrainAudioQueue_l(delayUs / 2);
488            }
489            break;
490        }
491
492        case kWhatDrainVideoQueue:
493        {
494            int32_t generation;
495            CHECK(msg->findInt32("drainGeneration", &generation));
496            if (generation != getDrainGeneration(false /* audio */)) {
497                break;
498            }
499
500            mDrainVideoQueuePending = false;
501
502            onDrainVideoQueue();
503
504            postDrainVideoQueue();
505            break;
506        }
507
508        case kWhatPostDrainVideoQueue:
509        {
510            int32_t generation;
511            CHECK(msg->findInt32("drainGeneration", &generation));
512            if (generation != getDrainGeneration(false /* audio */)) {
513                break;
514            }
515
516            mDrainVideoQueuePending = false;
517            postDrainVideoQueue();
518            break;
519        }
520
521        case kWhatQueueBuffer:
522        {
523            onQueueBuffer(msg);
524            break;
525        }
526
527        case kWhatQueueEOS:
528        {
529            onQueueEOS(msg);
530            break;
531        }
532
533        case kWhatEOS:
534        {
535            int32_t generation;
536            CHECK(msg->findInt32("audioEOSGeneration", &generation));
537            if (generation != mAudioEOSGeneration) {
538                break;
539            }
540            status_t finalResult;
541            CHECK(msg->findInt32("finalResult", &finalResult));
542            notifyEOS(true /* audio */, finalResult);
543            break;
544        }
545
546        case kWhatConfigPlayback:
547        {
548            sp<AReplyToken> replyID;
549            CHECK(msg->senderAwaitsResponse(&replyID));
550            AudioPlaybackRate rate;
551            readFromAMessage(msg, &rate);
552            status_t err = onConfigPlayback(rate);
553            sp<AMessage> response = new AMessage;
554            response->setInt32("err", err);
555            response->postReply(replyID);
556            break;
557        }
558
559        case kWhatGetPlaybackSettings:
560        {
561            sp<AReplyToken> replyID;
562            CHECK(msg->senderAwaitsResponse(&replyID));
563            AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT;
564            status_t err = onGetPlaybackSettings(&rate);
565            sp<AMessage> response = new AMessage;
566            if (err == OK) {
567                writeToAMessage(response, rate);
568            }
569            response->setInt32("err", err);
570            response->postReply(replyID);
571            break;
572        }
573
574        case kWhatConfigSync:
575        {
576            sp<AReplyToken> replyID;
577            CHECK(msg->senderAwaitsResponse(&replyID));
578            AVSyncSettings sync;
579            float videoFpsHint;
580            readFromAMessage(msg, &sync, &videoFpsHint);
581            status_t err = onConfigSync(sync, videoFpsHint);
582            sp<AMessage> response = new AMessage;
583            response->setInt32("err", err);
584            response->postReply(replyID);
585            break;
586        }
587
588        case kWhatGetSyncSettings:
589        {
590            sp<AReplyToken> replyID;
591            CHECK(msg->senderAwaitsResponse(&replyID));
592
593            ALOGV("kWhatGetSyncSettings");
594            AVSyncSettings sync;
595            float videoFps = -1.f;
596            status_t err = onGetSyncSettings(&sync, &videoFps);
597            sp<AMessage> response = new AMessage;
598            if (err == OK) {
599                writeToAMessage(response, sync, videoFps);
600            }
601            response->setInt32("err", err);
602            response->postReply(replyID);
603            break;
604        }
605
606        case kWhatFlush:
607        {
608            onFlush(msg);
609            break;
610        }
611
612        case kWhatDisableOffloadAudio:
613        {
614            onDisableOffloadAudio();
615            break;
616        }
617
618        case kWhatEnableOffloadAudio:
619        {
620            onEnableOffloadAudio();
621            break;
622        }
623
624        case kWhatPause:
625        {
626            onPause();
627            break;
628        }
629
630        case kWhatResume:
631        {
632            onResume();
633            break;
634        }
635
636        case kWhatSetVideoFrameRate:
637        {
638            float fps;
639            CHECK(msg->findFloat("frame-rate", &fps));
640            onSetVideoFrameRate(fps);
641            break;
642        }
643
644        case kWhatAudioTearDown:
645        {
646            onAudioTearDown(kDueToError);
647            break;
648        }
649
650        case kWhatAudioOffloadPauseTimeout:
651        {
652            int32_t generation;
653            CHECK(msg->findInt32("drainGeneration", &generation));
654            if (generation != mAudioOffloadPauseTimeoutGeneration) {
655                break;
656            }
657            ALOGV("Audio Offload tear down due to pause timeout.");
658            onAudioTearDown(kDueToTimeout);
659            mWakeLock->release();
660            break;
661        }
662
663        default:
664            TRESPASS();
665            break;
666    }
667}
668
669void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
670    if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) {
671        return;
672    }
673
674    if (mAudioQueue.empty()) {
675        return;
676    }
677
678    // FIXME: if paused, wait until AudioTrack stop() is complete before delivering data.
679    if (mPaused) {
680        const int64_t diffUs = mPauseDrainAudioAllowedUs - ALooper::GetNowUs();
681        if (diffUs > delayUs) {
682            delayUs = diffUs;
683        }
684    }
685
686    mDrainAudioQueuePending = true;
687    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
688    msg->setInt32("drainGeneration", mAudioDrainGeneration);
689    msg->post(delayUs);
690}
691
692void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
693    mAudioRenderingStartGeneration = mAudioDrainGeneration;
694    mVideoRenderingStartGeneration = mVideoDrainGeneration;
695    mRenderingDataDelivered = false;
696}
697
698void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
699    if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
700        mAudioRenderingStartGeneration == mAudioDrainGeneration) {
701        mRenderingDataDelivered = true;
702        if (mPaused) {
703            return;
704        }
705        mVideoRenderingStartGeneration = -1;
706        mAudioRenderingStartGeneration = -1;
707
708        sp<AMessage> notify = mNotify->dup();
709        notify->setInt32("what", kWhatMediaRenderingStart);
710        notify->post();
711    }
712}
713
714// static
715size_t NuPlayer::Renderer::AudioSinkCallback(
716        MediaPlayerBase::AudioSink * /* audioSink */,
717        void *buffer,
718        size_t size,
719        void *cookie,
720        MediaPlayerBase::AudioSink::cb_event_t event) {
721    NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
722
723    switch (event) {
724        case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
725        {
726            return me->fillAudioBuffer(buffer, size);
727            break;
728        }
729
730        case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
731        {
732            ALOGV("AudioSink::CB_EVENT_STREAM_END");
733            me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
734            break;
735        }
736
737        case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
738        {
739            ALOGV("AudioSink::CB_EVENT_TEAR_DOWN");
740            me->notifyAudioTearDown();
741            break;
742        }
743    }
744
745    return 0;
746}
747
748size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
749    Mutex::Autolock autoLock(mLock);
750
751    if (!mUseAudioCallback) {
752        return 0;
753    }
754
755    bool hasEOS = false;
756
757    size_t sizeCopied = 0;
758    bool firstEntry = true;
759    QueueEntry *entry;  // will be valid after while loop if hasEOS is set.
760    while (sizeCopied < size && !mAudioQueue.empty()) {
761        entry = &*mAudioQueue.begin();
762
763        if (entry->mBuffer == NULL) { // EOS
764            hasEOS = true;
765            mAudioQueue.erase(mAudioQueue.begin());
766            break;
767        }
768
769        if (firstEntry && entry->mOffset == 0) {
770            firstEntry = false;
771            int64_t mediaTimeUs;
772            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
773            ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
774            setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
775        }
776
777        size_t copy = entry->mBuffer->size() - entry->mOffset;
778        size_t sizeRemaining = size - sizeCopied;
779        if (copy > sizeRemaining) {
780            copy = sizeRemaining;
781        }
782
783        memcpy((char *)buffer + sizeCopied,
784               entry->mBuffer->data() + entry->mOffset,
785               copy);
786
787        entry->mOffset += copy;
788        if (entry->mOffset == entry->mBuffer->size()) {
789            entry->mNotifyConsumed->post();
790            mAudioQueue.erase(mAudioQueue.begin());
791            entry = NULL;
792        }
793        sizeCopied += copy;
794
795        notifyIfMediaRenderingStarted_l();
796    }
797
798    if (mAudioFirstAnchorTimeMediaUs >= 0) {
799        int64_t nowUs = ALooper::GetNowUs();
800        int64_t nowMediaUs =
801            mAudioFirstAnchorTimeMediaUs + mAudioSink->getPlayedOutDurationUs(nowUs);
802        // we don't know how much data we are queueing for offloaded tracks.
803        mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
804    }
805
806    // for non-offloaded audio, we need to compute the frames written because
807    // there is no EVENT_STREAM_END notification. The frames written gives
808    // an estimate on the pending played out duration.
809    if (!offloadingAudio()) {
810        mNumFramesWritten += sizeCopied / mAudioSink->frameSize();
811    }
812
813    if (hasEOS) {
814        (new AMessage(kWhatStopAudioSink, this))->post();
815        // As there is currently no EVENT_STREAM_END callback notification for
816        // non-offloaded audio tracks, we need to post the EOS ourselves.
817        if (!offloadingAudio()) {
818            int64_t postEOSDelayUs = 0;
819            if (mAudioSink->needsTrailingPadding()) {
820                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
821            }
822            ALOGV("fillAudioBuffer: notifyEOS "
823                    "mNumFramesWritten:%u  finalResult:%d  postEOSDelay:%lld",
824                    mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs);
825            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
826        }
827    }
828    return sizeCopied;
829}
830
831void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() {
832    List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it;
833    bool foundEOS = false;
834    while (it != mAudioQueue.end()) {
835        int32_t eos;
836        QueueEntry *entry = &*it++;
837        if (entry->mBuffer == NULL
838                || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) {
839            itEOS = it;
840            foundEOS = true;
841        }
842    }
843
844    if (foundEOS) {
845        // post all replies before EOS and drop the samples
846        for (it = mAudioQueue.begin(); it != itEOS; it++) {
847            if (it->mBuffer == NULL) {
848                // delay doesn't matter as we don't even have an AudioTrack
849                notifyEOS(true /* audio */, it->mFinalResult);
850            } else {
851                it->mNotifyConsumed->post();
852            }
853        }
854        mAudioQueue.erase(mAudioQueue.begin(), itEOS);
855    }
856}
857
858bool NuPlayer::Renderer::onDrainAudioQueue() {
859    // do not drain audio during teardown as queued buffers may be invalid.
860    if (mAudioTornDown) {
861        return false;
862    }
863    // TODO: This call to getPosition checks if AudioTrack has been created
864    // in AudioSink before draining audio. If AudioTrack doesn't exist, then
865    // CHECKs on getPosition will fail.
866    // We still need to figure out why AudioTrack is not created when
867    // this function is called. One possible reason could be leftover
868    // audio. Another possible place is to check whether decoder
869    // has received INFO_FORMAT_CHANGED as the first buffer since
870    // AudioSink is opened there, and possible interactions with flush
871    // immediately after start. Investigate error message
872    // "vorbis_dsp_synthesis returned -135", along with RTSP.
873    uint32_t numFramesPlayed;
874    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
875        // When getPosition fails, renderer will not reschedule the draining
876        // unless new samples are queued.
877        // If we have pending EOS (or "eos" marker for discontinuities), we need
878        // to post these now as NuPlayerDecoder might be waiting for it.
879        drainAudioQueueUntilLastEOS();
880
881        ALOGW("onDrainAudioQueue(): audio sink is not ready");
882        return false;
883    }
884
885#if 0
886    ssize_t numFramesAvailableToWrite =
887        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
888
889    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
890        ALOGI("audio sink underrun");
891    } else {
892        ALOGV("audio queue has %d frames left to play",
893             mAudioSink->frameCount() - numFramesAvailableToWrite);
894    }
895#endif
896
897    uint32_t prevFramesWritten = mNumFramesWritten;
898    while (!mAudioQueue.empty()) {
899        QueueEntry *entry = &*mAudioQueue.begin();
900
901        mLastAudioBufferDrained = entry->mBufferOrdinal;
902
903        if (entry->mBuffer == NULL) {
904            // EOS
905            int64_t postEOSDelayUs = 0;
906            if (mAudioSink->needsTrailingPadding()) {
907                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
908            }
909            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
910            mLastAudioMediaTimeUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
911
912            mAudioQueue.erase(mAudioQueue.begin());
913            entry = NULL;
914            if (mAudioSink->needsTrailingPadding()) {
915                // If we're not in gapless playback (i.e. through setNextPlayer), we
916                // need to stop the track here, because that will play out the last
917                // little bit at the end of the file. Otherwise short files won't play.
918                mAudioSink->stop();
919                mNumFramesWritten = 0;
920            }
921            return false;
922        }
923
924        // ignore 0-sized buffer which could be EOS marker with no data
925        if (entry->mOffset == 0 && entry->mBuffer->size() > 0) {
926            int64_t mediaTimeUs;
927            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
928            ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs",
929                    mediaTimeUs / 1E6);
930            onNewAudioMediaTime(mediaTimeUs);
931        }
932
933        size_t copy = entry->mBuffer->size() - entry->mOffset;
934
935        ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
936                                            copy, false /* blocking */);
937        if (written < 0) {
938            // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
939            if (written == WOULD_BLOCK) {
940                ALOGV("AudioSink write would block when writing %zu bytes", copy);
941            } else {
942                ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
943                // This can only happen when AudioSink was opened with doNotReconnect flag set to
944                // true, in which case the NuPlayer will handle the reconnect.
945                notifyAudioTearDown();
946            }
947            break;
948        }
949
950        entry->mOffset += written;
951        if (entry->mOffset == entry->mBuffer->size()) {
952            entry->mNotifyConsumed->post();
953            mAudioQueue.erase(mAudioQueue.begin());
954
955            entry = NULL;
956        }
957
958        size_t copiedFrames = written / mAudioSink->frameSize();
959        mNumFramesWritten += copiedFrames;
960
961        {
962            Mutex::Autolock autoLock(mLock);
963            int64_t maxTimeMedia;
964            maxTimeMedia =
965                mAnchorTimeMediaUs +
966                        (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
967                                * 1000LL * mAudioSink->msecsPerFrame());
968            mMediaClock->updateMaxTimeMedia(maxTimeMedia);
969
970            notifyIfMediaRenderingStarted_l();
971        }
972
973        if (written != (ssize_t)copy) {
974            // A short count was received from AudioSink::write()
975            //
976            // AudioSink write is called in non-blocking mode.
977            // It may return with a short count when:
978            //
979            // 1) Size to be copied is not a multiple of the frame size. We consider this fatal.
980            // 2) The data to be copied exceeds the available buffer in AudioSink.
981            // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
982            // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
983
984            // (Case 1)
985            // Must be a multiple of the frame size.  If it is not a multiple of a frame size, it
986            // needs to fail, as we should not carry over fractional frames between calls.
987            CHECK_EQ(copy % mAudioSink->frameSize(), 0);
988
989            // (Case 2, 3, 4)
990            // Return early to the caller.
991            // Beware of calling immediately again as this may busy-loop if you are not careful.
992            ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
993            break;
994        }
995    }
996
997    // calculate whether we need to reschedule another write.
998    bool reschedule = !mAudioQueue.empty()
999            && (!mPaused
1000                || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers
1001    //ALOGD("reschedule:%d  empty:%d  mPaused:%d  prevFramesWritten:%u  mNumFramesWritten:%u",
1002    //        reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten);
1003    return reschedule;
1004}
1005
1006int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
1007    int32_t sampleRate = offloadingAudio() ?
1008            mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
1009    if (sampleRate == 0) {
1010        ALOGE("sampleRate is 0 in %s mode", offloadingAudio() ? "offload" : "non-offload");
1011        return 0;
1012    }
1013    // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
1014    return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate);
1015}
1016
1017// Calculate duration of pending samples if played at normal rate (i.e., 1.0).
1018int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
1019    int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
1020    return writtenAudioDurationUs - mAudioSink->getPlayedOutDurationUs(nowUs);
1021}
1022
1023int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
1024    int64_t realUs;
1025    if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
1026        // If failed to get current position, e.g. due to audio clock is
1027        // not ready, then just play out video immediately without delay.
1028        return nowUs;
1029    }
1030    return realUs;
1031}
1032
1033void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
1034    Mutex::Autolock autoLock(mLock);
1035    // TRICKY: vorbis decoder generates multiple frames with the same
1036    // timestamp, so only update on the first frame with a given timestamp
1037    if (mediaTimeUs == mAnchorTimeMediaUs) {
1038        return;
1039    }
1040    setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
1041
1042    AudioTimestamp ts;
1043    status_t res = mAudioSink->getTimestamp(ts);
1044    int64_t nowUs = ALooper::GetNowUs();
1045    if (res == OK) {
1046        int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
1047        mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
1048    } else {
1049        int64_t unused;
1050        if ((mMediaClock->getMediaTime(nowUs, &unused) != OK)
1051                && (getDurationUsIfPlayedAtSampleRate(mNumFramesWritten)
1052                        > kMaxAllowedAudioSinkDelayUs)) {
1053            // Enough data has been sent to AudioSink, but AudioSink has not rendered
1054            // any data yet. Something is wrong with AudioSink, e.g., the device is not
1055            // connected to audio out.
1056            // Switch to system clock. This essentially creates a virtual AudioSink with
1057            // initial latenty of getDurationUsIfPlayedAtSampleRate(mNumFramesWritten).
1058            // This virtual AudioSink renders audio data starting from the very first sample
1059            // and it's paced by system clock.
1060            ALOGW(""AudioSink stuck. ARE YOU CONNECTED TO AUDIO OUT? Switching to system clock.");
1061            mMediaClock->updateAnchor(mAudioFirstAnchorTimeMediaUs, nowUs, mediaTimeUs);
1062        }
1063    }
1064    mAnchorNumFramesWritten = mNumFramesWritten;
1065    mAnchorTimeMediaUs = mediaTimeUs;
1066}
1067
1068// Called without mLock acquired.
1069void NuPlayer::Renderer::postDrainVideoQueue() {
1070    if (mDrainVideoQueuePending
1071            || getSyncQueues()
1072            || (mPaused && mVideoSampleReceived)) {
1073        return;
1074    }
1075
1076    if (mVideoQueue.empty()) {
1077        return;
1078    }
1079
1080    QueueEntry &entry = *mVideoQueue.begin();
1081
1082    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
1083    msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
1084
1085    if (entry.mBuffer == NULL) {
1086        // EOS doesn't carry a timestamp.
1087        msg->post();
1088        mDrainVideoQueuePending = true;
1089        return;
1090    }
1091
1092    bool needRepostDrainVideoQueue = false;
1093    int64_t delayUs;
1094    int64_t nowUs = ALooper::GetNowUs();
1095    int64_t realTimeUs;
1096    if (mFlags & FLAG_REAL_TIME) {
1097        int64_t mediaTimeUs;
1098        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1099        realTimeUs = mediaTimeUs;
1100    } else {
1101        int64_t mediaTimeUs;
1102        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1103
1104        {
1105            Mutex::Autolock autoLock(mLock);
1106            if (mAnchorTimeMediaUs < 0) {
1107                mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
1108                mAnchorTimeMediaUs = mediaTimeUs;
1109                realTimeUs = nowUs;
1110            } else if (!mVideoSampleReceived) {
1111                // Always render the first video frame.
1112                realTimeUs = nowUs;
1113            } else if (mAudioFirstAnchorTimeMediaUs < 0
1114                || mMediaClock->getRealTimeFor(mediaTimeUs, &realTimeUs) == OK) {
1115                realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1116            } else if (mediaTimeUs - mAudioFirstAnchorTimeMediaUs >= 0) {
1117                needRepostDrainVideoQueue = true;
1118                realTimeUs = nowUs;
1119            } else {
1120                realTimeUs = nowUs;
1121            }
1122        }
1123        if (!mHasAudio) {
1124            // smooth out videos >= 10fps
1125            mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
1126        }
1127
1128        // Heuristics to handle situation when media time changed without a
1129        // discontinuity. If we have not drained an audio buffer that was
1130        // received after this buffer, repost in 10 msec. Otherwise repost
1131        // in 500 msec.
1132        delayUs = realTimeUs - nowUs;
1133        int64_t postDelayUs = -1;
1134        if (delayUs > 500000) {
1135            postDelayUs = 500000;
1136            if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) {
1137                postDelayUs = 10000;
1138            }
1139        } else if (needRepostDrainVideoQueue) {
1140            // CHECK(mPlaybackRate > 0);
1141            // CHECK(mAudioFirstAnchorTimeMediaUs >= 0);
1142            // CHECK(mediaTimeUs - mAudioFirstAnchorTimeMediaUs >= 0);
1143            postDelayUs = mediaTimeUs - mAudioFirstAnchorTimeMediaUs;
1144            postDelayUs /= mPlaybackRate;
1145        }
1146
1147        if (postDelayUs >= 0) {
1148            msg->setWhat(kWhatPostDrainVideoQueue);
1149            msg->post(postDelayUs);
1150            mVideoScheduler->restart();
1151            ALOGI("possible video time jump of %dms or uninitialized media clock, retrying in %dms",
1152                    (int)(delayUs / 1000), (int)(postDelayUs / 1000));
1153            mDrainVideoQueuePending = true;
1154            return;
1155        }
1156    }
1157
1158    realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
1159    int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
1160
1161    delayUs = realTimeUs - nowUs;
1162
1163    ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
1164    // post 2 display refreshes before rendering is due
1165    msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
1166
1167    mDrainVideoQueuePending = true;
1168}
1169
1170void NuPlayer::Renderer::onDrainVideoQueue() {
1171    if (mVideoQueue.empty()) {
1172        return;
1173    }
1174
1175    QueueEntry *entry = &*mVideoQueue.begin();
1176
1177    if (entry->mBuffer == NULL) {
1178        // EOS
1179
1180        notifyEOS(false /* audio */, entry->mFinalResult);
1181
1182        mVideoQueue.erase(mVideoQueue.begin());
1183        entry = NULL;
1184
1185        setVideoLateByUs(0);
1186        return;
1187    }
1188
1189    int64_t nowUs = ALooper::GetNowUs();
1190    int64_t realTimeUs;
1191    int64_t mediaTimeUs = -1;
1192    if (mFlags & FLAG_REAL_TIME) {
1193        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
1194    } else {
1195        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1196
1197        realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1198    }
1199
1200    bool tooLate = false;
1201
1202    if (!mPaused) {
1203        setVideoLateByUs(nowUs - realTimeUs);
1204        tooLate = (mVideoLateByUs > 40000);
1205
1206        if (tooLate) {
1207            ALOGV("video late by %lld us (%.2f secs)",
1208                 (long long)mVideoLateByUs, mVideoLateByUs / 1E6);
1209        } else {
1210            int64_t mediaUs = 0;
1211            mMediaClock->getMediaTime(realTimeUs, &mediaUs);
1212            ALOGV("rendering video at media time %.2f secs",
1213                    (mFlags & FLAG_REAL_TIME ? realTimeUs :
1214                    mediaUs) / 1E6);
1215
1216            if (!(mFlags & FLAG_REAL_TIME)
1217                    && mLastAudioMediaTimeUs != -1
1218                    && mediaTimeUs > mLastAudioMediaTimeUs) {
1219                // If audio ends before video, video continues to drive media clock.
1220                // Also smooth out videos >= 10fps.
1221                mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
1222            }
1223        }
1224    } else {
1225        setVideoLateByUs(0);
1226        if (!mVideoSampleReceived && !mHasAudio) {
1227            // This will ensure that the first frame after a flush won't be used as anchor
1228            // when renderer is in paused state, because resume can happen any time after seek.
1229            Mutex::Autolock autoLock(mLock);
1230            clearAnchorTime_l();
1231        }
1232    }
1233
1234    // Always render the first video frame while keeping stats on A/V sync.
1235    if (!mVideoSampleReceived) {
1236        realTimeUs = nowUs;
1237        tooLate = false;
1238    }
1239
1240    entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
1241    entry->mNotifyConsumed->setInt32("render", !tooLate);
1242    entry->mNotifyConsumed->post();
1243    mVideoQueue.erase(mVideoQueue.begin());
1244    entry = NULL;
1245
1246    mVideoSampleReceived = true;
1247
1248    if (!mPaused) {
1249        if (!mVideoRenderingStarted) {
1250            mVideoRenderingStarted = true;
1251            notifyVideoRenderingStart();
1252        }
1253        Mutex::Autolock autoLock(mLock);
1254        notifyIfMediaRenderingStarted_l();
1255    }
1256}
1257
1258void NuPlayer::Renderer::notifyVideoRenderingStart() {
1259    sp<AMessage> notify = mNotify->dup();
1260    notify->setInt32("what", kWhatVideoRenderingStart);
1261    notify->post();
1262}
1263
1264void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
1265    if (audio && delayUs > 0) {
1266        sp<AMessage> msg = new AMessage(kWhatEOS, this);
1267        msg->setInt32("audioEOSGeneration", mAudioEOSGeneration);
1268        msg->setInt32("finalResult", finalResult);
1269        msg->post(delayUs);
1270        return;
1271    }
1272    sp<AMessage> notify = mNotify->dup();
1273    notify->setInt32("what", kWhatEOS);
1274    notify->setInt32("audio", static_cast<int32_t>(audio));
1275    notify->setInt32("finalResult", finalResult);
1276    notify->post(delayUs);
1277}
1278
1279void NuPlayer::Renderer::notifyAudioTearDown() {
1280    (new AMessage(kWhatAudioTearDown, this))->post();
1281}
1282
1283void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
1284    int32_t audio;
1285    CHECK(msg->findInt32("audio", &audio));
1286
1287    if (dropBufferIfStale(audio, msg)) {
1288        return;
1289    }
1290
1291    if (audio) {
1292        mHasAudio = true;
1293    } else {
1294        mHasVideo = true;
1295    }
1296
1297    if (mHasVideo) {
1298        if (mVideoScheduler == NULL) {
1299            mVideoScheduler = new VideoFrameScheduler();
1300            mVideoScheduler->init();
1301        }
1302    }
1303
1304    sp<ABuffer> buffer;
1305    CHECK(msg->findBuffer("buffer", &buffer));
1306
1307    sp<AMessage> notifyConsumed;
1308    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
1309
1310    QueueEntry entry;
1311    entry.mBuffer = buffer;
1312    entry.mNotifyConsumed = notifyConsumed;
1313    entry.mOffset = 0;
1314    entry.mFinalResult = OK;
1315    entry.mBufferOrdinal = ++mTotalBuffersQueued;
1316
1317    if (audio) {
1318        Mutex::Autolock autoLock(mLock);
1319        mAudioQueue.push_back(entry);
1320        postDrainAudioQueue_l();
1321    } else {
1322        mVideoQueue.push_back(entry);
1323        postDrainVideoQueue();
1324    }
1325
1326    Mutex::Autolock autoLock(mLock);
1327    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
1328        return;
1329    }
1330
1331    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
1332    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
1333
1334    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
1335        // EOS signalled on either queue.
1336        syncQueuesDone_l();
1337        return;
1338    }
1339
1340    int64_t firstAudioTimeUs;
1341    int64_t firstVideoTimeUs;
1342    CHECK(firstAudioBuffer->meta()
1343            ->findInt64("timeUs", &firstAudioTimeUs));
1344    CHECK(firstVideoBuffer->meta()
1345            ->findInt64("timeUs", &firstVideoTimeUs));
1346
1347    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
1348
1349    ALOGV("queueDiff = %.2f secs", diff / 1E6);
1350
1351    if (diff > 100000ll) {
1352        // Audio data starts More than 0.1 secs before video.
1353        // Drop some audio.
1354
1355        (*mAudioQueue.begin()).mNotifyConsumed->post();
1356        mAudioQueue.erase(mAudioQueue.begin());
1357        return;
1358    }
1359
1360    syncQueuesDone_l();
1361}
1362
1363void NuPlayer::Renderer::syncQueuesDone_l() {
1364    if (!mSyncQueues) {
1365        return;
1366    }
1367
1368    mSyncQueues = false;
1369
1370    if (!mAudioQueue.empty()) {
1371        postDrainAudioQueue_l();
1372    }
1373
1374    if (!mVideoQueue.empty()) {
1375        mLock.unlock();
1376        postDrainVideoQueue();
1377        mLock.lock();
1378    }
1379}
1380
1381void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1382    int32_t audio;
1383    CHECK(msg->findInt32("audio", &audio));
1384
1385    if (dropBufferIfStale(audio, msg)) {
1386        return;
1387    }
1388
1389    int32_t finalResult;
1390    CHECK(msg->findInt32("finalResult", &finalResult));
1391
1392    QueueEntry entry;
1393    entry.mOffset = 0;
1394    entry.mFinalResult = finalResult;
1395
1396    if (audio) {
1397        Mutex::Autolock autoLock(mLock);
1398        if (mAudioQueue.empty() && mSyncQueues) {
1399            syncQueuesDone_l();
1400        }
1401        mAudioQueue.push_back(entry);
1402        postDrainAudioQueue_l();
1403    } else {
1404        if (mVideoQueue.empty() && getSyncQueues()) {
1405            Mutex::Autolock autoLock(mLock);
1406            syncQueuesDone_l();
1407        }
1408        mVideoQueue.push_back(entry);
1409        postDrainVideoQueue();
1410    }
1411}
1412
1413void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
1414    int32_t audio, notifyComplete;
1415    CHECK(msg->findInt32("audio", &audio));
1416
1417    {
1418        Mutex::Autolock autoLock(mLock);
1419        if (audio) {
1420            notifyComplete = mNotifyCompleteAudio;
1421            mNotifyCompleteAudio = false;
1422            mLastAudioMediaTimeUs = -1;
1423        } else {
1424            notifyComplete = mNotifyCompleteVideo;
1425            mNotifyCompleteVideo = false;
1426        }
1427
1428        // If we're currently syncing the queues, i.e. dropping audio while
1429        // aligning the first audio/video buffer times and only one of the
1430        // two queues has data, we may starve that queue by not requesting
1431        // more buffers from the decoder. If the other source then encounters
1432        // a discontinuity that leads to flushing, we'll never find the
1433        // corresponding discontinuity on the other queue.
1434        // Therefore we'll stop syncing the queues if at least one of them
1435        // is flushed.
1436        syncQueuesDone_l();
1437        clearAnchorTime_l();
1438    }
1439
1440    ALOGV("flushing %s", audio ? "audio" : "video");
1441    if (audio) {
1442        {
1443            Mutex::Autolock autoLock(mLock);
1444            flushQueue(&mAudioQueue);
1445
1446            ++mAudioDrainGeneration;
1447            ++mAudioEOSGeneration;
1448            prepareForMediaRenderingStart_l();
1449
1450            // the frame count will be reset after flush.
1451            clearAudioFirstAnchorTime_l();
1452        }
1453
1454        mDrainAudioQueuePending = false;
1455
1456        if (offloadingAudio()) {
1457            mAudioSink->pause();
1458            mAudioSink->flush();
1459            if (!mPaused) {
1460                mAudioSink->start();
1461            }
1462        } else {
1463            mAudioSink->pause();
1464            mAudioSink->flush();
1465            // Call stop() to signal to the AudioSink to completely fill the
1466            // internal buffer before resuming playback.
1467            // FIXME: this is ignored after flush().
1468            mAudioSink->stop();
1469            if (mPaused) {
1470                // Race condition: if renderer is paused and audio sink is stopped,
1471                // we need to make sure that the audio track buffer fully drains
1472                // before delivering data.
1473                // FIXME: remove this if we can detect if stop() is complete.
1474                const int delayUs = 2 * 50 * 1000; // (2 full mixer thread cycles at 50ms)
1475                mPauseDrainAudioAllowedUs = ALooper::GetNowUs() + delayUs;
1476            } else {
1477                mAudioSink->start();
1478            }
1479            mNumFramesWritten = 0;
1480        }
1481    } else {
1482        flushQueue(&mVideoQueue);
1483
1484        mDrainVideoQueuePending = false;
1485
1486        if (mVideoScheduler != NULL) {
1487            mVideoScheduler->restart();
1488        }
1489
1490        Mutex::Autolock autoLock(mLock);
1491        ++mVideoDrainGeneration;
1492        prepareForMediaRenderingStart_l();
1493    }
1494
1495    mVideoSampleReceived = false;
1496
1497    if (notifyComplete) {
1498        notifyFlushComplete(audio);
1499    }
1500}
1501
1502void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1503    while (!queue->empty()) {
1504        QueueEntry *entry = &*queue->begin();
1505
1506        if (entry->mBuffer != NULL) {
1507            entry->mNotifyConsumed->post();
1508        }
1509
1510        queue->erase(queue->begin());
1511        entry = NULL;
1512    }
1513}
1514
1515void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1516    sp<AMessage> notify = mNotify->dup();
1517    notify->setInt32("what", kWhatFlushComplete);
1518    notify->setInt32("audio", static_cast<int32_t>(audio));
1519    notify->post();
1520}
1521
1522bool NuPlayer::Renderer::dropBufferIfStale(
1523        bool audio, const sp<AMessage> &msg) {
1524    int32_t queueGeneration;
1525    CHECK(msg->findInt32("queueGeneration", &queueGeneration));
1526
1527    if (queueGeneration == getQueueGeneration(audio)) {
1528        return false;
1529    }
1530
1531    sp<AMessage> notifyConsumed;
1532    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1533        notifyConsumed->post();
1534    }
1535
1536    return true;
1537}
1538
1539void NuPlayer::Renderer::onAudioSinkChanged() {
1540    if (offloadingAudio()) {
1541        return;
1542    }
1543    CHECK(!mDrainAudioQueuePending);
1544    mNumFramesWritten = 0;
1545    {
1546        Mutex::Autolock autoLock(mLock);
1547        mAnchorNumFramesWritten = -1;
1548    }
1549    uint32_t written;
1550    if (mAudioSink->getFramesWritten(&written) == OK) {
1551        mNumFramesWritten = written;
1552    }
1553}
1554
1555void NuPlayer::Renderer::onDisableOffloadAudio() {
1556    Mutex::Autolock autoLock(mLock);
1557    mFlags &= ~FLAG_OFFLOAD_AUDIO;
1558    ++mAudioDrainGeneration;
1559    if (mAudioRenderingStartGeneration != -1) {
1560        prepareForMediaRenderingStart_l();
1561    }
1562}
1563
1564void NuPlayer::Renderer::onEnableOffloadAudio() {
1565    Mutex::Autolock autoLock(mLock);
1566    mFlags |= FLAG_OFFLOAD_AUDIO;
1567    ++mAudioDrainGeneration;
1568    if (mAudioRenderingStartGeneration != -1) {
1569        prepareForMediaRenderingStart_l();
1570    }
1571}
1572
1573void NuPlayer::Renderer::onPause() {
1574    if (mPaused) {
1575        return;
1576    }
1577
1578    {
1579        Mutex::Autolock autoLock(mLock);
1580        // we do not increment audio drain generation so that we fill audio buffer during pause.
1581        ++mVideoDrainGeneration;
1582        prepareForMediaRenderingStart_l();
1583        mPaused = true;
1584        mMediaClock->setPlaybackRate(0.0);
1585    }
1586
1587    mDrainAudioQueuePending = false;
1588    mDrainVideoQueuePending = false;
1589
1590    // Note: audio data may not have been decoded, and the AudioSink may not be opened.
1591    mAudioSink->pause();
1592    startAudioOffloadPauseTimeout();
1593
1594    ALOGV("now paused audio queue has %zu entries, video has %zu entries",
1595          mAudioQueue.size(), mVideoQueue.size());
1596}
1597
1598void NuPlayer::Renderer::onResume() {
1599    if (!mPaused) {
1600        return;
1601    }
1602
1603    // Note: audio data may not have been decoded, and the AudioSink may not be opened.
1604    cancelAudioOffloadPauseTimeout();
1605    if (mAudioSink->ready()) {
1606        status_t err = mAudioSink->start();
1607        if (err != OK) {
1608            ALOGE("cannot start AudioSink err %d", err);
1609            notifyAudioTearDown();
1610        }
1611    }
1612
1613    {
1614        Mutex::Autolock autoLock(mLock);
1615        mPaused = false;
1616        // rendering started message may have been delayed if we were paused.
1617        if (mRenderingDataDelivered) {
1618            notifyIfMediaRenderingStarted_l();
1619        }
1620        // configure audiosink as we did not do it when pausing
1621        if (mAudioSink != NULL && mAudioSink->ready()) {
1622            mAudioSink->setPlaybackRate(mPlaybackSettings);
1623        }
1624
1625        mMediaClock->setPlaybackRate(mPlaybackRate);
1626
1627        if (!mAudioQueue.empty()) {
1628            postDrainAudioQueue_l();
1629        }
1630    }
1631
1632    if (!mVideoQueue.empty()) {
1633        postDrainVideoQueue();
1634    }
1635}
1636
1637void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1638    if (mVideoScheduler == NULL) {
1639        mVideoScheduler = new VideoFrameScheduler();
1640    }
1641    mVideoScheduler->init(fps);
1642}
1643
1644int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
1645    Mutex::Autolock autoLock(mLock);
1646    return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
1647}
1648
1649int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
1650    Mutex::Autolock autoLock(mLock);
1651    return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
1652}
1653
1654bool NuPlayer::Renderer::getSyncQueues() {
1655    Mutex::Autolock autoLock(mLock);
1656    return mSyncQueues;
1657}
1658
1659void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) {
1660    if (mAudioTornDown) {
1661        return;
1662    }
1663    mAudioTornDown = true;
1664
1665    int64_t currentPositionUs;
1666    sp<AMessage> notify = mNotify->dup();
1667    if (getCurrentPosition(&currentPositionUs) == OK) {
1668        notify->setInt64("positionUs", currentPositionUs);
1669    }
1670
1671    mAudioSink->stop();
1672    mAudioSink->flush();
1673
1674    notify->setInt32("what", kWhatAudioTearDown);
1675    notify->setInt32("reason", reason);
1676    notify->post();
1677}
1678
1679void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1680    if (offloadingAudio()) {
1681        mWakeLock->acquire();
1682        sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
1683        msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
1684        msg->post(kOffloadPauseMaxUs);
1685    }
1686}
1687
1688void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
1689    if (offloadingAudio()) {
1690        mWakeLock->release(true);
1691        ++mAudioOffloadPauseTimeoutGeneration;
1692    }
1693}
1694
1695status_t NuPlayer::Renderer::onOpenAudioSink(
1696        const sp<AMessage> &format,
1697        bool offloadOnly,
1698        bool hasVideo,
1699        uint32_t flags) {
1700    ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1701            offloadOnly, offloadingAudio());
1702    bool audioSinkChanged = false;
1703
1704    int32_t numChannels;
1705    CHECK(format->findInt32("channel-count", &numChannels));
1706
1707    int32_t channelMask;
1708    if (!format->findInt32("channel-mask", &channelMask)) {
1709        // signal to the AudioSink to derive the mask from count.
1710        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1711    }
1712
1713    int32_t sampleRate;
1714    CHECK(format->findInt32("sample-rate", &sampleRate));
1715
1716    if (offloadingAudio()) {
1717        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1718        AString mime;
1719        CHECK(format->findString("mime", &mime));
1720        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1721
1722        if (err != OK) {
1723            ALOGE("Couldn't map mime \"%s\" to a valid "
1724                    "audio_format", mime.c_str());
1725            onDisableOffloadAudio();
1726        } else {
1727            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1728                    mime.c_str(), audioFormat);
1729
1730            int avgBitRate = -1;
1731            format->findInt32("bit-rate", &avgBitRate);
1732
1733            int32_t aacProfile = -1;
1734            if (audioFormat == AUDIO_FORMAT_AAC
1735                    && format->findInt32("aac-profile", &aacProfile)) {
1736                // Redefine AAC format as per aac profile
1737                mapAACProfileToAudioFormat(
1738                        audioFormat,
1739                        aacProfile);
1740            }
1741
1742            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1743            offloadInfo.duration_us = -1;
1744            format->findInt64(
1745                    "durationUs", &offloadInfo.duration_us);
1746            offloadInfo.sample_rate = sampleRate;
1747            offloadInfo.channel_mask = channelMask;
1748            offloadInfo.format = audioFormat;
1749            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1750            offloadInfo.bit_rate = avgBitRate;
1751            offloadInfo.has_video = hasVideo;
1752            offloadInfo.is_streaming = true;
1753
1754            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1755                ALOGV("openAudioSink: no change in offload mode");
1756                // no change from previous configuration, everything ok.
1757                return OK;
1758            }
1759            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1760
1761            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1762            uint32_t offloadFlags = flags;
1763            offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1764            offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1765            audioSinkChanged = true;
1766            mAudioSink->close();
1767
1768            err = mAudioSink->open(
1769                    sampleRate,
1770                    numChannels,
1771                    (audio_channel_mask_t)channelMask,
1772                    audioFormat,
1773                    0 /* bufferCount - unused */,
1774                    &NuPlayer::Renderer::AudioSinkCallback,
1775                    this,
1776                    (audio_output_flags_t)offloadFlags,
1777                    &offloadInfo);
1778
1779            if (err == OK) {
1780                err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1781            }
1782
1783            if (err == OK) {
1784                // If the playback is offloaded to h/w, we pass
1785                // the HAL some metadata information.
1786                // We don't want to do this for PCM because it
1787                // will be going through the AudioFlinger mixer
1788                // before reaching the hardware.
1789                // TODO
1790                mCurrentOffloadInfo = offloadInfo;
1791                if (!mPaused) { // for preview mode, don't start if paused
1792                    err = mAudioSink->start();
1793                }
1794                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1795            }
1796            if (err != OK) {
1797                // Clean up, fall back to non offload mode.
1798                mAudioSink->close();
1799                onDisableOffloadAudio();
1800                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1801                ALOGV("openAudioSink: offload failed");
1802            } else {
1803                mUseAudioCallback = true;  // offload mode transfers data through callback
1804                ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
1805            }
1806        }
1807    }
1808    if (!offloadOnly && !offloadingAudio()) {
1809        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1810        uint32_t pcmFlags = flags;
1811        pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1812
1813        const PcmInfo info = {
1814                (audio_channel_mask_t)channelMask,
1815                (audio_output_flags_t)pcmFlags,
1816                AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat
1817                numChannels,
1818                sampleRate
1819        };
1820        if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
1821            ALOGV("openAudioSink: no change in pcm mode");
1822            // no change from previous configuration, everything ok.
1823            return OK;
1824        }
1825
1826        audioSinkChanged = true;
1827        mAudioSink->close();
1828        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1829        // Note: It is possible to set up the callback, but not use it to send audio data.
1830        // This requires a fix in AudioSink to explicitly specify the transfer mode.
1831        mUseAudioCallback = getUseAudioCallbackSetting();
1832        if (mUseAudioCallback) {
1833            ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
1834        }
1835
1836        // Compute the desired buffer size.
1837        // For callback mode, the amount of time before wakeup is about half the buffer size.
1838        const uint32_t frameCount =
1839                (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000;
1840
1841        // The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct
1842        // AudioSink. We don't want this when there's video because it will cause a video seek to
1843        // the previous I frame. But we do want this when there's only audio because it will give
1844        // NuPlayer a chance to switch from non-offload mode to offload mode.
1845        // So we only set doNotReconnect when there's no video.
1846        const bool doNotReconnect = !hasVideo;
1847        status_t err = mAudioSink->open(
1848                    sampleRate,
1849                    numChannels,
1850                    (audio_channel_mask_t)channelMask,
1851                    AUDIO_FORMAT_PCM_16_BIT,
1852                    0 /* bufferCount - unused */,
1853                    mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL,
1854                    mUseAudioCallback ? this : NULL,
1855                    (audio_output_flags_t)pcmFlags,
1856                    NULL,
1857                    doNotReconnect,
1858                    frameCount);
1859        if (err == OK) {
1860            err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1861        }
1862        if (err != OK) {
1863            ALOGW("openAudioSink: non offloaded open failed status: %d", err);
1864            mAudioSink->close();
1865            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1866            return err;
1867        }
1868        mCurrentPcmInfo = info;
1869        if (!mPaused) { // for preview mode, don't start if paused
1870            mAudioSink->start();
1871        }
1872    }
1873    if (audioSinkChanged) {
1874        onAudioSinkChanged();
1875    }
1876    mAudioTornDown = false;
1877    return OK;
1878}
1879
1880void NuPlayer::Renderer::onCloseAudioSink() {
1881    mAudioSink->close();
1882    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1883    mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1884}
1885
1886}  // namespace android
1887
1888