NuPlayerRenderer.cpp revision 00541e293c250ea9e31b6ab2dc134802e34fe843 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22#include <cutils/properties.h>
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26#include <media/stagefright/foundation/AUtils.h>
27#include <media/stagefright/foundation/AWakeLock.h>
28#include <media/stagefright/MediaClock.h>
29#include <media/stagefright/MediaErrors.h>
30#include <media/stagefright/MetaData.h>
31#include <media/stagefright/Utils.h>
32#include <media/stagefright/VideoFrameScheduler.h>
33
34#include <inttypes.h>
35
36namespace android {
37
38/*
39 * Example of common configuration settings in shell script form
40
41   #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager
42   adb shell setprop audio.offload.disable 1
43
44   #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager
45   adb shell setprop audio.offload.video 1
46
47   #Use audio callbacks for PCM data
48   adb shell setprop media.stagefright.audio.cbk 1
49
50   #Use deep buffer for PCM data with video (it is generally enabled for audio-only)
51   adb shell setprop media.stagefright.audio.deep 1
52
53   #Set size of buffers for pcm audio sink in msec (example: 1000 msec)
54   adb shell setprop media.stagefright.audio.sink 1000
55
56 * These configurations take effect for the next track played (not the current track).
57 */
58
59static inline bool getUseAudioCallbackSetting() {
60    return property_get_bool("media.stagefright.audio.cbk", false /* default_value */);
61}
62
63static inline int32_t getAudioSinkPcmMsSetting() {
64    return property_get_int32(
65            "media.stagefright.audio.sink", 500 /* default_value */);
66}
67
68// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
69// is closed to allow the audio DSP to power down.
70static const int64_t kOffloadPauseMaxUs = 10000000ll;
71
72// static
73const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
74        AUDIO_CHANNEL_NONE,
75        AUDIO_OUTPUT_FLAG_NONE,
76        AUDIO_FORMAT_INVALID,
77        0, // mNumChannels
78        0 // mSampleRate
79};
80
81// static
82const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
83
84NuPlayer::Renderer::Renderer(
85        const sp<MediaPlayerBase::AudioSink> &sink,
86        const sp<AMessage> &notify,
87        uint32_t flags)
88    : mAudioSink(sink),
89      mNotify(notify),
90      mFlags(flags),
91      mNumFramesWritten(0),
92      mDrainAudioQueuePending(false),
93      mDrainVideoQueuePending(false),
94      mAudioQueueGeneration(0),
95      mVideoQueueGeneration(0),
96      mAudioDrainGeneration(0),
97      mVideoDrainGeneration(0),
98      mAudioEOSGeneration(0),
99      mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
100      mAudioFirstAnchorTimeMediaUs(-1),
101      mAnchorTimeMediaUs(-1),
102      mAnchorNumFramesWritten(-1),
103      mVideoLateByUs(0ll),
104      mHasAudio(false),
105      mHasVideo(false),
106      mNotifyCompleteAudio(false),
107      mNotifyCompleteVideo(false),
108      mSyncQueues(false),
109      mPaused(false),
110      mPauseDrainAudioAllowedUs(0),
111      mVideoSampleReceived(false),
112      mVideoRenderingStarted(false),
113      mVideoRenderingStartGeneration(0),
114      mAudioRenderingStartGeneration(0),
115      mRenderingDataDelivered(false),
116      mLastAudioMediaTimeUs(-1),
117      mAudioOffloadPauseTimeoutGeneration(0),
118      mAudioTornDown(false),
119      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
120      mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
121      mTotalBuffersQueued(0),
122      mLastAudioBufferDrained(0),
123      mUseAudioCallback(false),
124      mWakeLock(new AWakeLock()) {
125    mMediaClock = new MediaClock;
126    mPlaybackRate = mPlaybackSettings.mSpeed;
127    mMediaClock->setPlaybackRate(mPlaybackRate);
128}
129
130NuPlayer::Renderer::~Renderer() {
131    if (offloadingAudio()) {
132        mAudioSink->stop();
133        mAudioSink->flush();
134        mAudioSink->close();
135    }
136}
137
138void NuPlayer::Renderer::queueBuffer(
139        bool audio,
140        const sp<ABuffer> &buffer,
141        const sp<AMessage> &notifyConsumed) {
142    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
143    msg->setInt32("queueGeneration", getQueueGeneration(audio));
144    msg->setInt32("audio", static_cast<int32_t>(audio));
145    msg->setBuffer("buffer", buffer);
146    msg->setMessage("notifyConsumed", notifyConsumed);
147    msg->post();
148}
149
150void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
151    CHECK_NE(finalResult, (status_t)OK);
152
153    sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
154    msg->setInt32("queueGeneration", getQueueGeneration(audio));
155    msg->setInt32("audio", static_cast<int32_t>(audio));
156    msg->setInt32("finalResult", finalResult);
157    msg->post();
158}
159
160status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) {
161    sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
162    writeToAMessage(msg, rate);
163    sp<AMessage> response;
164    status_t err = msg->postAndAwaitResponse(&response);
165    if (err == OK && response != NULL) {
166        CHECK(response->findInt32("err", &err));
167    }
168    return err;
169}
170
171status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) {
172    if (rate.mSpeed == 0.f) {
173        onPause();
174        // don't call audiosink's setPlaybackRate if pausing, as pitch does not
175        // have to correspond to the any non-0 speed (e.g old speed). Keep
176        // settings nonetheless, using the old speed, in case audiosink changes.
177        AudioPlaybackRate newRate = rate;
178        newRate.mSpeed = mPlaybackSettings.mSpeed;
179        mPlaybackSettings = newRate;
180        return OK;
181    }
182
183    if (mAudioSink != NULL && mAudioSink->ready()) {
184        status_t err = mAudioSink->setPlaybackRate(rate);
185        if (err != OK) {
186            return err;
187        }
188    }
189    mPlaybackSettings = rate;
190    mPlaybackRate = rate.mSpeed;
191    mMediaClock->setPlaybackRate(mPlaybackRate);
192    return OK;
193}
194
195status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
196    sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
197    sp<AMessage> response;
198    status_t err = msg->postAndAwaitResponse(&response);
199    if (err == OK && response != NULL) {
200        CHECK(response->findInt32("err", &err));
201        if (err == OK) {
202            readFromAMessage(response, rate);
203        }
204    }
205    return err;
206}
207
208status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
209    if (mAudioSink != NULL && mAudioSink->ready()) {
210        status_t err = mAudioSink->getPlaybackRate(rate);
211        if (err == OK) {
212            if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) {
213                ALOGW("correcting mismatch in internal/external playback rate");
214            }
215            // get playback settings used by audiosink, as it may be
216            // slightly off due to audiosink not taking small changes.
217            mPlaybackSettings = *rate;
218            if (mPaused) {
219                rate->mSpeed = 0.f;
220            }
221        }
222        return err;
223    }
224    *rate = mPlaybackSettings;
225    return OK;
226}
227
228status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
229    sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
230    writeToAMessage(msg, sync, videoFpsHint);
231    sp<AMessage> response;
232    status_t err = msg->postAndAwaitResponse(&response);
233    if (err == OK && response != NULL) {
234        CHECK(response->findInt32("err", &err));
235    }
236    return err;
237}
238
239status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) {
240    if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
241        return BAD_VALUE;
242    }
243    // TODO: support sync sources
244    return INVALID_OPERATION;
245}
246
247status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) {
248    sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
249    sp<AMessage> response;
250    status_t err = msg->postAndAwaitResponse(&response);
251    if (err == OK && response != NULL) {
252        CHECK(response->findInt32("err", &err));
253        if (err == OK) {
254            readFromAMessage(response, sync, videoFps);
255        }
256    }
257    return err;
258}
259
260status_t NuPlayer::Renderer::onGetSyncSettings(
261        AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
262    *sync = mSyncSettings;
263    *videoFps = -1.f;
264    return OK;
265}
266
267void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
268    {
269        Mutex::Autolock autoLock(mLock);
270        if (audio) {
271            mNotifyCompleteAudio |= notifyComplete;
272            clearAudioFirstAnchorTime_l();
273            ++mAudioQueueGeneration;
274            ++mAudioDrainGeneration;
275        } else {
276            mNotifyCompleteVideo |= notifyComplete;
277            ++mVideoQueueGeneration;
278            ++mVideoDrainGeneration;
279        }
280
281        clearAnchorTime_l();
282        mVideoLateByUs = 0;
283        mSyncQueues = false;
284    }
285
286    sp<AMessage> msg = new AMessage(kWhatFlush, this);
287    msg->setInt32("audio", static_cast<int32_t>(audio));
288    msg->post();
289}
290
291void NuPlayer::Renderer::signalTimeDiscontinuity() {
292}
293
294void NuPlayer::Renderer::signalDisableOffloadAudio() {
295    (new AMessage(kWhatDisableOffloadAudio, this))->post();
296}
297
298void NuPlayer::Renderer::signalEnableOffloadAudio() {
299    (new AMessage(kWhatEnableOffloadAudio, this))->post();
300}
301
302void NuPlayer::Renderer::pause() {
303    (new AMessage(kWhatPause, this))->post();
304}
305
306void NuPlayer::Renderer::resume() {
307    (new AMessage(kWhatResume, this))->post();
308}
309
310void NuPlayer::Renderer::setVideoFrameRate(float fps) {
311    sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
312    msg->setFloat("frame-rate", fps);
313    msg->post();
314}
315
316// Called on any threads without mLock acquired.
317status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
318    status_t result = mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
319    if (result == OK) {
320        return result;
321    }
322
323    // MediaClock has not started yet. Try to start it if possible.
324    {
325        Mutex::Autolock autoLock(mLock);
326        if (mAudioFirstAnchorTimeMediaUs == -1) {
327            return result;
328        }
329
330        AudioTimestamp ts;
331        status_t res = mAudioSink->getTimestamp(ts);
332        if (res != OK) {
333            return result;
334        }
335
336        // AudioSink has rendered some frames.
337        int64_t nowUs = ALooper::GetNowUs();
338        int64_t nowMediaUs = mAudioSink->getPlayedOutDurationUs(nowUs)
339                + mAudioFirstAnchorTimeMediaUs;
340        mMediaClock->updateAnchor(nowMediaUs, nowUs, -1);
341    }
342
343    return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
344}
345
346void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
347    mAudioFirstAnchorTimeMediaUs = -1;
348    mMediaClock->setStartingTimeMedia(-1);
349}
350
351void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
352    if (mAudioFirstAnchorTimeMediaUs == -1) {
353        mAudioFirstAnchorTimeMediaUs = mediaUs;
354        mMediaClock->setStartingTimeMedia(mediaUs);
355    }
356}
357
358void NuPlayer::Renderer::clearAnchorTime_l() {
359    mMediaClock->clearAnchor();
360    mAnchorTimeMediaUs = -1;
361    mAnchorNumFramesWritten = -1;
362}
363
364void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
365    Mutex::Autolock autoLock(mLock);
366    mVideoLateByUs = lateUs;
367}
368
369int64_t NuPlayer::Renderer::getVideoLateByUs() {
370    Mutex::Autolock autoLock(mLock);
371    return mVideoLateByUs;
372}
373
374status_t NuPlayer::Renderer::openAudioSink(
375        const sp<AMessage> &format,
376        bool offloadOnly,
377        bool hasVideo,
378        uint32_t flags,
379        bool *isOffloaded) {
380    sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
381    msg->setMessage("format", format);
382    msg->setInt32("offload-only", offloadOnly);
383    msg->setInt32("has-video", hasVideo);
384    msg->setInt32("flags", flags);
385
386    sp<AMessage> response;
387    msg->postAndAwaitResponse(&response);
388
389    int32_t err;
390    if (!response->findInt32("err", &err)) {
391        err = INVALID_OPERATION;
392    } else if (err == OK && isOffloaded != NULL) {
393        int32_t offload;
394        CHECK(response->findInt32("offload", &offload));
395        *isOffloaded = (offload != 0);
396    }
397    return err;
398}
399
400void NuPlayer::Renderer::closeAudioSink() {
401    sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
402
403    sp<AMessage> response;
404    msg->postAndAwaitResponse(&response);
405}
406
407void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
408    switch (msg->what()) {
409        case kWhatOpenAudioSink:
410        {
411            sp<AMessage> format;
412            CHECK(msg->findMessage("format", &format));
413
414            int32_t offloadOnly;
415            CHECK(msg->findInt32("offload-only", &offloadOnly));
416
417            int32_t hasVideo;
418            CHECK(msg->findInt32("has-video", &hasVideo));
419
420            uint32_t flags;
421            CHECK(msg->findInt32("flags", (int32_t *)&flags));
422
423            status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
424
425            sp<AMessage> response = new AMessage;
426            response->setInt32("err", err);
427            response->setInt32("offload", offloadingAudio());
428
429            sp<AReplyToken> replyID;
430            CHECK(msg->senderAwaitsResponse(&replyID));
431            response->postReply(replyID);
432
433            break;
434        }
435
436        case kWhatCloseAudioSink:
437        {
438            sp<AReplyToken> replyID;
439            CHECK(msg->senderAwaitsResponse(&replyID));
440
441            onCloseAudioSink();
442
443            sp<AMessage> response = new AMessage;
444            response->postReply(replyID);
445            break;
446        }
447
448        case kWhatStopAudioSink:
449        {
450            mAudioSink->stop();
451            break;
452        }
453
454        case kWhatDrainAudioQueue:
455        {
456            mDrainAudioQueuePending = false;
457
458            int32_t generation;
459            CHECK(msg->findInt32("drainGeneration", &generation));
460            if (generation != getDrainGeneration(true /* audio */)) {
461                break;
462            }
463
464            if (onDrainAudioQueue()) {
465                uint32_t numFramesPlayed;
466                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
467                         (status_t)OK);
468
469                uint32_t numFramesPendingPlayout =
470                    mNumFramesWritten - numFramesPlayed;
471
472                // This is how long the audio sink will have data to
473                // play back.
474                int64_t delayUs =
475                    mAudioSink->msecsPerFrame()
476                        * numFramesPendingPlayout * 1000ll;
477                if (mPlaybackRate > 1.0f) {
478                    delayUs /= mPlaybackRate;
479                }
480
481                // Let's give it more data after about half that time
482                // has elapsed.
483                Mutex::Autolock autoLock(mLock);
484                postDrainAudioQueue_l(delayUs / 2);
485            }
486            break;
487        }
488
489        case kWhatDrainVideoQueue:
490        {
491            int32_t generation;
492            CHECK(msg->findInt32("drainGeneration", &generation));
493            if (generation != getDrainGeneration(false /* audio */)) {
494                break;
495            }
496
497            mDrainVideoQueuePending = false;
498
499            onDrainVideoQueue();
500
501            postDrainVideoQueue();
502            break;
503        }
504
505        case kWhatPostDrainVideoQueue:
506        {
507            int32_t generation;
508            CHECK(msg->findInt32("drainGeneration", &generation));
509            if (generation != getDrainGeneration(false /* audio */)) {
510                break;
511            }
512
513            mDrainVideoQueuePending = false;
514            postDrainVideoQueue();
515            break;
516        }
517
518        case kWhatQueueBuffer:
519        {
520            onQueueBuffer(msg);
521            break;
522        }
523
524        case kWhatQueueEOS:
525        {
526            onQueueEOS(msg);
527            break;
528        }
529
530        case kWhatEOS:
531        {
532            int32_t generation;
533            CHECK(msg->findInt32("audioEOSGeneration", &generation));
534            if (generation != mAudioEOSGeneration) {
535                break;
536            }
537            status_t finalResult;
538            CHECK(msg->findInt32("finalResult", &finalResult));
539            notifyEOS(true /* audio */, finalResult);
540            break;
541        }
542
543        case kWhatConfigPlayback:
544        {
545            sp<AReplyToken> replyID;
546            CHECK(msg->senderAwaitsResponse(&replyID));
547            AudioPlaybackRate rate;
548            readFromAMessage(msg, &rate);
549            status_t err = onConfigPlayback(rate);
550            sp<AMessage> response = new AMessage;
551            response->setInt32("err", err);
552            response->postReply(replyID);
553            break;
554        }
555
556        case kWhatGetPlaybackSettings:
557        {
558            sp<AReplyToken> replyID;
559            CHECK(msg->senderAwaitsResponse(&replyID));
560            AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT;
561            status_t err = onGetPlaybackSettings(&rate);
562            sp<AMessage> response = new AMessage;
563            if (err == OK) {
564                writeToAMessage(response, rate);
565            }
566            response->setInt32("err", err);
567            response->postReply(replyID);
568            break;
569        }
570
571        case kWhatConfigSync:
572        {
573            sp<AReplyToken> replyID;
574            CHECK(msg->senderAwaitsResponse(&replyID));
575            AVSyncSettings sync;
576            float videoFpsHint;
577            readFromAMessage(msg, &sync, &videoFpsHint);
578            status_t err = onConfigSync(sync, videoFpsHint);
579            sp<AMessage> response = new AMessage;
580            response->setInt32("err", err);
581            response->postReply(replyID);
582            break;
583        }
584
585        case kWhatGetSyncSettings:
586        {
587            sp<AReplyToken> replyID;
588            CHECK(msg->senderAwaitsResponse(&replyID));
589
590            ALOGV("kWhatGetSyncSettings");
591            AVSyncSettings sync;
592            float videoFps = -1.f;
593            status_t err = onGetSyncSettings(&sync, &videoFps);
594            sp<AMessage> response = new AMessage;
595            if (err == OK) {
596                writeToAMessage(response, sync, videoFps);
597            }
598            response->setInt32("err", err);
599            response->postReply(replyID);
600            break;
601        }
602
603        case kWhatFlush:
604        {
605            onFlush(msg);
606            break;
607        }
608
609        case kWhatDisableOffloadAudio:
610        {
611            onDisableOffloadAudio();
612            break;
613        }
614
615        case kWhatEnableOffloadAudio:
616        {
617            onEnableOffloadAudio();
618            break;
619        }
620
621        case kWhatPause:
622        {
623            onPause();
624            break;
625        }
626
627        case kWhatResume:
628        {
629            onResume();
630            break;
631        }
632
633        case kWhatSetVideoFrameRate:
634        {
635            float fps;
636            CHECK(msg->findFloat("frame-rate", &fps));
637            onSetVideoFrameRate(fps);
638            break;
639        }
640
641        case kWhatAudioTearDown:
642        {
643            onAudioTearDown(kDueToError);
644            break;
645        }
646
647        case kWhatAudioOffloadPauseTimeout:
648        {
649            int32_t generation;
650            CHECK(msg->findInt32("drainGeneration", &generation));
651            if (generation != mAudioOffloadPauseTimeoutGeneration) {
652                break;
653            }
654            ALOGV("Audio Offload tear down due to pause timeout.");
655            onAudioTearDown(kDueToTimeout);
656            mWakeLock->release();
657            break;
658        }
659
660        default:
661            TRESPASS();
662            break;
663    }
664}
665
666void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
667    if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) {
668        return;
669    }
670
671    if (mAudioQueue.empty()) {
672        return;
673    }
674
675    // FIXME: if paused, wait until AudioTrack stop() is complete before delivering data.
676    if (mPaused) {
677        const int64_t diffUs = mPauseDrainAudioAllowedUs - ALooper::GetNowUs();
678        if (diffUs > delayUs) {
679            delayUs = diffUs;
680        }
681    }
682
683    mDrainAudioQueuePending = true;
684    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
685    msg->setInt32("drainGeneration", mAudioDrainGeneration);
686    msg->post(delayUs);
687}
688
689void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
690    mAudioRenderingStartGeneration = mAudioDrainGeneration;
691    mVideoRenderingStartGeneration = mVideoDrainGeneration;
692    mRenderingDataDelivered = false;
693}
694
695void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
696    if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
697        mAudioRenderingStartGeneration == mAudioDrainGeneration) {
698        mRenderingDataDelivered = true;
699        if (mPaused) {
700            return;
701        }
702        mVideoRenderingStartGeneration = -1;
703        mAudioRenderingStartGeneration = -1;
704
705        sp<AMessage> notify = mNotify->dup();
706        notify->setInt32("what", kWhatMediaRenderingStart);
707        notify->post();
708    }
709}
710
711// static
712size_t NuPlayer::Renderer::AudioSinkCallback(
713        MediaPlayerBase::AudioSink * /* audioSink */,
714        void *buffer,
715        size_t size,
716        void *cookie,
717        MediaPlayerBase::AudioSink::cb_event_t event) {
718    NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
719
720    switch (event) {
721        case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
722        {
723            return me->fillAudioBuffer(buffer, size);
724            break;
725        }
726
727        case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
728        {
729            ALOGV("AudioSink::CB_EVENT_STREAM_END");
730            me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
731            break;
732        }
733
734        case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
735        {
736            ALOGV("AudioSink::CB_EVENT_TEAR_DOWN");
737            me->notifyAudioTearDown();
738            break;
739        }
740    }
741
742    return 0;
743}
744
745size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
746    Mutex::Autolock autoLock(mLock);
747
748    if (!mUseAudioCallback) {
749        return 0;
750    }
751
752    bool hasEOS = false;
753
754    size_t sizeCopied = 0;
755    bool firstEntry = true;
756    QueueEntry *entry;  // will be valid after while loop if hasEOS is set.
757    while (sizeCopied < size && !mAudioQueue.empty()) {
758        entry = &*mAudioQueue.begin();
759
760        if (entry->mBuffer == NULL) { // EOS
761            hasEOS = true;
762            mAudioQueue.erase(mAudioQueue.begin());
763            break;
764        }
765
766        if (firstEntry && entry->mOffset == 0) {
767            firstEntry = false;
768            int64_t mediaTimeUs;
769            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
770            ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
771            setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
772        }
773
774        size_t copy = entry->mBuffer->size() - entry->mOffset;
775        size_t sizeRemaining = size - sizeCopied;
776        if (copy > sizeRemaining) {
777            copy = sizeRemaining;
778        }
779
780        memcpy((char *)buffer + sizeCopied,
781               entry->mBuffer->data() + entry->mOffset,
782               copy);
783
784        entry->mOffset += copy;
785        if (entry->mOffset == entry->mBuffer->size()) {
786            entry->mNotifyConsumed->post();
787            mAudioQueue.erase(mAudioQueue.begin());
788            entry = NULL;
789        }
790        sizeCopied += copy;
791
792        notifyIfMediaRenderingStarted_l();
793    }
794
795    if (mAudioFirstAnchorTimeMediaUs >= 0) {
796        int64_t nowUs = ALooper::GetNowUs();
797        int64_t nowMediaUs =
798            mAudioFirstAnchorTimeMediaUs + mAudioSink->getPlayedOutDurationUs(nowUs);
799        // we don't know how much data we are queueing for offloaded tracks.
800        mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
801    }
802
803    // for non-offloaded audio, we need to compute the frames written because
804    // there is no EVENT_STREAM_END notification. The frames written gives
805    // an estimate on the pending played out duration.
806    if (!offloadingAudio()) {
807        mNumFramesWritten += sizeCopied / mAudioSink->frameSize();
808    }
809
810    if (hasEOS) {
811        (new AMessage(kWhatStopAudioSink, this))->post();
812        // As there is currently no EVENT_STREAM_END callback notification for
813        // non-offloaded audio tracks, we need to post the EOS ourselves.
814        if (!offloadingAudio()) {
815            int64_t postEOSDelayUs = 0;
816            if (mAudioSink->needsTrailingPadding()) {
817                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
818            }
819            ALOGV("fillAudioBuffer: notifyEOS "
820                    "mNumFramesWritten:%u  finalResult:%d  postEOSDelay:%lld",
821                    mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs);
822            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
823        }
824    }
825    return sizeCopied;
826}
827
828void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() {
829    List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it;
830    bool foundEOS = false;
831    while (it != mAudioQueue.end()) {
832        int32_t eos;
833        QueueEntry *entry = &*it++;
834        if (entry->mBuffer == NULL
835                || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) {
836            itEOS = it;
837            foundEOS = true;
838        }
839    }
840
841    if (foundEOS) {
842        // post all replies before EOS and drop the samples
843        for (it = mAudioQueue.begin(); it != itEOS; it++) {
844            if (it->mBuffer == NULL) {
845                // delay doesn't matter as we don't even have an AudioTrack
846                notifyEOS(true /* audio */, it->mFinalResult);
847            } else {
848                it->mNotifyConsumed->post();
849            }
850        }
851        mAudioQueue.erase(mAudioQueue.begin(), itEOS);
852    }
853}
854
855bool NuPlayer::Renderer::onDrainAudioQueue() {
856    // do not drain audio during teardown as queued buffers may be invalid.
857    if (mAudioTornDown) {
858        return false;
859    }
860    // TODO: This call to getPosition checks if AudioTrack has been created
861    // in AudioSink before draining audio. If AudioTrack doesn't exist, then
862    // CHECKs on getPosition will fail.
863    // We still need to figure out why AudioTrack is not created when
864    // this function is called. One possible reason could be leftover
865    // audio. Another possible place is to check whether decoder
866    // has received INFO_FORMAT_CHANGED as the first buffer since
867    // AudioSink is opened there, and possible interactions with flush
868    // immediately after start. Investigate error message
869    // "vorbis_dsp_synthesis returned -135", along with RTSP.
870    uint32_t numFramesPlayed;
871    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
872        // When getPosition fails, renderer will not reschedule the draining
873        // unless new samples are queued.
874        // If we have pending EOS (or "eos" marker for discontinuities), we need
875        // to post these now as NuPlayerDecoder might be waiting for it.
876        drainAudioQueueUntilLastEOS();
877
878        ALOGW("onDrainAudioQueue(): audio sink is not ready");
879        return false;
880    }
881
882#if 0
883    ssize_t numFramesAvailableToWrite =
884        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
885
886    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
887        ALOGI("audio sink underrun");
888    } else {
889        ALOGV("audio queue has %d frames left to play",
890             mAudioSink->frameCount() - numFramesAvailableToWrite);
891    }
892#endif
893
894    uint32_t prevFramesWritten = mNumFramesWritten;
895    while (!mAudioQueue.empty()) {
896        QueueEntry *entry = &*mAudioQueue.begin();
897
898        mLastAudioBufferDrained = entry->mBufferOrdinal;
899
900        if (entry->mBuffer == NULL) {
901            // EOS
902            int64_t postEOSDelayUs = 0;
903            if (mAudioSink->needsTrailingPadding()) {
904                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
905            }
906            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
907            mLastAudioMediaTimeUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
908
909            mAudioQueue.erase(mAudioQueue.begin());
910            entry = NULL;
911            if (mAudioSink->needsTrailingPadding()) {
912                // If we're not in gapless playback (i.e. through setNextPlayer), we
913                // need to stop the track here, because that will play out the last
914                // little bit at the end of the file. Otherwise short files won't play.
915                mAudioSink->stop();
916                mNumFramesWritten = 0;
917            }
918            return false;
919        }
920
921        // ignore 0-sized buffer which could be EOS marker with no data
922        if (entry->mOffset == 0 && entry->mBuffer->size() > 0) {
923            int64_t mediaTimeUs;
924            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
925            ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs",
926                    mediaTimeUs / 1E6);
927            onNewAudioMediaTime(mediaTimeUs);
928        }
929
930        size_t copy = entry->mBuffer->size() - entry->mOffset;
931
932        ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
933                                            copy, false /* blocking */);
934        if (written < 0) {
935            // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
936            if (written == WOULD_BLOCK) {
937                ALOGV("AudioSink write would block when writing %zu bytes", copy);
938            } else {
939                ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
940                // This can only happen when AudioSink was opened with doNotReconnect flag set to
941                // true, in which case the NuPlayer will handle the reconnect.
942                notifyAudioTearDown();
943            }
944            break;
945        }
946
947        entry->mOffset += written;
948        if (entry->mOffset == entry->mBuffer->size()) {
949            entry->mNotifyConsumed->post();
950            mAudioQueue.erase(mAudioQueue.begin());
951
952            entry = NULL;
953        }
954
955        size_t copiedFrames = written / mAudioSink->frameSize();
956        mNumFramesWritten += copiedFrames;
957
958        {
959            Mutex::Autolock autoLock(mLock);
960            int64_t maxTimeMedia;
961            maxTimeMedia =
962                mAnchorTimeMediaUs +
963                        (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
964                                * 1000LL * mAudioSink->msecsPerFrame());
965            mMediaClock->updateMaxTimeMedia(maxTimeMedia);
966
967            notifyIfMediaRenderingStarted_l();
968        }
969
970        if (written != (ssize_t)copy) {
971            // A short count was received from AudioSink::write()
972            //
973            // AudioSink write is called in non-blocking mode.
974            // It may return with a short count when:
975            //
976            // 1) Size to be copied is not a multiple of the frame size. We consider this fatal.
977            // 2) The data to be copied exceeds the available buffer in AudioSink.
978            // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
979            // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
980
981            // (Case 1)
982            // Must be a multiple of the frame size.  If it is not a multiple of a frame size, it
983            // needs to fail, as we should not carry over fractional frames between calls.
984            CHECK_EQ(copy % mAudioSink->frameSize(), 0);
985
986            // (Case 2, 3, 4)
987            // Return early to the caller.
988            // Beware of calling immediately again as this may busy-loop if you are not careful.
989            ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
990            break;
991        }
992    }
993
994    // calculate whether we need to reschedule another write.
995    bool reschedule = !mAudioQueue.empty()
996            && (!mPaused
997                || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers
998    //ALOGD("reschedule:%d  empty:%d  mPaused:%d  prevFramesWritten:%u  mNumFramesWritten:%u",
999    //        reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten);
1000    return reschedule;
1001}
1002
1003int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
1004    int32_t sampleRate = offloadingAudio() ?
1005            mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
1006    if (sampleRate == 0) {
1007        ALOGE("sampleRate is 0 in %s mode", offloadingAudio() ? "offload" : "non-offload");
1008        return 0;
1009    }
1010    // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
1011    return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate);
1012}
1013
1014// Calculate duration of pending samples if played at normal rate (i.e., 1.0).
1015int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
1016    int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
1017    return writtenAudioDurationUs - mAudioSink->getPlayedOutDurationUs(nowUs);
1018}
1019
1020int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
1021    int64_t realUs;
1022    if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
1023        // If failed to get current position, e.g. due to audio clock is
1024        // not ready, then just play out video immediately without delay.
1025        return nowUs;
1026    }
1027    return realUs;
1028}
1029
1030void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
1031    Mutex::Autolock autoLock(mLock);
1032    // TRICKY: vorbis decoder generates multiple frames with the same
1033    // timestamp, so only update on the first frame with a given timestamp
1034    if (mediaTimeUs == mAnchorTimeMediaUs) {
1035        return;
1036    }
1037    setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
1038
1039    AudioTimestamp ts;
1040    status_t res = mAudioSink->getTimestamp(ts);
1041    if (res == OK) {
1042        int64_t nowUs = ALooper::GetNowUs();
1043        int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
1044        mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
1045    }
1046    mAnchorNumFramesWritten = mNumFramesWritten;
1047    mAnchorTimeMediaUs = mediaTimeUs;
1048}
1049
1050// Called without mLock acquired.
1051void NuPlayer::Renderer::postDrainVideoQueue() {
1052    if (mDrainVideoQueuePending
1053            || getSyncQueues()
1054            || (mPaused && mVideoSampleReceived)) {
1055        return;
1056    }
1057
1058    if (mVideoQueue.empty()) {
1059        return;
1060    }
1061
1062    QueueEntry &entry = *mVideoQueue.begin();
1063
1064    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
1065    msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
1066
1067    if (entry.mBuffer == NULL) {
1068        // EOS doesn't carry a timestamp.
1069        msg->post();
1070        mDrainVideoQueuePending = true;
1071        return;
1072    }
1073
1074    bool needRepostDrainVideoQueue = false;
1075    int64_t delayUs;
1076    int64_t nowUs = ALooper::GetNowUs();
1077    int64_t realTimeUs;
1078    if (mFlags & FLAG_REAL_TIME) {
1079        int64_t mediaTimeUs;
1080        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1081        realTimeUs = mediaTimeUs;
1082    } else {
1083        int64_t mediaTimeUs;
1084        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1085
1086        {
1087            Mutex::Autolock autoLock(mLock);
1088            if (mAnchorTimeMediaUs < 0) {
1089                mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
1090                mAnchorTimeMediaUs = mediaTimeUs;
1091                realTimeUs = nowUs;
1092            } else if (!mVideoSampleReceived) {
1093                // Always render the first video frame.
1094                realTimeUs = nowUs;
1095            } else if (mAudioFirstAnchorTimeMediaUs < 0
1096                || mMediaClock->getRealTimeFor(mediaTimeUs, &realTimeUs) == OK) {
1097                realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1098            } else if (mediaTimeUs - mAudioFirstAnchorTimeMediaUs >= 0) {
1099                needRepostDrainVideoQueue = true;
1100                realTimeUs = nowUs;
1101            } else {
1102                realTimeUs = nowUs;
1103            }
1104        }
1105        if (!mHasAudio) {
1106            // smooth out videos >= 10fps
1107            mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
1108        }
1109
1110        // Heuristics to handle situation when media time changed without a
1111        // discontinuity. If we have not drained an audio buffer that was
1112        // received after this buffer, repost in 10 msec. Otherwise repost
1113        // in 500 msec.
1114        delayUs = realTimeUs - nowUs;
1115        int64_t postDelayUs = -1;
1116        if (delayUs > 500000) {
1117            postDelayUs = 500000;
1118            if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) {
1119                postDelayUs = 10000;
1120            }
1121        } else if (needRepostDrainVideoQueue) {
1122            // CHECK(mPlaybackRate > 0);
1123            // CHECK(mAudioFirstAnchorTimeMediaUs >= 0);
1124            // CHECK(mediaTimeUs - mAudioFirstAnchorTimeMediaUs >= 0);
1125            postDelayUs = mediaTimeUs - mAudioFirstAnchorTimeMediaUs;
1126            postDelayUs /= mPlaybackRate;
1127        }
1128
1129        if (postDelayUs >= 0) {
1130            msg->setWhat(kWhatPostDrainVideoQueue);
1131            msg->post(postDelayUs);
1132            mVideoScheduler->restart();
1133            ALOGI("possible video time jump of %dms or uninitialized media clock, retrying in %dms",
1134                    (int)(delayUs / 1000), (int)(postDelayUs / 1000));
1135            mDrainVideoQueuePending = true;
1136            return;
1137        }
1138    }
1139
1140    realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
1141    int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
1142
1143    delayUs = realTimeUs - nowUs;
1144
1145    ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
1146    // post 2 display refreshes before rendering is due
1147    msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
1148
1149    mDrainVideoQueuePending = true;
1150}
1151
1152void NuPlayer::Renderer::onDrainVideoQueue() {
1153    if (mVideoQueue.empty()) {
1154        return;
1155    }
1156
1157    QueueEntry *entry = &*mVideoQueue.begin();
1158
1159    if (entry->mBuffer == NULL) {
1160        // EOS
1161
1162        notifyEOS(false /* audio */, entry->mFinalResult);
1163
1164        mVideoQueue.erase(mVideoQueue.begin());
1165        entry = NULL;
1166
1167        setVideoLateByUs(0);
1168        return;
1169    }
1170
1171    int64_t nowUs = ALooper::GetNowUs();
1172    int64_t realTimeUs;
1173    int64_t mediaTimeUs = -1;
1174    if (mFlags & FLAG_REAL_TIME) {
1175        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
1176    } else {
1177        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1178
1179        realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1180    }
1181
1182    bool tooLate = false;
1183
1184    if (!mPaused) {
1185        setVideoLateByUs(nowUs - realTimeUs);
1186        tooLate = (mVideoLateByUs > 40000);
1187
1188        if (tooLate) {
1189            ALOGV("video late by %lld us (%.2f secs)",
1190                 (long long)mVideoLateByUs, mVideoLateByUs / 1E6);
1191        } else {
1192            int64_t mediaUs = 0;
1193            mMediaClock->getMediaTime(realTimeUs, &mediaUs);
1194            ALOGV("rendering video at media time %.2f secs",
1195                    (mFlags & FLAG_REAL_TIME ? realTimeUs :
1196                    mediaUs) / 1E6);
1197
1198            if (!(mFlags & FLAG_REAL_TIME)
1199                    && mLastAudioMediaTimeUs != -1
1200                    && mediaTimeUs > mLastAudioMediaTimeUs) {
1201                // If audio ends before video, video continues to drive media clock.
1202                // Also smooth out videos >= 10fps.
1203                mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
1204            }
1205        }
1206    } else {
1207        setVideoLateByUs(0);
1208        if (!mVideoSampleReceived && !mHasAudio) {
1209            // This will ensure that the first frame after a flush won't be used as anchor
1210            // when renderer is in paused state, because resume can happen any time after seek.
1211            Mutex::Autolock autoLock(mLock);
1212            clearAnchorTime_l();
1213        }
1214    }
1215
1216    // Always render the first video frame while keeping stats on A/V sync.
1217    if (!mVideoSampleReceived) {
1218        realTimeUs = nowUs;
1219        tooLate = false;
1220    }
1221
1222    entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
1223    entry->mNotifyConsumed->setInt32("render", !tooLate);
1224    entry->mNotifyConsumed->post();
1225    mVideoQueue.erase(mVideoQueue.begin());
1226    entry = NULL;
1227
1228    mVideoSampleReceived = true;
1229
1230    if (!mPaused) {
1231        if (!mVideoRenderingStarted) {
1232            mVideoRenderingStarted = true;
1233            notifyVideoRenderingStart();
1234        }
1235        Mutex::Autolock autoLock(mLock);
1236        notifyIfMediaRenderingStarted_l();
1237    }
1238}
1239
1240void NuPlayer::Renderer::notifyVideoRenderingStart() {
1241    sp<AMessage> notify = mNotify->dup();
1242    notify->setInt32("what", kWhatVideoRenderingStart);
1243    notify->post();
1244}
1245
1246void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
1247    if (audio && delayUs > 0) {
1248        sp<AMessage> msg = new AMessage(kWhatEOS, this);
1249        msg->setInt32("audioEOSGeneration", mAudioEOSGeneration);
1250        msg->setInt32("finalResult", finalResult);
1251        msg->post(delayUs);
1252        return;
1253    }
1254    sp<AMessage> notify = mNotify->dup();
1255    notify->setInt32("what", kWhatEOS);
1256    notify->setInt32("audio", static_cast<int32_t>(audio));
1257    notify->setInt32("finalResult", finalResult);
1258    notify->post(delayUs);
1259}
1260
1261void NuPlayer::Renderer::notifyAudioTearDown() {
1262    (new AMessage(kWhatAudioTearDown, this))->post();
1263}
1264
1265void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
1266    int32_t audio;
1267    CHECK(msg->findInt32("audio", &audio));
1268
1269    if (dropBufferIfStale(audio, msg)) {
1270        return;
1271    }
1272
1273    if (audio) {
1274        mHasAudio = true;
1275    } else {
1276        mHasVideo = true;
1277    }
1278
1279    if (mHasVideo) {
1280        if (mVideoScheduler == NULL) {
1281            mVideoScheduler = new VideoFrameScheduler();
1282            mVideoScheduler->init();
1283        }
1284    }
1285
1286    sp<ABuffer> buffer;
1287    CHECK(msg->findBuffer("buffer", &buffer));
1288
1289    sp<AMessage> notifyConsumed;
1290    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
1291
1292    QueueEntry entry;
1293    entry.mBuffer = buffer;
1294    entry.mNotifyConsumed = notifyConsumed;
1295    entry.mOffset = 0;
1296    entry.mFinalResult = OK;
1297    entry.mBufferOrdinal = ++mTotalBuffersQueued;
1298
1299    if (audio) {
1300        Mutex::Autolock autoLock(mLock);
1301        mAudioQueue.push_back(entry);
1302        postDrainAudioQueue_l();
1303    } else {
1304        mVideoQueue.push_back(entry);
1305        postDrainVideoQueue();
1306    }
1307
1308    Mutex::Autolock autoLock(mLock);
1309    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
1310        return;
1311    }
1312
1313    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
1314    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
1315
1316    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
1317        // EOS signalled on either queue.
1318        syncQueuesDone_l();
1319        return;
1320    }
1321
1322    int64_t firstAudioTimeUs;
1323    int64_t firstVideoTimeUs;
1324    CHECK(firstAudioBuffer->meta()
1325            ->findInt64("timeUs", &firstAudioTimeUs));
1326    CHECK(firstVideoBuffer->meta()
1327            ->findInt64("timeUs", &firstVideoTimeUs));
1328
1329    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
1330
1331    ALOGV("queueDiff = %.2f secs", diff / 1E6);
1332
1333    if (diff > 100000ll) {
1334        // Audio data starts More than 0.1 secs before video.
1335        // Drop some audio.
1336
1337        (*mAudioQueue.begin()).mNotifyConsumed->post();
1338        mAudioQueue.erase(mAudioQueue.begin());
1339        return;
1340    }
1341
1342    syncQueuesDone_l();
1343}
1344
1345void NuPlayer::Renderer::syncQueuesDone_l() {
1346    if (!mSyncQueues) {
1347        return;
1348    }
1349
1350    mSyncQueues = false;
1351
1352    if (!mAudioQueue.empty()) {
1353        postDrainAudioQueue_l();
1354    }
1355
1356    if (!mVideoQueue.empty()) {
1357        mLock.unlock();
1358        postDrainVideoQueue();
1359        mLock.lock();
1360    }
1361}
1362
1363void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1364    int32_t audio;
1365    CHECK(msg->findInt32("audio", &audio));
1366
1367    if (dropBufferIfStale(audio, msg)) {
1368        return;
1369    }
1370
1371    int32_t finalResult;
1372    CHECK(msg->findInt32("finalResult", &finalResult));
1373
1374    QueueEntry entry;
1375    entry.mOffset = 0;
1376    entry.mFinalResult = finalResult;
1377
1378    if (audio) {
1379        Mutex::Autolock autoLock(mLock);
1380        if (mAudioQueue.empty() && mSyncQueues) {
1381            syncQueuesDone_l();
1382        }
1383        mAudioQueue.push_back(entry);
1384        postDrainAudioQueue_l();
1385    } else {
1386        if (mVideoQueue.empty() && getSyncQueues()) {
1387            Mutex::Autolock autoLock(mLock);
1388            syncQueuesDone_l();
1389        }
1390        mVideoQueue.push_back(entry);
1391        postDrainVideoQueue();
1392    }
1393}
1394
1395void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
1396    int32_t audio, notifyComplete;
1397    CHECK(msg->findInt32("audio", &audio));
1398
1399    {
1400        Mutex::Autolock autoLock(mLock);
1401        if (audio) {
1402            notifyComplete = mNotifyCompleteAudio;
1403            mNotifyCompleteAudio = false;
1404            mLastAudioMediaTimeUs = -1;
1405        } else {
1406            notifyComplete = mNotifyCompleteVideo;
1407            mNotifyCompleteVideo = false;
1408        }
1409
1410        // If we're currently syncing the queues, i.e. dropping audio while
1411        // aligning the first audio/video buffer times and only one of the
1412        // two queues has data, we may starve that queue by not requesting
1413        // more buffers from the decoder. If the other source then encounters
1414        // a discontinuity that leads to flushing, we'll never find the
1415        // corresponding discontinuity on the other queue.
1416        // Therefore we'll stop syncing the queues if at least one of them
1417        // is flushed.
1418        syncQueuesDone_l();
1419        clearAnchorTime_l();
1420    }
1421
1422    ALOGV("flushing %s", audio ? "audio" : "video");
1423    if (audio) {
1424        {
1425            Mutex::Autolock autoLock(mLock);
1426            flushQueue(&mAudioQueue);
1427
1428            ++mAudioDrainGeneration;
1429            ++mAudioEOSGeneration;
1430            prepareForMediaRenderingStart_l();
1431
1432            // the frame count will be reset after flush.
1433            clearAudioFirstAnchorTime_l();
1434        }
1435
1436        mDrainAudioQueuePending = false;
1437
1438        if (offloadingAudio()) {
1439            mAudioSink->pause();
1440            mAudioSink->flush();
1441            if (!mPaused) {
1442                mAudioSink->start();
1443            }
1444        } else {
1445            mAudioSink->pause();
1446            mAudioSink->flush();
1447            // Call stop() to signal to the AudioSink to completely fill the
1448            // internal buffer before resuming playback.
1449            // FIXME: this is ignored after flush().
1450            mAudioSink->stop();
1451            if (mPaused) {
1452                // Race condition: if renderer is paused and audio sink is stopped,
1453                // we need to make sure that the audio track buffer fully drains
1454                // before delivering data.
1455                // FIXME: remove this if we can detect if stop() is complete.
1456                const int delayUs = 2 * 50 * 1000; // (2 full mixer thread cycles at 50ms)
1457                mPauseDrainAudioAllowedUs = ALooper::GetNowUs() + delayUs;
1458            } else {
1459                mAudioSink->start();
1460            }
1461            mNumFramesWritten = 0;
1462        }
1463    } else {
1464        flushQueue(&mVideoQueue);
1465
1466        mDrainVideoQueuePending = false;
1467
1468        if (mVideoScheduler != NULL) {
1469            mVideoScheduler->restart();
1470        }
1471
1472        Mutex::Autolock autoLock(mLock);
1473        ++mVideoDrainGeneration;
1474        prepareForMediaRenderingStart_l();
1475    }
1476
1477    mVideoSampleReceived = false;
1478
1479    if (notifyComplete) {
1480        notifyFlushComplete(audio);
1481    }
1482}
1483
1484void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1485    while (!queue->empty()) {
1486        QueueEntry *entry = &*queue->begin();
1487
1488        if (entry->mBuffer != NULL) {
1489            entry->mNotifyConsumed->post();
1490        }
1491
1492        queue->erase(queue->begin());
1493        entry = NULL;
1494    }
1495}
1496
1497void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1498    sp<AMessage> notify = mNotify->dup();
1499    notify->setInt32("what", kWhatFlushComplete);
1500    notify->setInt32("audio", static_cast<int32_t>(audio));
1501    notify->post();
1502}
1503
1504bool NuPlayer::Renderer::dropBufferIfStale(
1505        bool audio, const sp<AMessage> &msg) {
1506    int32_t queueGeneration;
1507    CHECK(msg->findInt32("queueGeneration", &queueGeneration));
1508
1509    if (queueGeneration == getQueueGeneration(audio)) {
1510        return false;
1511    }
1512
1513    sp<AMessage> notifyConsumed;
1514    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1515        notifyConsumed->post();
1516    }
1517
1518    return true;
1519}
1520
1521void NuPlayer::Renderer::onAudioSinkChanged() {
1522    if (offloadingAudio()) {
1523        return;
1524    }
1525    CHECK(!mDrainAudioQueuePending);
1526    mNumFramesWritten = 0;
1527    {
1528        Mutex::Autolock autoLock(mLock);
1529        mAnchorNumFramesWritten = -1;
1530    }
1531    uint32_t written;
1532    if (mAudioSink->getFramesWritten(&written) == OK) {
1533        mNumFramesWritten = written;
1534    }
1535}
1536
1537void NuPlayer::Renderer::onDisableOffloadAudio() {
1538    Mutex::Autolock autoLock(mLock);
1539    mFlags &= ~FLAG_OFFLOAD_AUDIO;
1540    ++mAudioDrainGeneration;
1541    if (mAudioRenderingStartGeneration != -1) {
1542        prepareForMediaRenderingStart_l();
1543    }
1544}
1545
1546void NuPlayer::Renderer::onEnableOffloadAudio() {
1547    Mutex::Autolock autoLock(mLock);
1548    mFlags |= FLAG_OFFLOAD_AUDIO;
1549    ++mAudioDrainGeneration;
1550    if (mAudioRenderingStartGeneration != -1) {
1551        prepareForMediaRenderingStart_l();
1552    }
1553}
1554
1555void NuPlayer::Renderer::onPause() {
1556    if (mPaused) {
1557        return;
1558    }
1559
1560    {
1561        Mutex::Autolock autoLock(mLock);
1562        // we do not increment audio drain generation so that we fill audio buffer during pause.
1563        ++mVideoDrainGeneration;
1564        prepareForMediaRenderingStart_l();
1565        mPaused = true;
1566        mMediaClock->setPlaybackRate(0.0);
1567    }
1568
1569    mDrainAudioQueuePending = false;
1570    mDrainVideoQueuePending = false;
1571
1572    // Note: audio data may not have been decoded, and the AudioSink may not be opened.
1573    mAudioSink->pause();
1574    startAudioOffloadPauseTimeout();
1575
1576    ALOGV("now paused audio queue has %zu entries, video has %zu entries",
1577          mAudioQueue.size(), mVideoQueue.size());
1578}
1579
1580void NuPlayer::Renderer::onResume() {
1581    if (!mPaused) {
1582        return;
1583    }
1584
1585    // Note: audio data may not have been decoded, and the AudioSink may not be opened.
1586    cancelAudioOffloadPauseTimeout();
1587    if (mAudioSink->ready()) {
1588        status_t err = mAudioSink->start();
1589        if (err != OK) {
1590            ALOGE("cannot start AudioSink err %d", err);
1591            notifyAudioTearDown();
1592        }
1593    }
1594
1595    {
1596        Mutex::Autolock autoLock(mLock);
1597        mPaused = false;
1598        // rendering started message may have been delayed if we were paused.
1599        if (mRenderingDataDelivered) {
1600            notifyIfMediaRenderingStarted_l();
1601        }
1602        // configure audiosink as we did not do it when pausing
1603        if (mAudioSink != NULL && mAudioSink->ready()) {
1604            mAudioSink->setPlaybackRate(mPlaybackSettings);
1605        }
1606
1607        mMediaClock->setPlaybackRate(mPlaybackRate);
1608
1609        if (!mAudioQueue.empty()) {
1610            postDrainAudioQueue_l();
1611        }
1612    }
1613
1614    if (!mVideoQueue.empty()) {
1615        postDrainVideoQueue();
1616    }
1617}
1618
1619void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1620    if (mVideoScheduler == NULL) {
1621        mVideoScheduler = new VideoFrameScheduler();
1622    }
1623    mVideoScheduler->init(fps);
1624}
1625
1626int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
1627    Mutex::Autolock autoLock(mLock);
1628    return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
1629}
1630
1631int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
1632    Mutex::Autolock autoLock(mLock);
1633    return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
1634}
1635
1636bool NuPlayer::Renderer::getSyncQueues() {
1637    Mutex::Autolock autoLock(mLock);
1638    return mSyncQueues;
1639}
1640
1641void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) {
1642    if (mAudioTornDown) {
1643        return;
1644    }
1645    mAudioTornDown = true;
1646
1647    int64_t currentPositionUs;
1648    sp<AMessage> notify = mNotify->dup();
1649    if (getCurrentPosition(&currentPositionUs) == OK) {
1650        notify->setInt64("positionUs", currentPositionUs);
1651    }
1652
1653    mAudioSink->stop();
1654    mAudioSink->flush();
1655
1656    notify->setInt32("what", kWhatAudioTearDown);
1657    notify->setInt32("reason", reason);
1658    notify->post();
1659}
1660
1661void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1662    if (offloadingAudio()) {
1663        mWakeLock->acquire();
1664        sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
1665        msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
1666        msg->post(kOffloadPauseMaxUs);
1667    }
1668}
1669
1670void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
1671    if (offloadingAudio()) {
1672        mWakeLock->release(true);
1673        ++mAudioOffloadPauseTimeoutGeneration;
1674    }
1675}
1676
1677status_t NuPlayer::Renderer::onOpenAudioSink(
1678        const sp<AMessage> &format,
1679        bool offloadOnly,
1680        bool hasVideo,
1681        uint32_t flags) {
1682    ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1683            offloadOnly, offloadingAudio());
1684    bool audioSinkChanged = false;
1685
1686    int32_t numChannels;
1687    CHECK(format->findInt32("channel-count", &numChannels));
1688
1689    int32_t channelMask;
1690    if (!format->findInt32("channel-mask", &channelMask)) {
1691        // signal to the AudioSink to derive the mask from count.
1692        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1693    }
1694
1695    int32_t sampleRate;
1696    CHECK(format->findInt32("sample-rate", &sampleRate));
1697
1698    if (offloadingAudio()) {
1699        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1700        AString mime;
1701        CHECK(format->findString("mime", &mime));
1702        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1703
1704        if (err != OK) {
1705            ALOGE("Couldn't map mime \"%s\" to a valid "
1706                    "audio_format", mime.c_str());
1707            onDisableOffloadAudio();
1708        } else {
1709            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1710                    mime.c_str(), audioFormat);
1711
1712            int avgBitRate = -1;
1713            format->findInt32("bit-rate", &avgBitRate);
1714
1715            int32_t aacProfile = -1;
1716            if (audioFormat == AUDIO_FORMAT_AAC
1717                    && format->findInt32("aac-profile", &aacProfile)) {
1718                // Redefine AAC format as per aac profile
1719                mapAACProfileToAudioFormat(
1720                        audioFormat,
1721                        aacProfile);
1722            }
1723
1724            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1725            offloadInfo.duration_us = -1;
1726            format->findInt64(
1727                    "durationUs", &offloadInfo.duration_us);
1728            offloadInfo.sample_rate = sampleRate;
1729            offloadInfo.channel_mask = channelMask;
1730            offloadInfo.format = audioFormat;
1731            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1732            offloadInfo.bit_rate = avgBitRate;
1733            offloadInfo.has_video = hasVideo;
1734            offloadInfo.is_streaming = true;
1735
1736            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1737                ALOGV("openAudioSink: no change in offload mode");
1738                // no change from previous configuration, everything ok.
1739                return OK;
1740            }
1741            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1742
1743            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1744            uint32_t offloadFlags = flags;
1745            offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1746            offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1747            audioSinkChanged = true;
1748            mAudioSink->close();
1749
1750            err = mAudioSink->open(
1751                    sampleRate,
1752                    numChannels,
1753                    (audio_channel_mask_t)channelMask,
1754                    audioFormat,
1755                    0 /* bufferCount - unused */,
1756                    &NuPlayer::Renderer::AudioSinkCallback,
1757                    this,
1758                    (audio_output_flags_t)offloadFlags,
1759                    &offloadInfo);
1760
1761            if (err == OK) {
1762                err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1763            }
1764
1765            if (err == OK) {
1766                // If the playback is offloaded to h/w, we pass
1767                // the HAL some metadata information.
1768                // We don't want to do this for PCM because it
1769                // will be going through the AudioFlinger mixer
1770                // before reaching the hardware.
1771                // TODO
1772                mCurrentOffloadInfo = offloadInfo;
1773                if (!mPaused) { // for preview mode, don't start if paused
1774                    err = mAudioSink->start();
1775                }
1776                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1777            }
1778            if (err != OK) {
1779                // Clean up, fall back to non offload mode.
1780                mAudioSink->close();
1781                onDisableOffloadAudio();
1782                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1783                ALOGV("openAudioSink: offload failed");
1784            } else {
1785                mUseAudioCallback = true;  // offload mode transfers data through callback
1786                ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
1787            }
1788        }
1789    }
1790    if (!offloadOnly && !offloadingAudio()) {
1791        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1792        uint32_t pcmFlags = flags;
1793        pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1794
1795        const PcmInfo info = {
1796                (audio_channel_mask_t)channelMask,
1797                (audio_output_flags_t)pcmFlags,
1798                AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat
1799                numChannels,
1800                sampleRate
1801        };
1802        if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
1803            ALOGV("openAudioSink: no change in pcm mode");
1804            // no change from previous configuration, everything ok.
1805            return OK;
1806        }
1807
1808        audioSinkChanged = true;
1809        mAudioSink->close();
1810        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1811        // Note: It is possible to set up the callback, but not use it to send audio data.
1812        // This requires a fix in AudioSink to explicitly specify the transfer mode.
1813        mUseAudioCallback = getUseAudioCallbackSetting();
1814        if (mUseAudioCallback) {
1815            ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
1816        }
1817
1818        // Compute the desired buffer size.
1819        // For callback mode, the amount of time before wakeup is about half the buffer size.
1820        const uint32_t frameCount =
1821                (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000;
1822
1823        // The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct
1824        // AudioSink. We don't want this when there's video because it will cause a video seek to
1825        // the previous I frame. But we do want this when there's only audio because it will give
1826        // NuPlayer a chance to switch from non-offload mode to offload mode.
1827        // So we only set doNotReconnect when there's no video.
1828        const bool doNotReconnect = !hasVideo;
1829        status_t err = mAudioSink->open(
1830                    sampleRate,
1831                    numChannels,
1832                    (audio_channel_mask_t)channelMask,
1833                    AUDIO_FORMAT_PCM_16_BIT,
1834                    0 /* bufferCount - unused */,
1835                    mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL,
1836                    mUseAudioCallback ? this : NULL,
1837                    (audio_output_flags_t)pcmFlags,
1838                    NULL,
1839                    doNotReconnect,
1840                    frameCount);
1841        if (err == OK) {
1842            err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1843        }
1844        if (err != OK) {
1845            ALOGW("openAudioSink: non offloaded open failed status: %d", err);
1846            mAudioSink->close();
1847            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1848            return err;
1849        }
1850        mCurrentPcmInfo = info;
1851        if (!mPaused) { // for preview mode, don't start if paused
1852            mAudioSink->start();
1853        }
1854    }
1855    if (audioSinkChanged) {
1856        onAudioSinkChanged();
1857    }
1858    mAudioTornDown = false;
1859    return OK;
1860}
1861
1862void NuPlayer::Renderer::onCloseAudioSink() {
1863    mAudioSink->close();
1864    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1865    mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1866}
1867
1868}  // namespace android
1869
1870