NuPlayerRenderer.cpp revision 3f5ff68327b1df21196f18a020aec474a0dd95fe 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22#include <cutils/properties.h>
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26#include <media/stagefright/foundation/AUtils.h>
27#include <media/stagefright/foundation/AWakeLock.h>
28#include <media/stagefright/MediaClock.h>
29#include <media/stagefright/MediaErrors.h>
30#include <media/stagefright/MetaData.h>
31#include <media/stagefright/Utils.h>
32#include <media/stagefright/VideoFrameScheduler.h>
33
34#include <inttypes.h>
35
36namespace android {
37
38/*
39 * Example of common configuration settings in shell script form
40
41   #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager
42   adb shell setprop audio.offload.disable 1
43
44   #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager
45   adb shell setprop audio.offload.video 1
46
47   #Use audio callbacks for PCM data
48   adb shell setprop media.stagefright.audio.cbk 1
49
50   #Use deep buffer for PCM data with video (it is generally enabled for audio-only)
51   adb shell setprop media.stagefright.audio.deep 1
52
53   #Set size of buffers for pcm audio sink in msec (example: 1000 msec)
54   adb shell setprop media.stagefright.audio.sink 1000
55
56 * These configurations take effect for the next track played (not the current track).
57 */
58
59static inline bool getUseAudioCallbackSetting() {
60    return property_get_bool("media.stagefright.audio.cbk", false /* default_value */);
61}
62
63static inline int32_t getAudioSinkPcmMsSetting() {
64    return property_get_int32(
65            "media.stagefright.audio.sink", 500 /* default_value */);
66}
67
68// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
69// is closed to allow the audio DSP to power down.
70static const int64_t kOffloadPauseMaxUs = 10000000ll;
71
72// static
73const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
74        AUDIO_CHANNEL_NONE,
75        AUDIO_OUTPUT_FLAG_NONE,
76        AUDIO_FORMAT_INVALID,
77        0, // mNumChannels
78        0 // mSampleRate
79};
80
81// static
82const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
83
84NuPlayer::Renderer::Renderer(
85        const sp<MediaPlayerBase::AudioSink> &sink,
86        const sp<AMessage> &notify,
87        uint32_t flags)
88    : mAudioSink(sink),
89      mNotify(notify),
90      mFlags(flags),
91      mNumFramesWritten(0),
92      mDrainAudioQueuePending(false),
93      mDrainVideoQueuePending(false),
94      mAudioQueueGeneration(0),
95      mVideoQueueGeneration(0),
96      mAudioDrainGeneration(0),
97      mVideoDrainGeneration(0),
98      mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
99      mAudioFirstAnchorTimeMediaUs(-1),
100      mAnchorTimeMediaUs(-1),
101      mAnchorNumFramesWritten(-1),
102      mVideoLateByUs(0ll),
103      mHasAudio(false),
104      mHasVideo(false),
105      mNotifyCompleteAudio(false),
106      mNotifyCompleteVideo(false),
107      mSyncQueues(false),
108      mPaused(false),
109      mVideoSampleReceived(false),
110      mVideoRenderingStarted(false),
111      mVideoRenderingStartGeneration(0),
112      mAudioRenderingStartGeneration(0),
113      mAudioOffloadPauseTimeoutGeneration(0),
114      mAudioTornDown(false),
115      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
116      mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
117      mTotalBuffersQueued(0),
118      mLastAudioBufferDrained(0),
119      mUseAudioCallback(false),
120      mWakeLock(new AWakeLock()) {
121    mMediaClock = new MediaClock;
122    mPlaybackRate = mPlaybackSettings.mSpeed;
123    mMediaClock->setPlaybackRate(mPlaybackRate);
124}
125
126NuPlayer::Renderer::~Renderer() {
127    if (offloadingAudio()) {
128        mAudioSink->stop();
129        mAudioSink->flush();
130        mAudioSink->close();
131    }
132}
133
134void NuPlayer::Renderer::queueBuffer(
135        bool audio,
136        const sp<ABuffer> &buffer,
137        const sp<AMessage> &notifyConsumed) {
138    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
139    msg->setInt32("queueGeneration", getQueueGeneration(audio));
140    msg->setInt32("audio", static_cast<int32_t>(audio));
141    msg->setBuffer("buffer", buffer);
142    msg->setMessage("notifyConsumed", notifyConsumed);
143    msg->post();
144}
145
146void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
147    CHECK_NE(finalResult, (status_t)OK);
148
149    sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
150    msg->setInt32("queueGeneration", getQueueGeneration(audio));
151    msg->setInt32("audio", static_cast<int32_t>(audio));
152    msg->setInt32("finalResult", finalResult);
153    msg->post();
154}
155
156status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) {
157    sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
158    writeToAMessage(msg, rate);
159    sp<AMessage> response;
160    status_t err = msg->postAndAwaitResponse(&response);
161    if (err == OK && response != NULL) {
162        CHECK(response->findInt32("err", &err));
163    }
164    return err;
165}
166
167status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) {
168    if (rate.mSpeed == 0.f) {
169        onPause();
170        // don't call audiosink's setPlaybackRate if pausing, as pitch does not
171        // have to correspond to the any non-0 speed (e.g old speed). Keep
172        // settings nonetheless, using the old speed, in case audiosink changes.
173        AudioPlaybackRate newRate = rate;
174        newRate.mSpeed = mPlaybackSettings.mSpeed;
175        mPlaybackSettings = newRate;
176        return OK;
177    }
178
179    if (mAudioSink != NULL && mAudioSink->ready()) {
180        status_t err = mAudioSink->setPlaybackRate(rate);
181        if (err != OK) {
182            return err;
183        }
184    }
185    mPlaybackSettings = rate;
186    mPlaybackRate = rate.mSpeed;
187    mMediaClock->setPlaybackRate(mPlaybackRate);
188    return OK;
189}
190
191status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
192    sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
193    sp<AMessage> response;
194    status_t err = msg->postAndAwaitResponse(&response);
195    if (err == OK && response != NULL) {
196        CHECK(response->findInt32("err", &err));
197        if (err == OK) {
198            readFromAMessage(response, rate);
199        }
200    }
201    return err;
202}
203
204status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
205    if (mAudioSink != NULL && mAudioSink->ready()) {
206        status_t err = mAudioSink->getPlaybackRate(rate);
207        if (err == OK) {
208            if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) {
209                ALOGW("correcting mismatch in internal/external playback rate");
210            }
211            // get playback settings used by audiosink, as it may be
212            // slightly off due to audiosink not taking small changes.
213            mPlaybackSettings = *rate;
214            if (mPaused) {
215                rate->mSpeed = 0.f;
216            }
217        }
218        return err;
219    }
220    *rate = mPlaybackSettings;
221    return OK;
222}
223
224status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
225    sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
226    writeToAMessage(msg, sync, videoFpsHint);
227    sp<AMessage> response;
228    status_t err = msg->postAndAwaitResponse(&response);
229    if (err == OK && response != NULL) {
230        CHECK(response->findInt32("err", &err));
231    }
232    return err;
233}
234
235status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) {
236    if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
237        return BAD_VALUE;
238    }
239    // TODO: support sync sources
240    return INVALID_OPERATION;
241}
242
243status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) {
244    sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
245    sp<AMessage> response;
246    status_t err = msg->postAndAwaitResponse(&response);
247    if (err == OK && response != NULL) {
248        CHECK(response->findInt32("err", &err));
249        if (err == OK) {
250            readFromAMessage(response, sync, videoFps);
251        }
252    }
253    return err;
254}
255
256status_t NuPlayer::Renderer::onGetSyncSettings(
257        AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
258    *sync = mSyncSettings;
259    *videoFps = -1.f;
260    return OK;
261}
262
263void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
264    {
265        Mutex::Autolock autoLock(mLock);
266        if (audio) {
267            mNotifyCompleteAudio |= notifyComplete;
268            clearAudioFirstAnchorTime_l();
269            ++mAudioQueueGeneration;
270            ++mAudioDrainGeneration;
271        } else {
272            mNotifyCompleteVideo |= notifyComplete;
273            ++mVideoQueueGeneration;
274            ++mVideoDrainGeneration;
275        }
276
277        clearAnchorTime_l();
278        mVideoLateByUs = 0;
279        mSyncQueues = false;
280    }
281
282    sp<AMessage> msg = new AMessage(kWhatFlush, this);
283    msg->setInt32("audio", static_cast<int32_t>(audio));
284    msg->post();
285}
286
287void NuPlayer::Renderer::signalTimeDiscontinuity() {
288}
289
290void NuPlayer::Renderer::signalDisableOffloadAudio() {
291    (new AMessage(kWhatDisableOffloadAudio, this))->post();
292}
293
294void NuPlayer::Renderer::signalEnableOffloadAudio() {
295    (new AMessage(kWhatEnableOffloadAudio, this))->post();
296}
297
298void NuPlayer::Renderer::pause() {
299    (new AMessage(kWhatPause, this))->post();
300}
301
302void NuPlayer::Renderer::resume() {
303    (new AMessage(kWhatResume, this))->post();
304}
305
306void NuPlayer::Renderer::setVideoFrameRate(float fps) {
307    sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
308    msg->setFloat("frame-rate", fps);
309    msg->post();
310}
311
312// Called on any threads.
313status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
314    return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
315}
316
317void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
318    mAudioFirstAnchorTimeMediaUs = -1;
319    mMediaClock->setStartingTimeMedia(-1);
320}
321
322void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
323    if (mAudioFirstAnchorTimeMediaUs == -1) {
324        mAudioFirstAnchorTimeMediaUs = mediaUs;
325        mMediaClock->setStartingTimeMedia(mediaUs);
326    }
327}
328
329void NuPlayer::Renderer::clearAnchorTime_l() {
330    mMediaClock->clearAnchor();
331    mAnchorTimeMediaUs = -1;
332    mAnchorNumFramesWritten = -1;
333}
334
335void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
336    Mutex::Autolock autoLock(mLock);
337    mVideoLateByUs = lateUs;
338}
339
340int64_t NuPlayer::Renderer::getVideoLateByUs() {
341    Mutex::Autolock autoLock(mLock);
342    return mVideoLateByUs;
343}
344
345status_t NuPlayer::Renderer::openAudioSink(
346        const sp<AMessage> &format,
347        bool offloadOnly,
348        bool hasVideo,
349        uint32_t flags,
350        bool *isOffloaded) {
351    sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
352    msg->setMessage("format", format);
353    msg->setInt32("offload-only", offloadOnly);
354    msg->setInt32("has-video", hasVideo);
355    msg->setInt32("flags", flags);
356
357    sp<AMessage> response;
358    msg->postAndAwaitResponse(&response);
359
360    int32_t err;
361    if (!response->findInt32("err", &err)) {
362        err = INVALID_OPERATION;
363    } else if (err == OK && isOffloaded != NULL) {
364        int32_t offload;
365        CHECK(response->findInt32("offload", &offload));
366        *isOffloaded = (offload != 0);
367    }
368    return err;
369}
370
371void NuPlayer::Renderer::closeAudioSink() {
372    sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
373
374    sp<AMessage> response;
375    msg->postAndAwaitResponse(&response);
376}
377
378void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
379    switch (msg->what()) {
380        case kWhatOpenAudioSink:
381        {
382            sp<AMessage> format;
383            CHECK(msg->findMessage("format", &format));
384
385            int32_t offloadOnly;
386            CHECK(msg->findInt32("offload-only", &offloadOnly));
387
388            int32_t hasVideo;
389            CHECK(msg->findInt32("has-video", &hasVideo));
390
391            uint32_t flags;
392            CHECK(msg->findInt32("flags", (int32_t *)&flags));
393
394            status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
395
396            sp<AMessage> response = new AMessage;
397            response->setInt32("err", err);
398            response->setInt32("offload", offloadingAudio());
399
400            sp<AReplyToken> replyID;
401            CHECK(msg->senderAwaitsResponse(&replyID));
402            response->postReply(replyID);
403
404            break;
405        }
406
407        case kWhatCloseAudioSink:
408        {
409            sp<AReplyToken> replyID;
410            CHECK(msg->senderAwaitsResponse(&replyID));
411
412            onCloseAudioSink();
413
414            sp<AMessage> response = new AMessage;
415            response->postReply(replyID);
416            break;
417        }
418
419        case kWhatStopAudioSink:
420        {
421            mAudioSink->stop();
422            break;
423        }
424
425        case kWhatDrainAudioQueue:
426        {
427            mDrainAudioQueuePending = false;
428
429            int32_t generation;
430            CHECK(msg->findInt32("drainGeneration", &generation));
431            if (generation != getDrainGeneration(true /* audio */)) {
432                break;
433            }
434
435            if (onDrainAudioQueue()) {
436                uint32_t numFramesPlayed;
437                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
438                         (status_t)OK);
439
440                uint32_t numFramesPendingPlayout =
441                    mNumFramesWritten - numFramesPlayed;
442
443                // This is how long the audio sink will have data to
444                // play back.
445                int64_t delayUs =
446                    mAudioSink->msecsPerFrame()
447                        * numFramesPendingPlayout * 1000ll;
448                if (mPlaybackRate > 1.0f) {
449                    delayUs /= mPlaybackRate;
450                }
451
452                // Let's give it more data after about half that time
453                // has elapsed.
454                Mutex::Autolock autoLock(mLock);
455                postDrainAudioQueue_l(delayUs / 2);
456            }
457            break;
458        }
459
460        case kWhatDrainVideoQueue:
461        {
462            int32_t generation;
463            CHECK(msg->findInt32("drainGeneration", &generation));
464            if (generation != getDrainGeneration(false /* audio */)) {
465                break;
466            }
467
468            mDrainVideoQueuePending = false;
469
470            onDrainVideoQueue();
471
472            postDrainVideoQueue();
473            break;
474        }
475
476        case kWhatPostDrainVideoQueue:
477        {
478            int32_t generation;
479            CHECK(msg->findInt32("drainGeneration", &generation));
480            if (generation != getDrainGeneration(false /* audio */)) {
481                break;
482            }
483
484            mDrainVideoQueuePending = false;
485            postDrainVideoQueue();
486            break;
487        }
488
489        case kWhatQueueBuffer:
490        {
491            onQueueBuffer(msg);
492            break;
493        }
494
495        case kWhatQueueEOS:
496        {
497            onQueueEOS(msg);
498            break;
499        }
500
501        case kWhatConfigPlayback:
502        {
503            sp<AReplyToken> replyID;
504            CHECK(msg->senderAwaitsResponse(&replyID));
505            AudioPlaybackRate rate;
506            readFromAMessage(msg, &rate);
507            status_t err = onConfigPlayback(rate);
508            sp<AMessage> response = new AMessage;
509            response->setInt32("err", err);
510            response->postReply(replyID);
511            break;
512        }
513
514        case kWhatGetPlaybackSettings:
515        {
516            sp<AReplyToken> replyID;
517            CHECK(msg->senderAwaitsResponse(&replyID));
518            AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT;
519            status_t err = onGetPlaybackSettings(&rate);
520            sp<AMessage> response = new AMessage;
521            if (err == OK) {
522                writeToAMessage(response, rate);
523            }
524            response->setInt32("err", err);
525            response->postReply(replyID);
526            break;
527        }
528
529        case kWhatConfigSync:
530        {
531            sp<AReplyToken> replyID;
532            CHECK(msg->senderAwaitsResponse(&replyID));
533            AVSyncSettings sync;
534            float videoFpsHint;
535            readFromAMessage(msg, &sync, &videoFpsHint);
536            status_t err = onConfigSync(sync, videoFpsHint);
537            sp<AMessage> response = new AMessage;
538            response->setInt32("err", err);
539            response->postReply(replyID);
540            break;
541        }
542
543        case kWhatGetSyncSettings:
544        {
545            sp<AReplyToken> replyID;
546            CHECK(msg->senderAwaitsResponse(&replyID));
547
548            ALOGV("kWhatGetSyncSettings");
549            AVSyncSettings sync;
550            float videoFps = -1.f;
551            status_t err = onGetSyncSettings(&sync, &videoFps);
552            sp<AMessage> response = new AMessage;
553            if (err == OK) {
554                writeToAMessage(response, sync, videoFps);
555            }
556            response->setInt32("err", err);
557            response->postReply(replyID);
558            break;
559        }
560
561        case kWhatFlush:
562        {
563            onFlush(msg);
564            break;
565        }
566
567        case kWhatDisableOffloadAudio:
568        {
569            onDisableOffloadAudio();
570            break;
571        }
572
573        case kWhatEnableOffloadAudio:
574        {
575            onEnableOffloadAudio();
576            break;
577        }
578
579        case kWhatPause:
580        {
581            onPause();
582            break;
583        }
584
585        case kWhatResume:
586        {
587            onResume();
588            break;
589        }
590
591        case kWhatSetVideoFrameRate:
592        {
593            float fps;
594            CHECK(msg->findFloat("frame-rate", &fps));
595            onSetVideoFrameRate(fps);
596            break;
597        }
598
599        case kWhatAudioTearDown:
600        {
601            onAudioTearDown(kDueToError);
602            break;
603        }
604
605        case kWhatAudioOffloadPauseTimeout:
606        {
607            int32_t generation;
608            CHECK(msg->findInt32("drainGeneration", &generation));
609            if (generation != mAudioOffloadPauseTimeoutGeneration) {
610                break;
611            }
612            ALOGV("Audio Offload tear down due to pause timeout.");
613            onAudioTearDown(kDueToTimeout);
614            mWakeLock->release();
615            break;
616        }
617
618        default:
619            TRESPASS();
620            break;
621    }
622}
623
624void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
625    if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) {
626        return;
627    }
628
629    if (mAudioQueue.empty()) {
630        return;
631    }
632
633    mDrainAudioQueuePending = true;
634    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
635    msg->setInt32("drainGeneration", mAudioDrainGeneration);
636    msg->post(delayUs);
637}
638
639void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
640    mAudioRenderingStartGeneration = mAudioDrainGeneration;
641    mVideoRenderingStartGeneration = mVideoDrainGeneration;
642}
643
644void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
645    if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
646        mAudioRenderingStartGeneration == mAudioDrainGeneration) {
647        mVideoRenderingStartGeneration = -1;
648        mAudioRenderingStartGeneration = -1;
649
650        sp<AMessage> notify = mNotify->dup();
651        notify->setInt32("what", kWhatMediaRenderingStart);
652        notify->post();
653    }
654}
655
656// static
657size_t NuPlayer::Renderer::AudioSinkCallback(
658        MediaPlayerBase::AudioSink * /* audioSink */,
659        void *buffer,
660        size_t size,
661        void *cookie,
662        MediaPlayerBase::AudioSink::cb_event_t event) {
663    NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
664
665    switch (event) {
666        case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
667        {
668            return me->fillAudioBuffer(buffer, size);
669            break;
670        }
671
672        case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
673        {
674            ALOGV("AudioSink::CB_EVENT_STREAM_END");
675            me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
676            break;
677        }
678
679        case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
680        {
681            ALOGV("AudioSink::CB_EVENT_TEAR_DOWN");
682            me->notifyAudioTearDown();
683            break;
684        }
685    }
686
687    return 0;
688}
689
690size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
691    Mutex::Autolock autoLock(mLock);
692
693    if (!mUseAudioCallback) {
694        return 0;
695    }
696
697    bool hasEOS = false;
698
699    size_t sizeCopied = 0;
700    bool firstEntry = true;
701    QueueEntry *entry;  // will be valid after while loop if hasEOS is set.
702    while (sizeCopied < size && !mAudioQueue.empty()) {
703        entry = &*mAudioQueue.begin();
704
705        if (entry->mBuffer == NULL) { // EOS
706            hasEOS = true;
707            mAudioQueue.erase(mAudioQueue.begin());
708            break;
709        }
710
711        if (firstEntry && entry->mOffset == 0) {
712            firstEntry = false;
713            int64_t mediaTimeUs;
714            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
715            ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
716            setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
717        }
718
719        size_t copy = entry->mBuffer->size() - entry->mOffset;
720        size_t sizeRemaining = size - sizeCopied;
721        if (copy > sizeRemaining) {
722            copy = sizeRemaining;
723        }
724
725        memcpy((char *)buffer + sizeCopied,
726               entry->mBuffer->data() + entry->mOffset,
727               copy);
728
729        entry->mOffset += copy;
730        if (entry->mOffset == entry->mBuffer->size()) {
731            entry->mNotifyConsumed->post();
732            mAudioQueue.erase(mAudioQueue.begin());
733            entry = NULL;
734        }
735        sizeCopied += copy;
736
737        notifyIfMediaRenderingStarted_l();
738    }
739
740    if (mAudioFirstAnchorTimeMediaUs >= 0) {
741        int64_t nowUs = ALooper::GetNowUs();
742        int64_t nowMediaUs =
743            mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs);
744        // we don't know how much data we are queueing for offloaded tracks.
745        mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
746    }
747
748    // for non-offloaded audio, we need to compute the frames written because
749    // there is no EVENT_STREAM_END notification. The frames written gives
750    // an estimate on the pending played out duration.
751    if (!offloadingAudio()) {
752        mNumFramesWritten += sizeCopied / mAudioSink->frameSize();
753    }
754
755    if (hasEOS) {
756        (new AMessage(kWhatStopAudioSink, this))->post();
757        // As there is currently no EVENT_STREAM_END callback notification for
758        // non-offloaded audio tracks, we need to post the EOS ourselves.
759        if (!offloadingAudio()) {
760            int64_t postEOSDelayUs = 0;
761            if (mAudioSink->needsTrailingPadding()) {
762                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
763            }
764            ALOGV("fillAudioBuffer: notifyEOS "
765                    "mNumFramesWritten:%u  finalResult:%d  postEOSDelay:%lld",
766                    mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs);
767            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
768        }
769    }
770    return sizeCopied;
771}
772
773void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() {
774    List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it;
775    bool foundEOS = false;
776    while (it != mAudioQueue.end()) {
777        int32_t eos;
778        QueueEntry *entry = &*it++;
779        if (entry->mBuffer == NULL
780                || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) {
781            itEOS = it;
782            foundEOS = true;
783        }
784    }
785
786    if (foundEOS) {
787        // post all replies before EOS and drop the samples
788        for (it = mAudioQueue.begin(); it != itEOS; it++) {
789            if (it->mBuffer == NULL) {
790                // delay doesn't matter as we don't even have an AudioTrack
791                notifyEOS(true /* audio */, it->mFinalResult);
792            } else {
793                it->mNotifyConsumed->post();
794            }
795        }
796        mAudioQueue.erase(mAudioQueue.begin(), itEOS);
797    }
798}
799
800bool NuPlayer::Renderer::onDrainAudioQueue() {
801    // do not drain audio during teardown as queued buffers may be invalid.
802    if (mAudioTornDown) {
803        return false;
804    }
805    // TODO: This call to getPosition checks if AudioTrack has been created
806    // in AudioSink before draining audio. If AudioTrack doesn't exist, then
807    // CHECKs on getPosition will fail.
808    // We still need to figure out why AudioTrack is not created when
809    // this function is called. One possible reason could be leftover
810    // audio. Another possible place is to check whether decoder
811    // has received INFO_FORMAT_CHANGED as the first buffer since
812    // AudioSink is opened there, and possible interactions with flush
813    // immediately after start. Investigate error message
814    // "vorbis_dsp_synthesis returned -135", along with RTSP.
815    uint32_t numFramesPlayed;
816    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
817        // When getPosition fails, renderer will not reschedule the draining
818        // unless new samples are queued.
819        // If we have pending EOS (or "eos" marker for discontinuities), we need
820        // to post these now as NuPlayerDecoder might be waiting for it.
821        drainAudioQueueUntilLastEOS();
822
823        ALOGW("onDrainAudioQueue(): audio sink is not ready");
824        return false;
825    }
826
827#if 0
828    ssize_t numFramesAvailableToWrite =
829        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
830
831    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
832        ALOGI("audio sink underrun");
833    } else {
834        ALOGV("audio queue has %d frames left to play",
835             mAudioSink->frameCount() - numFramesAvailableToWrite);
836    }
837#endif
838
839    uint32_t prevFramesWritten = mNumFramesWritten;
840    while (!mAudioQueue.empty()) {
841        QueueEntry *entry = &*mAudioQueue.begin();
842
843        mLastAudioBufferDrained = entry->mBufferOrdinal;
844
845        if (entry->mBuffer == NULL) {
846            // EOS
847            int64_t postEOSDelayUs = 0;
848            if (mAudioSink->needsTrailingPadding()) {
849                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
850            }
851            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
852
853            mAudioQueue.erase(mAudioQueue.begin());
854            entry = NULL;
855            if (mAudioSink->needsTrailingPadding()) {
856                // If we're not in gapless playback (i.e. through setNextPlayer), we
857                // need to stop the track here, because that will play out the last
858                // little bit at the end of the file. Otherwise short files won't play.
859                mAudioSink->stop();
860                mNumFramesWritten = 0;
861            }
862            return false;
863        }
864
865        // ignore 0-sized buffer which could be EOS marker with no data
866        if (entry->mOffset == 0 && entry->mBuffer->size() > 0) {
867            int64_t mediaTimeUs;
868            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
869            ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs",
870                    mediaTimeUs / 1E6);
871            onNewAudioMediaTime(mediaTimeUs);
872        }
873
874        size_t copy = entry->mBuffer->size() - entry->mOffset;
875
876        ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
877                                            copy, false /* blocking */);
878        if (written < 0) {
879            // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
880            if (written == WOULD_BLOCK) {
881                ALOGV("AudioSink write would block when writing %zu bytes", copy);
882            } else {
883                ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
884                // This can only happen when AudioSink was opened with doNotReconnect flag set to
885                // true, in which case the NuPlayer will handle the reconnect.
886                notifyAudioTearDown();
887            }
888            break;
889        }
890
891        entry->mOffset += written;
892        if (entry->mOffset == entry->mBuffer->size()) {
893            entry->mNotifyConsumed->post();
894            mAudioQueue.erase(mAudioQueue.begin());
895
896            entry = NULL;
897        }
898
899        size_t copiedFrames = written / mAudioSink->frameSize();
900        mNumFramesWritten += copiedFrames;
901
902        {
903            Mutex::Autolock autoLock(mLock);
904            notifyIfMediaRenderingStarted_l();
905        }
906
907        if (written != (ssize_t)copy) {
908            // A short count was received from AudioSink::write()
909            //
910            // AudioSink write is called in non-blocking mode.
911            // It may return with a short count when:
912            //
913            // 1) Size to be copied is not a multiple of the frame size. We consider this fatal.
914            // 2) The data to be copied exceeds the available buffer in AudioSink.
915            // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
916            // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
917
918            // (Case 1)
919            // Must be a multiple of the frame size.  If it is not a multiple of a frame size, it
920            // needs to fail, as we should not carry over fractional frames between calls.
921            CHECK_EQ(copy % mAudioSink->frameSize(), 0);
922
923            // (Case 2, 3, 4)
924            // Return early to the caller.
925            // Beware of calling immediately again as this may busy-loop if you are not careful.
926            ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
927            break;
928        }
929    }
930    int64_t maxTimeMedia;
931    {
932        Mutex::Autolock autoLock(mLock);
933        maxTimeMedia =
934            mAnchorTimeMediaUs +
935                    (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
936                            * 1000LL * mAudioSink->msecsPerFrame());
937    }
938    mMediaClock->updateMaxTimeMedia(maxTimeMedia);
939
940    // calculate whether we need to reschedule another write.
941    bool reschedule = !mAudioQueue.empty()
942            && (!mPaused
943                || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers
944    //ALOGD("reschedule:%d  empty:%d  mPaused:%d  prevFramesWritten:%u  mNumFramesWritten:%u",
945    //        reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten);
946    return reschedule;
947}
948
949int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
950    int32_t sampleRate = offloadingAudio() ?
951            mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
952    if (sampleRate == 0) {
953        ALOGE("sampleRate is 0 in %s mode", offloadingAudio() ? "offload" : "non-offload");
954        return 0;
955    }
956    // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
957    return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate);
958}
959
960// Calculate duration of pending samples if played at normal rate (i.e., 1.0).
961int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
962    int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
963    return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs);
964}
965
966int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
967    int64_t realUs;
968    if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
969        // If failed to get current position, e.g. due to audio clock is
970        // not ready, then just play out video immediately without delay.
971        return nowUs;
972    }
973    return realUs;
974}
975
976void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
977    Mutex::Autolock autoLock(mLock);
978    // TRICKY: vorbis decoder generates multiple frames with the same
979    // timestamp, so only update on the first frame with a given timestamp
980    if (mediaTimeUs == mAnchorTimeMediaUs) {
981        return;
982    }
983    setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
984    int64_t nowUs = ALooper::GetNowUs();
985    int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
986    mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
987    mAnchorNumFramesWritten = mNumFramesWritten;
988    mAnchorTimeMediaUs = mediaTimeUs;
989}
990
991// Called without mLock acquired.
992void NuPlayer::Renderer::postDrainVideoQueue() {
993    if (mDrainVideoQueuePending
994            || getSyncQueues()
995            || (mPaused && mVideoSampleReceived)) {
996        return;
997    }
998
999    if (mVideoQueue.empty()) {
1000        return;
1001    }
1002
1003    QueueEntry &entry = *mVideoQueue.begin();
1004
1005    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
1006    msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
1007
1008    if (entry.mBuffer == NULL) {
1009        // EOS doesn't carry a timestamp.
1010        msg->post();
1011        mDrainVideoQueuePending = true;
1012        return;
1013    }
1014
1015    int64_t delayUs;
1016    int64_t nowUs = ALooper::GetNowUs();
1017    int64_t realTimeUs;
1018    if (mFlags & FLAG_REAL_TIME) {
1019        int64_t mediaTimeUs;
1020        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1021        realTimeUs = mediaTimeUs;
1022    } else {
1023        int64_t mediaTimeUs;
1024        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1025
1026        {
1027            Mutex::Autolock autoLock(mLock);
1028            if (mAnchorTimeMediaUs < 0) {
1029                mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
1030                mAnchorTimeMediaUs = mediaTimeUs;
1031                realTimeUs = nowUs;
1032            } else {
1033                realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1034            }
1035        }
1036        if (!mHasAudio) {
1037            // smooth out videos >= 10fps
1038            mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
1039        }
1040
1041        // Heuristics to handle situation when media time changed without a
1042        // discontinuity. If we have not drained an audio buffer that was
1043        // received after this buffer, repost in 10 msec. Otherwise repost
1044        // in 500 msec.
1045        delayUs = realTimeUs - nowUs;
1046        if (delayUs > 500000) {
1047            int64_t postDelayUs = 500000;
1048            if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) {
1049                postDelayUs = 10000;
1050            }
1051            msg->setWhat(kWhatPostDrainVideoQueue);
1052            msg->post(postDelayUs);
1053            mVideoScheduler->restart();
1054            ALOGI("possible video time jump of %dms, retrying in %dms",
1055                    (int)(delayUs / 1000), (int)(postDelayUs / 1000));
1056            mDrainVideoQueuePending = true;
1057            return;
1058        }
1059    }
1060
1061    realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
1062    int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
1063
1064    delayUs = realTimeUs - nowUs;
1065
1066    ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
1067    // post 2 display refreshes before rendering is due
1068    msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
1069
1070    mDrainVideoQueuePending = true;
1071}
1072
1073void NuPlayer::Renderer::onDrainVideoQueue() {
1074    if (mVideoQueue.empty()) {
1075        return;
1076    }
1077
1078    QueueEntry *entry = &*mVideoQueue.begin();
1079
1080    if (entry->mBuffer == NULL) {
1081        // EOS
1082
1083        notifyEOS(false /* audio */, entry->mFinalResult);
1084
1085        mVideoQueue.erase(mVideoQueue.begin());
1086        entry = NULL;
1087
1088        setVideoLateByUs(0);
1089        return;
1090    }
1091
1092    int64_t nowUs = -1;
1093    int64_t realTimeUs;
1094    if (mFlags & FLAG_REAL_TIME) {
1095        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
1096    } else {
1097        int64_t mediaTimeUs;
1098        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1099
1100        nowUs = ALooper::GetNowUs();
1101        realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1102    }
1103
1104    bool tooLate = false;
1105
1106    if (!mPaused) {
1107        if (nowUs == -1) {
1108            nowUs = ALooper::GetNowUs();
1109        }
1110        setVideoLateByUs(nowUs - realTimeUs);
1111        tooLate = (mVideoLateByUs > 40000);
1112
1113        if (tooLate) {
1114            ALOGV("video late by %lld us (%.2f secs)",
1115                 (long long)mVideoLateByUs, mVideoLateByUs / 1E6);
1116        } else {
1117            int64_t mediaUs = 0;
1118            mMediaClock->getMediaTime(realTimeUs, &mediaUs);
1119            ALOGV("rendering video at media time %.2f secs",
1120                    (mFlags & FLAG_REAL_TIME ? realTimeUs :
1121                    mediaUs) / 1E6);
1122        }
1123    } else {
1124        setVideoLateByUs(0);
1125        if (!mVideoSampleReceived && !mHasAudio) {
1126            // This will ensure that the first frame after a flush won't be used as anchor
1127            // when renderer is in paused state, because resume can happen any time after seek.
1128            Mutex::Autolock autoLock(mLock);
1129            clearAnchorTime_l();
1130        }
1131    }
1132
1133    entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
1134    entry->mNotifyConsumed->setInt32("render", !tooLate);
1135    entry->mNotifyConsumed->post();
1136    mVideoQueue.erase(mVideoQueue.begin());
1137    entry = NULL;
1138
1139    mVideoSampleReceived = true;
1140
1141    if (!mPaused) {
1142        if (!mVideoRenderingStarted) {
1143            mVideoRenderingStarted = true;
1144            notifyVideoRenderingStart();
1145        }
1146        Mutex::Autolock autoLock(mLock);
1147        notifyIfMediaRenderingStarted_l();
1148    }
1149}
1150
1151void NuPlayer::Renderer::notifyVideoRenderingStart() {
1152    sp<AMessage> notify = mNotify->dup();
1153    notify->setInt32("what", kWhatVideoRenderingStart);
1154    notify->post();
1155}
1156
1157void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
1158    sp<AMessage> notify = mNotify->dup();
1159    notify->setInt32("what", kWhatEOS);
1160    notify->setInt32("audio", static_cast<int32_t>(audio));
1161    notify->setInt32("finalResult", finalResult);
1162    notify->post(delayUs);
1163}
1164
1165void NuPlayer::Renderer::notifyAudioTearDown() {
1166    (new AMessage(kWhatAudioTearDown, this))->post();
1167}
1168
1169void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
1170    int32_t audio;
1171    CHECK(msg->findInt32("audio", &audio));
1172
1173    if (dropBufferIfStale(audio, msg)) {
1174        return;
1175    }
1176
1177    if (audio) {
1178        mHasAudio = true;
1179    } else {
1180        mHasVideo = true;
1181    }
1182
1183    if (mHasVideo) {
1184        if (mVideoScheduler == NULL) {
1185            mVideoScheduler = new VideoFrameScheduler();
1186            mVideoScheduler->init();
1187        }
1188    }
1189
1190    sp<ABuffer> buffer;
1191    CHECK(msg->findBuffer("buffer", &buffer));
1192
1193    sp<AMessage> notifyConsumed;
1194    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
1195
1196    QueueEntry entry;
1197    entry.mBuffer = buffer;
1198    entry.mNotifyConsumed = notifyConsumed;
1199    entry.mOffset = 0;
1200    entry.mFinalResult = OK;
1201    entry.mBufferOrdinal = ++mTotalBuffersQueued;
1202
1203    if (audio) {
1204        Mutex::Autolock autoLock(mLock);
1205        mAudioQueue.push_back(entry);
1206        postDrainAudioQueue_l();
1207    } else {
1208        mVideoQueue.push_back(entry);
1209        postDrainVideoQueue();
1210    }
1211
1212    Mutex::Autolock autoLock(mLock);
1213    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
1214        return;
1215    }
1216
1217    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
1218    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
1219
1220    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
1221        // EOS signalled on either queue.
1222        syncQueuesDone_l();
1223        return;
1224    }
1225
1226    int64_t firstAudioTimeUs;
1227    int64_t firstVideoTimeUs;
1228    CHECK(firstAudioBuffer->meta()
1229            ->findInt64("timeUs", &firstAudioTimeUs));
1230    CHECK(firstVideoBuffer->meta()
1231            ->findInt64("timeUs", &firstVideoTimeUs));
1232
1233    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
1234
1235    ALOGV("queueDiff = %.2f secs", diff / 1E6);
1236
1237    if (diff > 100000ll) {
1238        // Audio data starts More than 0.1 secs before video.
1239        // Drop some audio.
1240
1241        (*mAudioQueue.begin()).mNotifyConsumed->post();
1242        mAudioQueue.erase(mAudioQueue.begin());
1243        return;
1244    }
1245
1246    syncQueuesDone_l();
1247}
1248
1249void NuPlayer::Renderer::syncQueuesDone_l() {
1250    if (!mSyncQueues) {
1251        return;
1252    }
1253
1254    mSyncQueues = false;
1255
1256    if (!mAudioQueue.empty()) {
1257        postDrainAudioQueue_l();
1258    }
1259
1260    if (!mVideoQueue.empty()) {
1261        mLock.unlock();
1262        postDrainVideoQueue();
1263        mLock.lock();
1264    }
1265}
1266
1267void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1268    int32_t audio;
1269    CHECK(msg->findInt32("audio", &audio));
1270
1271    if (dropBufferIfStale(audio, msg)) {
1272        return;
1273    }
1274
1275    int32_t finalResult;
1276    CHECK(msg->findInt32("finalResult", &finalResult));
1277
1278    QueueEntry entry;
1279    entry.mOffset = 0;
1280    entry.mFinalResult = finalResult;
1281
1282    if (audio) {
1283        Mutex::Autolock autoLock(mLock);
1284        if (mAudioQueue.empty() && mSyncQueues) {
1285            syncQueuesDone_l();
1286        }
1287        mAudioQueue.push_back(entry);
1288        postDrainAudioQueue_l();
1289    } else {
1290        if (mVideoQueue.empty() && getSyncQueues()) {
1291            Mutex::Autolock autoLock(mLock);
1292            syncQueuesDone_l();
1293        }
1294        mVideoQueue.push_back(entry);
1295        postDrainVideoQueue();
1296    }
1297}
1298
1299void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
1300    int32_t audio, notifyComplete;
1301    CHECK(msg->findInt32("audio", &audio));
1302
1303    {
1304        Mutex::Autolock autoLock(mLock);
1305        if (audio) {
1306            notifyComplete = mNotifyCompleteAudio;
1307            mNotifyCompleteAudio = false;
1308        } else {
1309            notifyComplete = mNotifyCompleteVideo;
1310            mNotifyCompleteVideo = false;
1311        }
1312
1313        // If we're currently syncing the queues, i.e. dropping audio while
1314        // aligning the first audio/video buffer times and only one of the
1315        // two queues has data, we may starve that queue by not requesting
1316        // more buffers from the decoder. If the other source then encounters
1317        // a discontinuity that leads to flushing, we'll never find the
1318        // corresponding discontinuity on the other queue.
1319        // Therefore we'll stop syncing the queues if at least one of them
1320        // is flushed.
1321        syncQueuesDone_l();
1322        clearAnchorTime_l();
1323    }
1324
1325    ALOGV("flushing %s", audio ? "audio" : "video");
1326    if (audio) {
1327        {
1328            Mutex::Autolock autoLock(mLock);
1329            flushQueue(&mAudioQueue);
1330
1331            ++mAudioDrainGeneration;
1332            prepareForMediaRenderingStart_l();
1333
1334            // the frame count will be reset after flush.
1335            clearAudioFirstAnchorTime_l();
1336        }
1337
1338        mDrainAudioQueuePending = false;
1339
1340        if (offloadingAudio()) {
1341            mAudioSink->pause();
1342            mAudioSink->flush();
1343            if (!mPaused) {
1344                mAudioSink->start();
1345            }
1346        } else {
1347            mAudioSink->pause();
1348            mAudioSink->flush();
1349            // Call stop() to signal to the AudioSink to completely fill the
1350            // internal buffer before resuming playback.
1351            mAudioSink->stop();
1352            if (!mPaused) {
1353                mAudioSink->start();
1354            }
1355            mNumFramesWritten = 0;
1356        }
1357    } else {
1358        flushQueue(&mVideoQueue);
1359
1360        mDrainVideoQueuePending = false;
1361
1362        if (mVideoScheduler != NULL) {
1363            mVideoScheduler->restart();
1364        }
1365
1366        Mutex::Autolock autoLock(mLock);
1367        ++mVideoDrainGeneration;
1368        prepareForMediaRenderingStart_l();
1369    }
1370
1371    mVideoSampleReceived = false;
1372
1373    if (notifyComplete) {
1374        notifyFlushComplete(audio);
1375    }
1376}
1377
1378void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1379    while (!queue->empty()) {
1380        QueueEntry *entry = &*queue->begin();
1381
1382        if (entry->mBuffer != NULL) {
1383            entry->mNotifyConsumed->post();
1384        }
1385
1386        queue->erase(queue->begin());
1387        entry = NULL;
1388    }
1389}
1390
1391void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1392    sp<AMessage> notify = mNotify->dup();
1393    notify->setInt32("what", kWhatFlushComplete);
1394    notify->setInt32("audio", static_cast<int32_t>(audio));
1395    notify->post();
1396}
1397
1398bool NuPlayer::Renderer::dropBufferIfStale(
1399        bool audio, const sp<AMessage> &msg) {
1400    int32_t queueGeneration;
1401    CHECK(msg->findInt32("queueGeneration", &queueGeneration));
1402
1403    if (queueGeneration == getQueueGeneration(audio)) {
1404        return false;
1405    }
1406
1407    sp<AMessage> notifyConsumed;
1408    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1409        notifyConsumed->post();
1410    }
1411
1412    return true;
1413}
1414
1415void NuPlayer::Renderer::onAudioSinkChanged() {
1416    if (offloadingAudio()) {
1417        return;
1418    }
1419    CHECK(!mDrainAudioQueuePending);
1420    mNumFramesWritten = 0;
1421    {
1422        Mutex::Autolock autoLock(mLock);
1423        mAnchorNumFramesWritten = -1;
1424    }
1425    uint32_t written;
1426    if (mAudioSink->getFramesWritten(&written) == OK) {
1427        mNumFramesWritten = written;
1428    }
1429}
1430
1431void NuPlayer::Renderer::onDisableOffloadAudio() {
1432    Mutex::Autolock autoLock(mLock);
1433    mFlags &= ~FLAG_OFFLOAD_AUDIO;
1434    ++mAudioDrainGeneration;
1435    if (mAudioRenderingStartGeneration != -1) {
1436        prepareForMediaRenderingStart_l();
1437    }
1438}
1439
1440void NuPlayer::Renderer::onEnableOffloadAudio() {
1441    Mutex::Autolock autoLock(mLock);
1442    mFlags |= FLAG_OFFLOAD_AUDIO;
1443    ++mAudioDrainGeneration;
1444    if (mAudioRenderingStartGeneration != -1) {
1445        prepareForMediaRenderingStart_l();
1446    }
1447}
1448
1449void NuPlayer::Renderer::onPause() {
1450    if (mPaused) {
1451        return;
1452    }
1453
1454    {
1455        Mutex::Autolock autoLock(mLock);
1456        // we do not increment audio drain generation so that we fill audio buffer during pause.
1457        ++mVideoDrainGeneration;
1458        prepareForMediaRenderingStart_l();
1459        mPaused = true;
1460        mMediaClock->setPlaybackRate(0.0);
1461    }
1462
1463    mDrainAudioQueuePending = false;
1464    mDrainVideoQueuePending = false;
1465
1466    if (mHasAudio) {
1467        mAudioSink->pause();
1468        startAudioOffloadPauseTimeout();
1469    }
1470
1471    ALOGV("now paused audio queue has %zu entries, video has %zu entries",
1472          mAudioQueue.size(), mVideoQueue.size());
1473}
1474
1475void NuPlayer::Renderer::onResume() {
1476    if (!mPaused) {
1477        return;
1478    }
1479
1480    if (mHasAudio) {
1481        cancelAudioOffloadPauseTimeout();
1482        status_t err = mAudioSink->start();
1483        if (err != OK) {
1484            ALOGE("cannot start AudioSink err %d", err);
1485            notifyAudioTearDown();
1486        }
1487    }
1488
1489    {
1490        Mutex::Autolock autoLock(mLock);
1491        mPaused = false;
1492
1493        // configure audiosink as we did not do it when pausing
1494        if (mAudioSink != NULL && mAudioSink->ready()) {
1495            mAudioSink->setPlaybackRate(mPlaybackSettings);
1496        }
1497
1498        mMediaClock->setPlaybackRate(mPlaybackRate);
1499
1500        if (!mAudioQueue.empty()) {
1501            postDrainAudioQueue_l();
1502        }
1503    }
1504
1505    if (!mVideoQueue.empty()) {
1506        postDrainVideoQueue();
1507    }
1508}
1509
1510void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1511    if (mVideoScheduler == NULL) {
1512        mVideoScheduler = new VideoFrameScheduler();
1513    }
1514    mVideoScheduler->init(fps);
1515}
1516
1517int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
1518    Mutex::Autolock autoLock(mLock);
1519    return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
1520}
1521
1522int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
1523    Mutex::Autolock autoLock(mLock);
1524    return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
1525}
1526
1527bool NuPlayer::Renderer::getSyncQueues() {
1528    Mutex::Autolock autoLock(mLock);
1529    return mSyncQueues;
1530}
1531
1532// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs()
1533// as it acquires locks and may query the audio driver.
1534//
1535// Some calls could conceivably retrieve extrapolated data instead of
1536// accessing getTimestamp() or getPosition() every time a data buffer with
1537// a media time is received.
1538//
1539// Calculate duration of played samples if played at normal rate (i.e., 1.0).
1540int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) {
1541    uint32_t numFramesPlayed;
1542    int64_t numFramesPlayedAt;
1543    AudioTimestamp ts;
1544    static const int64_t kStaleTimestamp100ms = 100000;
1545
1546    status_t res = mAudioSink->getTimestamp(ts);
1547    if (res == OK) {                 // case 1: mixing audio tracks and offloaded tracks.
1548        numFramesPlayed = ts.mPosition;
1549        numFramesPlayedAt =
1550            ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
1551        const int64_t timestampAge = nowUs - numFramesPlayedAt;
1552        if (timestampAge > kStaleTimestamp100ms) {
1553            // This is an audio FIXME.
1554            // getTimestamp returns a timestamp which may come from audio mixing threads.
1555            // After pausing, the MixerThread may go idle, thus the mTime estimate may
1556            // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms,
1557            // the max latency should be about 25ms with an average around 12ms (to be verified).
1558            // For safety we use 100ms.
1559            ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)",
1560                    (long long)nowUs, (long long)numFramesPlayedAt);
1561            numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
1562        }
1563        //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1564    } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track
1565        numFramesPlayed = 0;
1566        numFramesPlayedAt = nowUs;
1567        //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
1568        //        numFramesPlayed, (long long)numFramesPlayedAt);
1569    } else {                         // case 3: transitory at new track or audio fast tracks.
1570        res = mAudioSink->getPosition(&numFramesPlayed);
1571        CHECK_EQ(res, (status_t)OK);
1572        numFramesPlayedAt = nowUs;
1573        numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */
1574        //ALOGD("getPosition: %u %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1575    }
1576
1577    //CHECK_EQ(numFramesPlayed & (1 << 31), 0);  // can't be negative until 12.4 hrs, test
1578    int64_t durationUs = getDurationUsIfPlayedAtSampleRate(numFramesPlayed)
1579            + nowUs - numFramesPlayedAt;
1580    if (durationUs < 0) {
1581        // Occurs when numFramesPlayed position is very small and the following:
1582        // (1) In case 1, the time nowUs is computed before getTimestamp() is called and
1583        //     numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed.
1584        // (2) In case 3, using getPosition and adding mAudioSink->latency() to
1585        //     numFramesPlayedAt, by a time amount greater than numFramesPlayed.
1586        //
1587        // Both of these are transitory conditions.
1588        ALOGV("getPlayedOutAudioDurationUs: negative duration %lld set to zero", (long long)durationUs);
1589        durationUs = 0;
1590    }
1591    ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)",
1592            (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt);
1593    return durationUs;
1594}
1595
1596void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) {
1597    if (mAudioTornDown) {
1598        return;
1599    }
1600    mAudioTornDown = true;
1601
1602    int64_t currentPositionUs;
1603    sp<AMessage> notify = mNotify->dup();
1604    if (getCurrentPosition(&currentPositionUs) == OK) {
1605        notify->setInt64("positionUs", currentPositionUs);
1606    }
1607
1608    mAudioSink->stop();
1609    mAudioSink->flush();
1610
1611    notify->setInt32("what", kWhatAudioTearDown);
1612    notify->setInt32("reason", reason);
1613    notify->post();
1614}
1615
1616void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1617    if (offloadingAudio()) {
1618        mWakeLock->acquire();
1619        sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
1620        msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
1621        msg->post(kOffloadPauseMaxUs);
1622    }
1623}
1624
1625void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
1626    if (offloadingAudio()) {
1627        mWakeLock->release(true);
1628        ++mAudioOffloadPauseTimeoutGeneration;
1629    }
1630}
1631
1632status_t NuPlayer::Renderer::onOpenAudioSink(
1633        const sp<AMessage> &format,
1634        bool offloadOnly,
1635        bool hasVideo,
1636        uint32_t flags) {
1637    ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1638            offloadOnly, offloadingAudio());
1639    bool audioSinkChanged = false;
1640
1641    int32_t numChannels;
1642    CHECK(format->findInt32("channel-count", &numChannels));
1643
1644    int32_t channelMask;
1645    if (!format->findInt32("channel-mask", &channelMask)) {
1646        // signal to the AudioSink to derive the mask from count.
1647        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1648    }
1649
1650    int32_t sampleRate;
1651    CHECK(format->findInt32("sample-rate", &sampleRate));
1652
1653    if (offloadingAudio()) {
1654        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1655        AString mime;
1656        CHECK(format->findString("mime", &mime));
1657        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1658
1659        if (err != OK) {
1660            ALOGE("Couldn't map mime \"%s\" to a valid "
1661                    "audio_format", mime.c_str());
1662            onDisableOffloadAudio();
1663        } else {
1664            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1665                    mime.c_str(), audioFormat);
1666
1667            int avgBitRate = -1;
1668            format->findInt32("bit-rate", &avgBitRate);
1669
1670            int32_t aacProfile = -1;
1671            if (audioFormat == AUDIO_FORMAT_AAC
1672                    && format->findInt32("aac-profile", &aacProfile)) {
1673                // Redefine AAC format as per aac profile
1674                mapAACProfileToAudioFormat(
1675                        audioFormat,
1676                        aacProfile);
1677            }
1678
1679            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1680            offloadInfo.duration_us = -1;
1681            format->findInt64(
1682                    "durationUs", &offloadInfo.duration_us);
1683            offloadInfo.sample_rate = sampleRate;
1684            offloadInfo.channel_mask = channelMask;
1685            offloadInfo.format = audioFormat;
1686            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1687            offloadInfo.bit_rate = avgBitRate;
1688            offloadInfo.has_video = hasVideo;
1689            offloadInfo.is_streaming = true;
1690
1691            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1692                ALOGV("openAudioSink: no change in offload mode");
1693                // no change from previous configuration, everything ok.
1694                return OK;
1695            }
1696            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1697
1698            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1699            uint32_t offloadFlags = flags;
1700            offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1701            offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1702            audioSinkChanged = true;
1703            mAudioSink->close();
1704
1705            err = mAudioSink->open(
1706                    sampleRate,
1707                    numChannels,
1708                    (audio_channel_mask_t)channelMask,
1709                    audioFormat,
1710                    0 /* bufferCount - unused */,
1711                    &NuPlayer::Renderer::AudioSinkCallback,
1712                    this,
1713                    (audio_output_flags_t)offloadFlags,
1714                    &offloadInfo);
1715
1716            if (err == OK) {
1717                err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1718            }
1719
1720            if (err == OK) {
1721                // If the playback is offloaded to h/w, we pass
1722                // the HAL some metadata information.
1723                // We don't want to do this for PCM because it
1724                // will be going through the AudioFlinger mixer
1725                // before reaching the hardware.
1726                // TODO
1727                mCurrentOffloadInfo = offloadInfo;
1728                if (!mPaused) { // for preview mode, don't start if paused
1729                    err = mAudioSink->start();
1730                }
1731                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1732            }
1733            if (err != OK) {
1734                // Clean up, fall back to non offload mode.
1735                mAudioSink->close();
1736                onDisableOffloadAudio();
1737                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1738                ALOGV("openAudioSink: offload failed");
1739            } else {
1740                mUseAudioCallback = true;  // offload mode transfers data through callback
1741                ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
1742            }
1743        }
1744    }
1745    if (!offloadOnly && !offloadingAudio()) {
1746        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1747        uint32_t pcmFlags = flags;
1748        pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1749
1750        const PcmInfo info = {
1751                (audio_channel_mask_t)channelMask,
1752                (audio_output_flags_t)pcmFlags,
1753                AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat
1754                numChannels,
1755                sampleRate
1756        };
1757        if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
1758            ALOGV("openAudioSink: no change in pcm mode");
1759            // no change from previous configuration, everything ok.
1760            return OK;
1761        }
1762
1763        audioSinkChanged = true;
1764        mAudioSink->close();
1765        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1766        // Note: It is possible to set up the callback, but not use it to send audio data.
1767        // This requires a fix in AudioSink to explicitly specify the transfer mode.
1768        mUseAudioCallback = getUseAudioCallbackSetting();
1769        if (mUseAudioCallback) {
1770            ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
1771        }
1772
1773        // Compute the desired buffer size.
1774        // For callback mode, the amount of time before wakeup is about half the buffer size.
1775        const uint32_t frameCount =
1776                (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000;
1777
1778        // The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct
1779        // AudioSink. We don't want this when there's video because it will cause a video seek to
1780        // the previous I frame. But we do want this when there's only audio because it will give
1781        // NuPlayer a chance to switch from non-offload mode to offload mode.
1782        // So we only set doNotReconnect when there's no video.
1783        const bool doNotReconnect = !hasVideo;
1784        status_t err = mAudioSink->open(
1785                    sampleRate,
1786                    numChannels,
1787                    (audio_channel_mask_t)channelMask,
1788                    AUDIO_FORMAT_PCM_16_BIT,
1789                    0 /* bufferCount - unused */,
1790                    mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL,
1791                    mUseAudioCallback ? this : NULL,
1792                    (audio_output_flags_t)pcmFlags,
1793                    NULL,
1794                    doNotReconnect,
1795                    frameCount);
1796        if (err == OK) {
1797            err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1798        }
1799        if (err != OK) {
1800            ALOGW("openAudioSink: non offloaded open failed status: %d", err);
1801            mAudioSink->close();
1802            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1803            return err;
1804        }
1805        mCurrentPcmInfo = info;
1806        if (!mPaused) { // for preview mode, don't start if paused
1807            mAudioSink->start();
1808        }
1809    }
1810    if (audioSinkChanged) {
1811        onAudioSinkChanged();
1812    }
1813    mAudioTornDown = false;
1814    return OK;
1815}
1816
1817void NuPlayer::Renderer::onCloseAudioSink() {
1818    mAudioSink->close();
1819    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1820    mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1821}
1822
1823}  // namespace android
1824
1825