NuPlayerRenderer.cpp revision 179652ee2a508361df1aa18e99000373886f0816 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22#include <cutils/properties.h>
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26#include <media/stagefright/foundation/AUtils.h>
27#include <media/stagefright/foundation/AWakeLock.h>
28#include <media/stagefright/MediaClock.h>
29#include <media/stagefright/MediaErrors.h>
30#include <media/stagefright/MetaData.h>
31#include <media/stagefright/Utils.h>
32
33#include <VideoFrameScheduler.h>
34
35#include <inttypes.h>
36
37namespace android {
38
39/*
40 * Example of common configuration settings in shell script form
41
42   #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager
43   adb shell setprop audio.offload.disable 1
44
45   #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager
46   adb shell setprop audio.offload.video 1
47
48   #Use audio callbacks for PCM data
49   adb shell setprop media.stagefright.audio.cbk 1
50
51   #Set size of buffers for pcm audio sink in msec (example: 1000 msec)
52   adb shell setprop media.stagefright.audio.sink 1000
53
54 * These configurations take effect for the next track played (not the current track).
55 */
56
57static inline bool getUseAudioCallbackSetting() {
58    return property_get_bool("media.stagefright.audio.cbk", false /* default_value */);
59}
60
61static inline int32_t getAudioSinkPcmMsSetting() {
62    return property_get_int32(
63            "media.stagefright.audio.sink", 500 /* default_value */);
64}
65
66// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
67// is closed to allow the audio DSP to power down.
68static const int64_t kOffloadPauseMaxUs = 10000000ll;
69
70// static
71const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
72        AUDIO_CHANNEL_NONE,
73        AUDIO_OUTPUT_FLAG_NONE,
74        AUDIO_FORMAT_INVALID,
75        0, // mNumChannels
76        0 // mSampleRate
77};
78
79// static
80const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
81
82NuPlayer::Renderer::Renderer(
83        const sp<MediaPlayerBase::AudioSink> &sink,
84        const sp<AMessage> &notify,
85        uint32_t flags)
86    : mAudioSink(sink),
87      mNotify(notify),
88      mFlags(flags),
89      mNumFramesWritten(0),
90      mDrainAudioQueuePending(false),
91      mDrainVideoQueuePending(false),
92      mAudioQueueGeneration(0),
93      mVideoQueueGeneration(0),
94      mAudioDrainGeneration(0),
95      mVideoDrainGeneration(0),
96      mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
97      mAudioFirstAnchorTimeMediaUs(-1),
98      mAnchorTimeMediaUs(-1),
99      mAnchorNumFramesWritten(-1),
100      mVideoLateByUs(0ll),
101      mHasAudio(false),
102      mHasVideo(false),
103      mNotifyCompleteAudio(false),
104      mNotifyCompleteVideo(false),
105      mSyncQueues(false),
106      mPaused(false),
107      mVideoSampleReceived(false),
108      mVideoRenderingStarted(false),
109      mVideoRenderingStartGeneration(0),
110      mAudioRenderingStartGeneration(0),
111      mAudioOffloadPauseTimeoutGeneration(0),
112      mAudioTornDown(false),
113      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
114      mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
115      mTotalBuffersQueued(0),
116      mLastAudioBufferDrained(0),
117      mUseAudioCallback(false),
118      mWakeLock(new AWakeLock()) {
119    mMediaClock = new MediaClock;
120    mPlaybackRate = mPlaybackSettings.mSpeed;
121    mMediaClock->setPlaybackRate(mPlaybackRate);
122}
123
124NuPlayer::Renderer::~Renderer() {
125    if (offloadingAudio()) {
126        mAudioSink->stop();
127        mAudioSink->flush();
128        mAudioSink->close();
129    }
130}
131
132void NuPlayer::Renderer::queueBuffer(
133        bool audio,
134        const sp<ABuffer> &buffer,
135        const sp<AMessage> &notifyConsumed) {
136    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
137    msg->setInt32("queueGeneration", getQueueGeneration(audio));
138    msg->setInt32("audio", static_cast<int32_t>(audio));
139    msg->setBuffer("buffer", buffer);
140    msg->setMessage("notifyConsumed", notifyConsumed);
141    msg->post();
142}
143
144void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
145    CHECK_NE(finalResult, (status_t)OK);
146
147    sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
148    msg->setInt32("queueGeneration", getQueueGeneration(audio));
149    msg->setInt32("audio", static_cast<int32_t>(audio));
150    msg->setInt32("finalResult", finalResult);
151    msg->post();
152}
153
154status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) {
155    sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
156    writeToAMessage(msg, rate);
157    sp<AMessage> response;
158    status_t err = msg->postAndAwaitResponse(&response);
159    if (err == OK && response != NULL) {
160        CHECK(response->findInt32("err", &err));
161    }
162    return err;
163}
164
165status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) {
166    if (rate.mSpeed == 0.f) {
167        onPause();
168        // don't call audiosink's setPlaybackRate if pausing, as pitch does not
169        // have to correspond to the any non-0 speed (e.g old speed). Keep
170        // settings nonetheless, using the old speed, in case audiosink changes.
171        AudioPlaybackRate newRate = rate;
172        newRate.mSpeed = mPlaybackSettings.mSpeed;
173        mPlaybackSettings = newRate;
174        return OK;
175    }
176
177    if (mAudioSink != NULL && mAudioSink->ready()) {
178        status_t err = mAudioSink->setPlaybackRate(rate);
179        if (err != OK) {
180            return err;
181        }
182    }
183    mPlaybackSettings = rate;
184    mPlaybackRate = rate.mSpeed;
185    mMediaClock->setPlaybackRate(mPlaybackRate);
186    return OK;
187}
188
189status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
190    sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
191    sp<AMessage> response;
192    status_t err = msg->postAndAwaitResponse(&response);
193    if (err == OK && response != NULL) {
194        CHECK(response->findInt32("err", &err));
195        if (err == OK) {
196            readFromAMessage(response, rate);
197        }
198    }
199    return err;
200}
201
202status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
203    if (mAudioSink != NULL && mAudioSink->ready()) {
204        status_t err = mAudioSink->getPlaybackRate(rate);
205        if (err == OK) {
206            if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) {
207                ALOGW("correcting mismatch in internal/external playback rate");
208            }
209            // get playback settings used by audiosink, as it may be
210            // slightly off due to audiosink not taking small changes.
211            mPlaybackSettings = *rate;
212            if (mPaused) {
213                rate->mSpeed = 0.f;
214            }
215        }
216        return err;
217    }
218    *rate = mPlaybackSettings;
219    return OK;
220}
221
222status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
223    sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
224    writeToAMessage(msg, sync, videoFpsHint);
225    sp<AMessage> response;
226    status_t err = msg->postAndAwaitResponse(&response);
227    if (err == OK && response != NULL) {
228        CHECK(response->findInt32("err", &err));
229    }
230    return err;
231}
232
233status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) {
234    if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
235        return BAD_VALUE;
236    }
237    // TODO: support sync sources
238    return INVALID_OPERATION;
239}
240
241status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) {
242    sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
243    sp<AMessage> response;
244    status_t err = msg->postAndAwaitResponse(&response);
245    if (err == OK && response != NULL) {
246        CHECK(response->findInt32("err", &err));
247        if (err == OK) {
248            readFromAMessage(response, sync, videoFps);
249        }
250    }
251    return err;
252}
253
254status_t NuPlayer::Renderer::onGetSyncSettings(
255        AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
256    *sync = mSyncSettings;
257    *videoFps = -1.f;
258    return OK;
259}
260
261void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
262    {
263        Mutex::Autolock autoLock(mLock);
264        if (audio) {
265            mNotifyCompleteAudio |= notifyComplete;
266            ++mAudioQueueGeneration;
267            ++mAudioDrainGeneration;
268        } else {
269            mNotifyCompleteVideo |= notifyComplete;
270            ++mVideoQueueGeneration;
271            ++mVideoDrainGeneration;
272        }
273
274        clearAnchorTime_l();
275        clearAudioFirstAnchorTime_l();
276        mVideoLateByUs = 0;
277        mSyncQueues = false;
278    }
279
280    sp<AMessage> msg = new AMessage(kWhatFlush, this);
281    msg->setInt32("audio", static_cast<int32_t>(audio));
282    msg->post();
283}
284
285void NuPlayer::Renderer::signalTimeDiscontinuity() {
286}
287
288void NuPlayer::Renderer::signalDisableOffloadAudio() {
289    (new AMessage(kWhatDisableOffloadAudio, this))->post();
290}
291
292void NuPlayer::Renderer::signalEnableOffloadAudio() {
293    (new AMessage(kWhatEnableOffloadAudio, this))->post();
294}
295
296void NuPlayer::Renderer::pause() {
297    (new AMessage(kWhatPause, this))->post();
298}
299
300void NuPlayer::Renderer::resume() {
301    (new AMessage(kWhatResume, this))->post();
302}
303
304void NuPlayer::Renderer::setVideoFrameRate(float fps) {
305    sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
306    msg->setFloat("frame-rate", fps);
307    msg->post();
308}
309
310// Called on any threads.
311status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
312    return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
313}
314
315void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
316    mAudioFirstAnchorTimeMediaUs = -1;
317    mMediaClock->setStartingTimeMedia(-1);
318}
319
320void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
321    if (mAudioFirstAnchorTimeMediaUs == -1) {
322        mAudioFirstAnchorTimeMediaUs = mediaUs;
323        mMediaClock->setStartingTimeMedia(mediaUs);
324    }
325}
326
327void NuPlayer::Renderer::clearAnchorTime_l() {
328    mMediaClock->clearAnchor();
329    mAnchorTimeMediaUs = -1;
330    mAnchorNumFramesWritten = -1;
331}
332
333void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
334    Mutex::Autolock autoLock(mLock);
335    mVideoLateByUs = lateUs;
336}
337
338int64_t NuPlayer::Renderer::getVideoLateByUs() {
339    Mutex::Autolock autoLock(mLock);
340    return mVideoLateByUs;
341}
342
343status_t NuPlayer::Renderer::openAudioSink(
344        const sp<AMessage> &format,
345        bool offloadOnly,
346        bool hasVideo,
347        uint32_t flags,
348        bool *isOffloaded) {
349    sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
350    msg->setMessage("format", format);
351    msg->setInt32("offload-only", offloadOnly);
352    msg->setInt32("has-video", hasVideo);
353    msg->setInt32("flags", flags);
354
355    sp<AMessage> response;
356    msg->postAndAwaitResponse(&response);
357
358    int32_t err;
359    if (!response->findInt32("err", &err)) {
360        err = INVALID_OPERATION;
361    } else if (err == OK && isOffloaded != NULL) {
362        int32_t offload;
363        CHECK(response->findInt32("offload", &offload));
364        *isOffloaded = (offload != 0);
365    }
366    return err;
367}
368
369void NuPlayer::Renderer::closeAudioSink() {
370    sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
371
372    sp<AMessage> response;
373    msg->postAndAwaitResponse(&response);
374}
375
376void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
377    switch (msg->what()) {
378        case kWhatOpenAudioSink:
379        {
380            sp<AMessage> format;
381            CHECK(msg->findMessage("format", &format));
382
383            int32_t offloadOnly;
384            CHECK(msg->findInt32("offload-only", &offloadOnly));
385
386            int32_t hasVideo;
387            CHECK(msg->findInt32("has-video", &hasVideo));
388
389            uint32_t flags;
390            CHECK(msg->findInt32("flags", (int32_t *)&flags));
391
392            status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
393
394            sp<AMessage> response = new AMessage;
395            response->setInt32("err", err);
396            response->setInt32("offload", offloadingAudio());
397
398            sp<AReplyToken> replyID;
399            CHECK(msg->senderAwaitsResponse(&replyID));
400            response->postReply(replyID);
401
402            break;
403        }
404
405        case kWhatCloseAudioSink:
406        {
407            sp<AReplyToken> replyID;
408            CHECK(msg->senderAwaitsResponse(&replyID));
409
410            onCloseAudioSink();
411
412            sp<AMessage> response = new AMessage;
413            response->postReply(replyID);
414            break;
415        }
416
417        case kWhatStopAudioSink:
418        {
419            mAudioSink->stop();
420            break;
421        }
422
423        case kWhatDrainAudioQueue:
424        {
425            int32_t generation;
426            CHECK(msg->findInt32("drainGeneration", &generation));
427            if (generation != getDrainGeneration(true /* audio */)) {
428                break;
429            }
430
431            mDrainAudioQueuePending = false;
432
433            if (onDrainAudioQueue()) {
434                uint32_t numFramesPlayed;
435                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
436                         (status_t)OK);
437
438                uint32_t numFramesPendingPlayout =
439                    mNumFramesWritten - numFramesPlayed;
440
441                // This is how long the audio sink will have data to
442                // play back.
443                int64_t delayUs =
444                    mAudioSink->msecsPerFrame()
445                        * numFramesPendingPlayout * 1000ll;
446                if (mPlaybackRate > 1.0f) {
447                    delayUs /= mPlaybackRate;
448                }
449
450                // Let's give it more data after about half that time
451                // has elapsed.
452                Mutex::Autolock autoLock(mLock);
453                postDrainAudioQueue_l(delayUs / 2);
454            }
455            break;
456        }
457
458        case kWhatDrainVideoQueue:
459        {
460            int32_t generation;
461            CHECK(msg->findInt32("drainGeneration", &generation));
462            if (generation != getDrainGeneration(false /* audio */)) {
463                break;
464            }
465
466            mDrainVideoQueuePending = false;
467
468            onDrainVideoQueue();
469
470            postDrainVideoQueue();
471            break;
472        }
473
474        case kWhatPostDrainVideoQueue:
475        {
476            int32_t generation;
477            CHECK(msg->findInt32("drainGeneration", &generation));
478            if (generation != getDrainGeneration(false /* audio */)) {
479                break;
480            }
481
482            mDrainVideoQueuePending = false;
483            postDrainVideoQueue();
484            break;
485        }
486
487        case kWhatQueueBuffer:
488        {
489            onQueueBuffer(msg);
490            break;
491        }
492
493        case kWhatQueueEOS:
494        {
495            onQueueEOS(msg);
496            break;
497        }
498
499        case kWhatConfigPlayback:
500        {
501            sp<AReplyToken> replyID;
502            CHECK(msg->senderAwaitsResponse(&replyID));
503            AudioPlaybackRate rate;
504            readFromAMessage(msg, &rate);
505            status_t err = onConfigPlayback(rate);
506            sp<AMessage> response = new AMessage;
507            response->setInt32("err", err);
508            response->postReply(replyID);
509            break;
510        }
511
512        case kWhatGetPlaybackSettings:
513        {
514            sp<AReplyToken> replyID;
515            CHECK(msg->senderAwaitsResponse(&replyID));
516            AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT;
517            status_t err = onGetPlaybackSettings(&rate);
518            sp<AMessage> response = new AMessage;
519            if (err == OK) {
520                writeToAMessage(response, rate);
521            }
522            response->setInt32("err", err);
523            response->postReply(replyID);
524            break;
525        }
526
527        case kWhatConfigSync:
528        {
529            sp<AReplyToken> replyID;
530            CHECK(msg->senderAwaitsResponse(&replyID));
531            AVSyncSettings sync;
532            float videoFpsHint;
533            readFromAMessage(msg, &sync, &videoFpsHint);
534            status_t err = onConfigSync(sync, videoFpsHint);
535            sp<AMessage> response = new AMessage;
536            response->setInt32("err", err);
537            response->postReply(replyID);
538            break;
539        }
540
541        case kWhatGetSyncSettings:
542        {
543            sp<AReplyToken> replyID;
544            CHECK(msg->senderAwaitsResponse(&replyID));
545
546            ALOGV("kWhatGetSyncSettings");
547            AVSyncSettings sync;
548            float videoFps = -1.f;
549            status_t err = onGetSyncSettings(&sync, &videoFps);
550            sp<AMessage> response = new AMessage;
551            if (err == OK) {
552                writeToAMessage(response, sync, videoFps);
553            }
554            response->setInt32("err", err);
555            response->postReply(replyID);
556            break;
557        }
558
559        case kWhatFlush:
560        {
561            onFlush(msg);
562            break;
563        }
564
565        case kWhatDisableOffloadAudio:
566        {
567            onDisableOffloadAudio();
568            break;
569        }
570
571        case kWhatEnableOffloadAudio:
572        {
573            onEnableOffloadAudio();
574            break;
575        }
576
577        case kWhatPause:
578        {
579            onPause();
580            break;
581        }
582
583        case kWhatResume:
584        {
585            onResume();
586            break;
587        }
588
589        case kWhatSetVideoFrameRate:
590        {
591            float fps;
592            CHECK(msg->findFloat("frame-rate", &fps));
593            onSetVideoFrameRate(fps);
594            break;
595        }
596
597        case kWhatAudioTearDown:
598        {
599            onAudioTearDown(kDueToError);
600            break;
601        }
602
603        case kWhatAudioOffloadPauseTimeout:
604        {
605            int32_t generation;
606            CHECK(msg->findInt32("drainGeneration", &generation));
607            if (generation != mAudioOffloadPauseTimeoutGeneration) {
608                break;
609            }
610            ALOGV("Audio Offload tear down due to pause timeout.");
611            onAudioTearDown(kDueToTimeout);
612            mWakeLock->release();
613            break;
614        }
615
616        default:
617            TRESPASS();
618            break;
619    }
620}
621
622void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
623    if (mDrainAudioQueuePending || mSyncQueues || mPaused
624            || mUseAudioCallback) {
625        return;
626    }
627
628    if (mAudioQueue.empty()) {
629        return;
630    }
631
632    mDrainAudioQueuePending = true;
633    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
634    msg->setInt32("drainGeneration", mAudioDrainGeneration);
635    msg->post(delayUs);
636}
637
638void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
639    mAudioRenderingStartGeneration = mAudioDrainGeneration;
640    mVideoRenderingStartGeneration = mVideoDrainGeneration;
641}
642
643void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
644    if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
645        mAudioRenderingStartGeneration == mAudioDrainGeneration) {
646        mVideoRenderingStartGeneration = -1;
647        mAudioRenderingStartGeneration = -1;
648
649        sp<AMessage> notify = mNotify->dup();
650        notify->setInt32("what", kWhatMediaRenderingStart);
651        notify->post();
652    }
653}
654
655// static
656size_t NuPlayer::Renderer::AudioSinkCallback(
657        MediaPlayerBase::AudioSink * /* audioSink */,
658        void *buffer,
659        size_t size,
660        void *cookie,
661        MediaPlayerBase::AudioSink::cb_event_t event) {
662    NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
663
664    switch (event) {
665        case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
666        {
667            return me->fillAudioBuffer(buffer, size);
668            break;
669        }
670
671        case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
672        {
673            ALOGV("AudioSink::CB_EVENT_STREAM_END");
674            me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
675            break;
676        }
677
678        case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
679        {
680            ALOGV("AudioSink::CB_EVENT_TEAR_DOWN");
681            me->notifyAudioTearDown();
682            break;
683        }
684    }
685
686    return 0;
687}
688
689size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
690    Mutex::Autolock autoLock(mLock);
691
692    if (!mUseAudioCallback || mPaused) {
693        return 0;
694    }
695
696    bool hasEOS = false;
697
698    size_t sizeCopied = 0;
699    bool firstEntry = true;
700    QueueEntry *entry;  // will be valid after while loop if hasEOS is set.
701    while (sizeCopied < size && !mAudioQueue.empty()) {
702        entry = &*mAudioQueue.begin();
703
704        if (entry->mBuffer == NULL) { // EOS
705            hasEOS = true;
706            mAudioQueue.erase(mAudioQueue.begin());
707            break;
708        }
709
710        if (firstEntry && entry->mOffset == 0) {
711            firstEntry = false;
712            int64_t mediaTimeUs;
713            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
714            ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
715            setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
716        }
717
718        size_t copy = entry->mBuffer->size() - entry->mOffset;
719        size_t sizeRemaining = size - sizeCopied;
720        if (copy > sizeRemaining) {
721            copy = sizeRemaining;
722        }
723
724        memcpy((char *)buffer + sizeCopied,
725               entry->mBuffer->data() + entry->mOffset,
726               copy);
727
728        entry->mOffset += copy;
729        if (entry->mOffset == entry->mBuffer->size()) {
730            entry->mNotifyConsumed->post();
731            mAudioQueue.erase(mAudioQueue.begin());
732            entry = NULL;
733        }
734        sizeCopied += copy;
735
736        notifyIfMediaRenderingStarted_l();
737    }
738
739    if (mAudioFirstAnchorTimeMediaUs >= 0) {
740        int64_t nowUs = ALooper::GetNowUs();
741        int64_t nowMediaUs =
742            mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs);
743        // we don't know how much data we are queueing for offloaded tracks.
744        mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
745    }
746
747    // for non-offloaded audio, we need to compute the frames written because
748    // there is no EVENT_STREAM_END notification. The frames written gives
749    // an estimate on the pending played out duration.
750    if (!offloadingAudio()) {
751        mNumFramesWritten += sizeCopied / mAudioSink->frameSize();
752    }
753
754    if (hasEOS) {
755        (new AMessage(kWhatStopAudioSink, this))->post();
756        // As there is currently no EVENT_STREAM_END callback notification for
757        // non-offloaded audio tracks, we need to post the EOS ourselves.
758        if (!offloadingAudio()) {
759            int64_t postEOSDelayUs = 0;
760            if (mAudioSink->needsTrailingPadding()) {
761                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
762            }
763            ALOGV("fillAudioBuffer: notifyEOS "
764                    "mNumFramesWritten:%u  finalResult:%d  postEOSDelay:%lld",
765                    mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs);
766            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
767        }
768    }
769    return sizeCopied;
770}
771
772bool NuPlayer::Renderer::onDrainAudioQueue() {
773    // TODO: This call to getPosition checks if AudioTrack has been created
774    // in AudioSink before draining audio. If AudioTrack doesn't exist, then
775    // CHECKs on getPosition will fail.
776    // We still need to figure out why AudioTrack is not created when
777    // this function is called. One possible reason could be leftover
778    // audio. Another possible place is to check whether decoder
779    // has received INFO_FORMAT_CHANGED as the first buffer since
780    // AudioSink is opened there, and possible interactions with flush
781    // immediately after start. Investigate error message
782    // "vorbis_dsp_synthesis returned -135", along with RTSP.
783    uint32_t numFramesPlayed;
784    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
785        return false;
786    }
787
788#if 0
789    ssize_t numFramesAvailableToWrite =
790        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
791
792    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
793        ALOGI("audio sink underrun");
794    } else {
795        ALOGV("audio queue has %d frames left to play",
796             mAudioSink->frameCount() - numFramesAvailableToWrite);
797    }
798#endif
799
800    while (!mAudioQueue.empty()) {
801        QueueEntry *entry = &*mAudioQueue.begin();
802
803        mLastAudioBufferDrained = entry->mBufferOrdinal;
804
805        if (entry->mBuffer == NULL) {
806            // EOS
807            int64_t postEOSDelayUs = 0;
808            if (mAudioSink->needsTrailingPadding()) {
809                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
810            }
811            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
812
813            mAudioQueue.erase(mAudioQueue.begin());
814            entry = NULL;
815            if (mAudioSink->needsTrailingPadding()) {
816                // If we're not in gapless playback (i.e. through setNextPlayer), we
817                // need to stop the track here, because that will play out the last
818                // little bit at the end of the file. Otherwise short files won't play.
819                mAudioSink->stop();
820                mNumFramesWritten = 0;
821            }
822            return false;
823        }
824
825        // ignore 0-sized buffer which could be EOS marker with no data
826        if (entry->mOffset == 0 && entry->mBuffer->size() > 0) {
827            int64_t mediaTimeUs;
828            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
829            ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs",
830                    mediaTimeUs / 1E6);
831            onNewAudioMediaTime(mediaTimeUs);
832        }
833
834        size_t copy = entry->mBuffer->size() - entry->mOffset;
835
836        ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
837                                            copy, false /* blocking */);
838        if (written < 0) {
839            // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
840            if (written == WOULD_BLOCK) {
841                ALOGW("AudioSink write would block when writing %zu bytes", copy);
842            } else {
843                ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
844                notifyAudioTearDown();
845            }
846            break;
847        }
848
849        entry->mOffset += written;
850        if (entry->mOffset == entry->mBuffer->size()) {
851            entry->mNotifyConsumed->post();
852            mAudioQueue.erase(mAudioQueue.begin());
853
854            entry = NULL;
855        }
856
857        size_t copiedFrames = written / mAudioSink->frameSize();
858        mNumFramesWritten += copiedFrames;
859
860        {
861            Mutex::Autolock autoLock(mLock);
862            notifyIfMediaRenderingStarted_l();
863        }
864
865        if (written != (ssize_t)copy) {
866            // A short count was received from AudioSink::write()
867            //
868            // AudioSink write is called in non-blocking mode.
869            // It may return with a short count when:
870            //
871            // 1) Size to be copied is not a multiple of the frame size. We consider this fatal.
872            // 2) The data to be copied exceeds the available buffer in AudioSink.
873            // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
874            // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
875
876            // (Case 1)
877            // Must be a multiple of the frame size.  If it is not a multiple of a frame size, it
878            // needs to fail, as we should not carry over fractional frames between calls.
879            CHECK_EQ(copy % mAudioSink->frameSize(), 0);
880
881            // (Case 2, 3, 4)
882            // Return early to the caller.
883            // Beware of calling immediately again as this may busy-loop if you are not careful.
884            ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
885            break;
886        }
887    }
888    int64_t maxTimeMedia;
889    {
890        Mutex::Autolock autoLock(mLock);
891        maxTimeMedia =
892            mAnchorTimeMediaUs +
893                    (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
894                            * 1000LL * mAudioSink->msecsPerFrame());
895    }
896    mMediaClock->updateMaxTimeMedia(maxTimeMedia);
897
898    return !mAudioQueue.empty();
899}
900
901int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
902    int32_t sampleRate = offloadingAudio() ?
903            mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
904    // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
905    return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate);
906}
907
908// Calculate duration of pending samples if played at normal rate (i.e., 1.0).
909int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
910    int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
911    return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs);
912}
913
914int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
915    int64_t realUs;
916    if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
917        // If failed to get current position, e.g. due to audio clock is
918        // not ready, then just play out video immediately without delay.
919        return nowUs;
920    }
921    return realUs;
922}
923
924void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
925    Mutex::Autolock autoLock(mLock);
926    // TRICKY: vorbis decoder generates multiple frames with the same
927    // timestamp, so only update on the first frame with a given timestamp
928    if (mediaTimeUs == mAnchorTimeMediaUs) {
929        return;
930    }
931    setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
932    int64_t nowUs = ALooper::GetNowUs();
933    int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
934    mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
935    mAnchorNumFramesWritten = mNumFramesWritten;
936    mAnchorTimeMediaUs = mediaTimeUs;
937}
938
939// Called without mLock acquired.
940void NuPlayer::Renderer::postDrainVideoQueue() {
941    if (mDrainVideoQueuePending
942            || getSyncQueues()
943            || (mPaused && mVideoSampleReceived)) {
944        return;
945    }
946
947    if (mVideoQueue.empty()) {
948        return;
949    }
950
951    QueueEntry &entry = *mVideoQueue.begin();
952
953    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
954    msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
955
956    if (entry.mBuffer == NULL) {
957        // EOS doesn't carry a timestamp.
958        msg->post();
959        mDrainVideoQueuePending = true;
960        return;
961    }
962
963    int64_t delayUs;
964    int64_t nowUs = ALooper::GetNowUs();
965    int64_t realTimeUs;
966    if (mFlags & FLAG_REAL_TIME) {
967        int64_t mediaTimeUs;
968        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
969        realTimeUs = mediaTimeUs;
970    } else {
971        int64_t mediaTimeUs;
972        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
973
974        {
975            Mutex::Autolock autoLock(mLock);
976            if (mAnchorTimeMediaUs < 0) {
977                mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
978                mAnchorTimeMediaUs = mediaTimeUs;
979                realTimeUs = nowUs;
980            } else {
981                realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
982            }
983        }
984        if (!mHasAudio) {
985            // smooth out videos >= 10fps
986            mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
987        }
988
989        // Heuristics to handle situation when media time changed without a
990        // discontinuity. If we have not drained an audio buffer that was
991        // received after this buffer, repost in 10 msec. Otherwise repost
992        // in 500 msec.
993        delayUs = realTimeUs - nowUs;
994        if (delayUs > 500000) {
995            int64_t postDelayUs = 500000;
996            if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) {
997                postDelayUs = 10000;
998            }
999            msg->setWhat(kWhatPostDrainVideoQueue);
1000            msg->post(postDelayUs);
1001            mVideoScheduler->restart();
1002            ALOGI("possible video time jump of %dms, retrying in %dms",
1003                    (int)(delayUs / 1000), (int)(postDelayUs / 1000));
1004            mDrainVideoQueuePending = true;
1005            return;
1006        }
1007    }
1008
1009    realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
1010    int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
1011
1012    delayUs = realTimeUs - nowUs;
1013
1014    ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
1015    // post 2 display refreshes before rendering is due
1016    msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
1017
1018    mDrainVideoQueuePending = true;
1019}
1020
1021void NuPlayer::Renderer::onDrainVideoQueue() {
1022    if (mVideoQueue.empty()) {
1023        return;
1024    }
1025
1026    QueueEntry *entry = &*mVideoQueue.begin();
1027
1028    if (entry->mBuffer == NULL) {
1029        // EOS
1030
1031        notifyEOS(false /* audio */, entry->mFinalResult);
1032
1033        mVideoQueue.erase(mVideoQueue.begin());
1034        entry = NULL;
1035
1036        setVideoLateByUs(0);
1037        return;
1038    }
1039
1040    int64_t nowUs = -1;
1041    int64_t realTimeUs;
1042    if (mFlags & FLAG_REAL_TIME) {
1043        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
1044    } else {
1045        int64_t mediaTimeUs;
1046        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1047
1048        nowUs = ALooper::GetNowUs();
1049        realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1050    }
1051
1052    bool tooLate = false;
1053
1054    if (!mPaused) {
1055        if (nowUs == -1) {
1056            nowUs = ALooper::GetNowUs();
1057        }
1058        setVideoLateByUs(nowUs - realTimeUs);
1059        tooLate = (mVideoLateByUs > 40000);
1060
1061        if (tooLate) {
1062            ALOGV("video late by %lld us (%.2f secs)",
1063                 (long long)mVideoLateByUs, mVideoLateByUs / 1E6);
1064        } else {
1065            int64_t mediaUs = 0;
1066            mMediaClock->getMediaTime(realTimeUs, &mediaUs);
1067            ALOGV("rendering video at media time %.2f secs",
1068                    (mFlags & FLAG_REAL_TIME ? realTimeUs :
1069                    mediaUs) / 1E6);
1070        }
1071    } else {
1072        setVideoLateByUs(0);
1073        if (!mVideoSampleReceived && !mHasAudio) {
1074            // This will ensure that the first frame after a flush won't be used as anchor
1075            // when renderer is in paused state, because resume can happen any time after seek.
1076            Mutex::Autolock autoLock(mLock);
1077            clearAnchorTime_l();
1078        }
1079    }
1080
1081    entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
1082    entry->mNotifyConsumed->setInt32("render", !tooLate);
1083    entry->mNotifyConsumed->post();
1084    mVideoQueue.erase(mVideoQueue.begin());
1085    entry = NULL;
1086
1087    mVideoSampleReceived = true;
1088
1089    if (!mPaused) {
1090        if (!mVideoRenderingStarted) {
1091            mVideoRenderingStarted = true;
1092            notifyVideoRenderingStart();
1093        }
1094        Mutex::Autolock autoLock(mLock);
1095        notifyIfMediaRenderingStarted_l();
1096    }
1097}
1098
1099void NuPlayer::Renderer::notifyVideoRenderingStart() {
1100    sp<AMessage> notify = mNotify->dup();
1101    notify->setInt32("what", kWhatVideoRenderingStart);
1102    notify->post();
1103}
1104
1105void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
1106    sp<AMessage> notify = mNotify->dup();
1107    notify->setInt32("what", kWhatEOS);
1108    notify->setInt32("audio", static_cast<int32_t>(audio));
1109    notify->setInt32("finalResult", finalResult);
1110    notify->post(delayUs);
1111}
1112
1113void NuPlayer::Renderer::notifyAudioTearDown() {
1114    (new AMessage(kWhatAudioTearDown, this))->post();
1115}
1116
1117void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
1118    int32_t audio;
1119    CHECK(msg->findInt32("audio", &audio));
1120
1121    if (dropBufferIfStale(audio, msg)) {
1122        return;
1123    }
1124
1125    if (audio) {
1126        mHasAudio = true;
1127    } else {
1128        mHasVideo = true;
1129    }
1130
1131    if (mHasVideo) {
1132        if (mVideoScheduler == NULL) {
1133            mVideoScheduler = new VideoFrameScheduler();
1134            mVideoScheduler->init();
1135        }
1136    }
1137
1138    sp<ABuffer> buffer;
1139    CHECK(msg->findBuffer("buffer", &buffer));
1140
1141    sp<AMessage> notifyConsumed;
1142    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
1143
1144    QueueEntry entry;
1145    entry.mBuffer = buffer;
1146    entry.mNotifyConsumed = notifyConsumed;
1147    entry.mOffset = 0;
1148    entry.mFinalResult = OK;
1149    entry.mBufferOrdinal = ++mTotalBuffersQueued;
1150
1151    if (audio) {
1152        Mutex::Autolock autoLock(mLock);
1153        mAudioQueue.push_back(entry);
1154        postDrainAudioQueue_l();
1155    } else {
1156        mVideoQueue.push_back(entry);
1157        postDrainVideoQueue();
1158    }
1159
1160    Mutex::Autolock autoLock(mLock);
1161    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
1162        return;
1163    }
1164
1165    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
1166    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
1167
1168    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
1169        // EOS signalled on either queue.
1170        syncQueuesDone_l();
1171        return;
1172    }
1173
1174    int64_t firstAudioTimeUs;
1175    int64_t firstVideoTimeUs;
1176    CHECK(firstAudioBuffer->meta()
1177            ->findInt64("timeUs", &firstAudioTimeUs));
1178    CHECK(firstVideoBuffer->meta()
1179            ->findInt64("timeUs", &firstVideoTimeUs));
1180
1181    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
1182
1183    ALOGV("queueDiff = %.2f secs", diff / 1E6);
1184
1185    if (diff > 100000ll) {
1186        // Audio data starts More than 0.1 secs before video.
1187        // Drop some audio.
1188
1189        (*mAudioQueue.begin()).mNotifyConsumed->post();
1190        mAudioQueue.erase(mAudioQueue.begin());
1191        return;
1192    }
1193
1194    syncQueuesDone_l();
1195}
1196
1197void NuPlayer::Renderer::syncQueuesDone_l() {
1198    if (!mSyncQueues) {
1199        return;
1200    }
1201
1202    mSyncQueues = false;
1203
1204    if (!mAudioQueue.empty()) {
1205        postDrainAudioQueue_l();
1206    }
1207
1208    if (!mVideoQueue.empty()) {
1209        mLock.unlock();
1210        postDrainVideoQueue();
1211        mLock.lock();
1212    }
1213}
1214
1215void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1216    int32_t audio;
1217    CHECK(msg->findInt32("audio", &audio));
1218
1219    if (dropBufferIfStale(audio, msg)) {
1220        return;
1221    }
1222
1223    int32_t finalResult;
1224    CHECK(msg->findInt32("finalResult", &finalResult));
1225
1226    QueueEntry entry;
1227    entry.mOffset = 0;
1228    entry.mFinalResult = finalResult;
1229
1230    if (audio) {
1231        Mutex::Autolock autoLock(mLock);
1232        if (mAudioQueue.empty() && mSyncQueues) {
1233            syncQueuesDone_l();
1234        }
1235        mAudioQueue.push_back(entry);
1236        postDrainAudioQueue_l();
1237    } else {
1238        if (mVideoQueue.empty() && getSyncQueues()) {
1239            Mutex::Autolock autoLock(mLock);
1240            syncQueuesDone_l();
1241        }
1242        mVideoQueue.push_back(entry);
1243        postDrainVideoQueue();
1244    }
1245}
1246
1247void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
1248    int32_t audio, notifyComplete;
1249    CHECK(msg->findInt32("audio", &audio));
1250
1251    {
1252        Mutex::Autolock autoLock(mLock);
1253        if (audio) {
1254            notifyComplete = mNotifyCompleteAudio;
1255            mNotifyCompleteAudio = false;
1256        } else {
1257            notifyComplete = mNotifyCompleteVideo;
1258            mNotifyCompleteVideo = false;
1259        }
1260
1261        // If we're currently syncing the queues, i.e. dropping audio while
1262        // aligning the first audio/video buffer times and only one of the
1263        // two queues has data, we may starve that queue by not requesting
1264        // more buffers from the decoder. If the other source then encounters
1265        // a discontinuity that leads to flushing, we'll never find the
1266        // corresponding discontinuity on the other queue.
1267        // Therefore we'll stop syncing the queues if at least one of them
1268        // is flushed.
1269        syncQueuesDone_l();
1270        clearAnchorTime_l();
1271    }
1272
1273    ALOGV("flushing %s", audio ? "audio" : "video");
1274    if (audio) {
1275        {
1276            Mutex::Autolock autoLock(mLock);
1277            flushQueue(&mAudioQueue);
1278
1279            ++mAudioDrainGeneration;
1280            prepareForMediaRenderingStart_l();
1281
1282            // the frame count will be reset after flush.
1283            clearAudioFirstAnchorTime_l();
1284        }
1285
1286        mDrainAudioQueuePending = false;
1287
1288        if (offloadingAudio()) {
1289            mAudioSink->pause();
1290            mAudioSink->flush();
1291            mAudioSink->start();
1292        } else {
1293            mAudioSink->pause();
1294            mAudioSink->flush();
1295            // Call stop() to signal to the AudioSink to completely fill the
1296            // internal buffer before resuming playback.
1297            mAudioSink->stop();
1298            if (!mPaused) {
1299                mAudioSink->start();
1300            }
1301            mNumFramesWritten = 0;
1302        }
1303    } else {
1304        flushQueue(&mVideoQueue);
1305
1306        mDrainVideoQueuePending = false;
1307
1308        if (mVideoScheduler != NULL) {
1309            mVideoScheduler->restart();
1310        }
1311
1312        Mutex::Autolock autoLock(mLock);
1313        ++mVideoDrainGeneration;
1314        prepareForMediaRenderingStart_l();
1315    }
1316
1317    mVideoSampleReceived = false;
1318
1319    if (notifyComplete) {
1320        notifyFlushComplete(audio);
1321    }
1322}
1323
1324void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1325    while (!queue->empty()) {
1326        QueueEntry *entry = &*queue->begin();
1327
1328        if (entry->mBuffer != NULL) {
1329            entry->mNotifyConsumed->post();
1330        }
1331
1332        queue->erase(queue->begin());
1333        entry = NULL;
1334    }
1335}
1336
1337void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1338    sp<AMessage> notify = mNotify->dup();
1339    notify->setInt32("what", kWhatFlushComplete);
1340    notify->setInt32("audio", static_cast<int32_t>(audio));
1341    notify->post();
1342}
1343
1344bool NuPlayer::Renderer::dropBufferIfStale(
1345        bool audio, const sp<AMessage> &msg) {
1346    int32_t queueGeneration;
1347    CHECK(msg->findInt32("queueGeneration", &queueGeneration));
1348
1349    if (queueGeneration == getQueueGeneration(audio)) {
1350        return false;
1351    }
1352
1353    sp<AMessage> notifyConsumed;
1354    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1355        notifyConsumed->post();
1356    }
1357
1358    return true;
1359}
1360
1361void NuPlayer::Renderer::onAudioSinkChanged() {
1362    if (offloadingAudio()) {
1363        return;
1364    }
1365    CHECK(!mDrainAudioQueuePending);
1366    mNumFramesWritten = 0;
1367    {
1368        Mutex::Autolock autoLock(mLock);
1369        mAnchorNumFramesWritten = -1;
1370    }
1371    uint32_t written;
1372    if (mAudioSink->getFramesWritten(&written) == OK) {
1373        mNumFramesWritten = written;
1374    }
1375}
1376
1377void NuPlayer::Renderer::onDisableOffloadAudio() {
1378    Mutex::Autolock autoLock(mLock);
1379    mFlags &= ~FLAG_OFFLOAD_AUDIO;
1380    ++mAudioDrainGeneration;
1381}
1382
1383void NuPlayer::Renderer::onEnableOffloadAudio() {
1384    Mutex::Autolock autoLock(mLock);
1385    mFlags |= FLAG_OFFLOAD_AUDIO;
1386    ++mAudioDrainGeneration;
1387}
1388
1389void NuPlayer::Renderer::onPause() {
1390    if (mPaused) {
1391        return;
1392    }
1393
1394    {
1395        Mutex::Autolock autoLock(mLock);
1396        ++mAudioDrainGeneration;
1397        ++mVideoDrainGeneration;
1398        prepareForMediaRenderingStart_l();
1399        mPaused = true;
1400        mMediaClock->setPlaybackRate(0.0);
1401    }
1402
1403    mDrainAudioQueuePending = false;
1404    mDrainVideoQueuePending = false;
1405
1406    if (mHasAudio) {
1407        mAudioSink->pause();
1408        startAudioOffloadPauseTimeout();
1409    }
1410
1411    ALOGV("now paused audio queue has %zu entries, video has %zu entries",
1412          mAudioQueue.size(), mVideoQueue.size());
1413}
1414
1415void NuPlayer::Renderer::onResume() {
1416    if (!mPaused) {
1417        return;
1418    }
1419
1420    if (mHasAudio) {
1421        cancelAudioOffloadPauseTimeout();
1422        mAudioSink->start();
1423    }
1424
1425    {
1426        Mutex::Autolock autoLock(mLock);
1427        mPaused = false;
1428
1429        // configure audiosink as we did not do it when pausing
1430        if (mAudioSink != NULL && mAudioSink->ready()) {
1431            mAudioSink->setPlaybackRate(mPlaybackSettings);
1432        }
1433
1434        mMediaClock->setPlaybackRate(mPlaybackRate);
1435
1436        if (!mAudioQueue.empty()) {
1437            postDrainAudioQueue_l();
1438        }
1439    }
1440
1441    if (!mVideoQueue.empty()) {
1442        postDrainVideoQueue();
1443    }
1444}
1445
1446void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1447    if (mVideoScheduler == NULL) {
1448        mVideoScheduler = new VideoFrameScheduler();
1449    }
1450    mVideoScheduler->init(fps);
1451}
1452
1453int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
1454    Mutex::Autolock autoLock(mLock);
1455    return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
1456}
1457
1458int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
1459    Mutex::Autolock autoLock(mLock);
1460    return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
1461}
1462
1463bool NuPlayer::Renderer::getSyncQueues() {
1464    Mutex::Autolock autoLock(mLock);
1465    return mSyncQueues;
1466}
1467
1468// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs()
1469// as it acquires locks and may query the audio driver.
1470//
1471// Some calls could conceivably retrieve extrapolated data instead of
1472// accessing getTimestamp() or getPosition() every time a data buffer with
1473// a media time is received.
1474//
1475// Calculate duration of played samples if played at normal rate (i.e., 1.0).
1476int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) {
1477    uint32_t numFramesPlayed;
1478    int64_t numFramesPlayedAt;
1479    AudioTimestamp ts;
1480    static const int64_t kStaleTimestamp100ms = 100000;
1481
1482    status_t res = mAudioSink->getTimestamp(ts);
1483    if (res == OK) {                 // case 1: mixing audio tracks and offloaded tracks.
1484        numFramesPlayed = ts.mPosition;
1485        numFramesPlayedAt =
1486            ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
1487        const int64_t timestampAge = nowUs - numFramesPlayedAt;
1488        if (timestampAge > kStaleTimestamp100ms) {
1489            // This is an audio FIXME.
1490            // getTimestamp returns a timestamp which may come from audio mixing threads.
1491            // After pausing, the MixerThread may go idle, thus the mTime estimate may
1492            // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms,
1493            // the max latency should be about 25ms with an average around 12ms (to be verified).
1494            // For safety we use 100ms.
1495            ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)",
1496                    (long long)nowUs, (long long)numFramesPlayedAt);
1497            numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
1498        }
1499        //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1500    } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track
1501        numFramesPlayed = 0;
1502        numFramesPlayedAt = nowUs;
1503        //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
1504        //        numFramesPlayed, (long long)numFramesPlayedAt);
1505    } else {                         // case 3: transitory at new track or audio fast tracks.
1506        res = mAudioSink->getPosition(&numFramesPlayed);
1507        CHECK_EQ(res, (status_t)OK);
1508        numFramesPlayedAt = nowUs;
1509        numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */
1510        //ALOGD("getPosition: %u %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1511    }
1512
1513    //CHECK_EQ(numFramesPlayed & (1 << 31), 0);  // can't be negative until 12.4 hrs, test
1514    int64_t durationUs = getDurationUsIfPlayedAtSampleRate(numFramesPlayed)
1515            + nowUs - numFramesPlayedAt;
1516    if (durationUs < 0) {
1517        // Occurs when numFramesPlayed position is very small and the following:
1518        // (1) In case 1, the time nowUs is computed before getTimestamp() is called and
1519        //     numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed.
1520        // (2) In case 3, using getPosition and adding mAudioSink->latency() to
1521        //     numFramesPlayedAt, by a time amount greater than numFramesPlayed.
1522        //
1523        // Both of these are transitory conditions.
1524        ALOGV("getPlayedOutAudioDurationUs: negative duration %lld set to zero", (long long)durationUs);
1525        durationUs = 0;
1526    }
1527    ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)",
1528            (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt);
1529    return durationUs;
1530}
1531
1532void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) {
1533    if (mAudioTornDown) {
1534        return;
1535    }
1536    mAudioTornDown = true;
1537
1538    int64_t currentPositionUs;
1539    if (getCurrentPosition(&currentPositionUs) != OK) {
1540        currentPositionUs = 0;
1541    }
1542
1543    mAudioSink->stop();
1544    mAudioSink->flush();
1545
1546    sp<AMessage> notify = mNotify->dup();
1547    notify->setInt32("what", kWhatAudioTearDown);
1548    notify->setInt64("positionUs", currentPositionUs);
1549    notify->setInt32("reason", reason);
1550    notify->post();
1551}
1552
1553void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1554    if (offloadingAudio()) {
1555        mWakeLock->acquire();
1556        sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
1557        msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
1558        msg->post(kOffloadPauseMaxUs);
1559    }
1560}
1561
1562void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
1563    if (offloadingAudio()) {
1564        mWakeLock->release(true);
1565        ++mAudioOffloadPauseTimeoutGeneration;
1566    }
1567}
1568
1569status_t NuPlayer::Renderer::onOpenAudioSink(
1570        const sp<AMessage> &format,
1571        bool offloadOnly,
1572        bool hasVideo,
1573        uint32_t flags) {
1574    ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1575            offloadOnly, offloadingAudio());
1576    bool audioSinkChanged = false;
1577
1578    int32_t numChannels;
1579    CHECK(format->findInt32("channel-count", &numChannels));
1580
1581    int32_t channelMask;
1582    if (!format->findInt32("channel-mask", &channelMask)) {
1583        // signal to the AudioSink to derive the mask from count.
1584        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1585    }
1586
1587    int32_t sampleRate;
1588    CHECK(format->findInt32("sample-rate", &sampleRate));
1589
1590    if (offloadingAudio()) {
1591        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1592        AString mime;
1593        CHECK(format->findString("mime", &mime));
1594        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1595
1596        if (err != OK) {
1597            ALOGE("Couldn't map mime \"%s\" to a valid "
1598                    "audio_format", mime.c_str());
1599            onDisableOffloadAudio();
1600        } else {
1601            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1602                    mime.c_str(), audioFormat);
1603
1604            int avgBitRate = -1;
1605            format->findInt32("bit-rate", &avgBitRate);
1606
1607            int32_t aacProfile = -1;
1608            if (audioFormat == AUDIO_FORMAT_AAC
1609                    && format->findInt32("aac-profile", &aacProfile)) {
1610                // Redefine AAC format as per aac profile
1611                mapAACProfileToAudioFormat(
1612                        audioFormat,
1613                        aacProfile);
1614            }
1615
1616            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1617            offloadInfo.duration_us = -1;
1618            format->findInt64(
1619                    "durationUs", &offloadInfo.duration_us);
1620            offloadInfo.sample_rate = sampleRate;
1621            offloadInfo.channel_mask = channelMask;
1622            offloadInfo.format = audioFormat;
1623            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1624            offloadInfo.bit_rate = avgBitRate;
1625            offloadInfo.has_video = hasVideo;
1626            offloadInfo.is_streaming = true;
1627
1628            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1629                ALOGV("openAudioSink: no change in offload mode");
1630                // no change from previous configuration, everything ok.
1631                return OK;
1632            }
1633            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1634
1635            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1636            uint32_t offloadFlags = flags;
1637            offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1638            offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1639            audioSinkChanged = true;
1640            mAudioSink->close();
1641
1642            err = mAudioSink->open(
1643                    sampleRate,
1644                    numChannels,
1645                    (audio_channel_mask_t)channelMask,
1646                    audioFormat,
1647                    0 /* bufferCount - unused */,
1648                    &NuPlayer::Renderer::AudioSinkCallback,
1649                    this,
1650                    (audio_output_flags_t)offloadFlags,
1651                    &offloadInfo);
1652
1653            if (err == OK) {
1654                err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1655            }
1656
1657            if (err == OK) {
1658                // If the playback is offloaded to h/w, we pass
1659                // the HAL some metadata information.
1660                // We don't want to do this for PCM because it
1661                // will be going through the AudioFlinger mixer
1662                // before reaching the hardware.
1663                // TODO
1664                mCurrentOffloadInfo = offloadInfo;
1665                err = mAudioSink->start();
1666                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1667            }
1668            if (err != OK) {
1669                // Clean up, fall back to non offload mode.
1670                mAudioSink->close();
1671                onDisableOffloadAudio();
1672                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1673                ALOGV("openAudioSink: offload failed");
1674            }
1675            mUseAudioCallback = true;  // offload mode transfers data through callback
1676        }
1677    }
1678    if (!offloadOnly && !offloadingAudio()) {
1679        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1680        uint32_t pcmFlags = flags;
1681        pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1682
1683        const PcmInfo info = {
1684                (audio_channel_mask_t)channelMask,
1685                (audio_output_flags_t)pcmFlags,
1686                AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat
1687                numChannels,
1688                sampleRate
1689        };
1690        if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
1691            ALOGV("openAudioSink: no change in pcm mode");
1692            // no change from previous configuration, everything ok.
1693            return OK;
1694        }
1695
1696        audioSinkChanged = true;
1697        mAudioSink->close();
1698        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1699        // Note: It is possible to set up the callback, but not use it to send audio data.
1700        // This requires a fix in AudioSink to explicitly specify the transfer mode.
1701        mUseAudioCallback = getUseAudioCallbackSetting();
1702
1703        // Compute the desired buffer size.
1704        // For callback mode, the amount of time before wakeup is about half the buffer size.
1705        const uint32_t frameCount =
1706                (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000;
1707
1708        status_t err = mAudioSink->open(
1709                    sampleRate,
1710                    numChannels,
1711                    (audio_channel_mask_t)channelMask,
1712                    AUDIO_FORMAT_PCM_16_BIT,
1713                    0 /* bufferCount - unused */,
1714                    mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL,
1715                    mUseAudioCallback ? this : NULL,
1716                    (audio_output_flags_t)pcmFlags,
1717                    NULL,
1718                    true /* doNotReconnect */,
1719                    frameCount);
1720        if (err == OK) {
1721            err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1722        }
1723        if (err != OK) {
1724            ALOGW("openAudioSink: non offloaded open failed status: %d", err);
1725            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1726            return err;
1727        }
1728        mCurrentPcmInfo = info;
1729        mAudioSink->start();
1730    }
1731    if (audioSinkChanged) {
1732        onAudioSinkChanged();
1733    }
1734    mAudioTornDown = false;
1735    return OK;
1736}
1737
1738void NuPlayer::Renderer::onCloseAudioSink() {
1739    mAudioSink->close();
1740    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1741    mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1742}
1743
1744}  // namespace android
1745
1746