NuPlayerRenderer.cpp revision d9c2e9c81a6f75c4dd6818a3d5075a875d25a2d4 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26#include <media/stagefright/foundation/AUtils.h>
27#include <media/stagefright/foundation/AWakeLock.h>
28#include <media/stagefright/MediaClock.h>
29#include <media/stagefright/MediaErrors.h>
30#include <media/stagefright/MetaData.h>
31#include <media/stagefright/Utils.h>
32
33#include <VideoFrameScheduler.h>
34
35#include <inttypes.h>
36
37namespace android {
38
39// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
40// is closed to allow the audio DSP to power down.
41static const int64_t kOffloadPauseMaxUs = 10000000ll;
42
43// static
44const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
45        AUDIO_CHANNEL_NONE,
46        AUDIO_OUTPUT_FLAG_NONE,
47        AUDIO_FORMAT_INVALID,
48        0, // mNumChannels
49        0 // mSampleRate
50};
51
52// static
53const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
54
55NuPlayer::Renderer::Renderer(
56        const sp<MediaPlayerBase::AudioSink> &sink,
57        const sp<AMessage> &notify,
58        uint32_t flags)
59    : mAudioSink(sink),
60      mNotify(notify),
61      mFlags(flags),
62      mNumFramesWritten(0),
63      mDrainAudioQueuePending(false),
64      mDrainVideoQueuePending(false),
65      mAudioQueueGeneration(0),
66      mVideoQueueGeneration(0),
67      mAudioDrainGeneration(0),
68      mVideoDrainGeneration(0),
69      mPlaybackRate(1.0),
70      mAudioFirstAnchorTimeMediaUs(-1),
71      mAnchorTimeMediaUs(-1),
72      mAnchorNumFramesWritten(-1),
73      mVideoLateByUs(0ll),
74      mHasAudio(false),
75      mHasVideo(false),
76      mNotifyCompleteAudio(false),
77      mNotifyCompleteVideo(false),
78      mSyncQueues(false),
79      mPaused(false),
80      mVideoSampleReceived(false),
81      mVideoRenderingStarted(false),
82      mVideoRenderingStartGeneration(0),
83      mAudioRenderingStartGeneration(0),
84      mAudioOffloadPauseTimeoutGeneration(0),
85      mAudioOffloadTornDown(false),
86      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
87      mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
88      mTotalBuffersQueued(0),
89      mLastAudioBufferDrained(0),
90      mWakeLock(new AWakeLock()) {
91    mMediaClock = new MediaClock;
92}
93
94NuPlayer::Renderer::~Renderer() {
95    if (offloadingAudio()) {
96        mAudioSink->stop();
97        mAudioSink->flush();
98        mAudioSink->close();
99    }
100}
101
102void NuPlayer::Renderer::queueBuffer(
103        bool audio,
104        const sp<ABuffer> &buffer,
105        const sp<AMessage> &notifyConsumed) {
106    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
107    msg->setInt32("queueGeneration", getQueueGeneration(audio));
108    msg->setInt32("audio", static_cast<int32_t>(audio));
109    msg->setBuffer("buffer", buffer);
110    msg->setMessage("notifyConsumed", notifyConsumed);
111    msg->post();
112}
113
114void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
115    CHECK_NE(finalResult, (status_t)OK);
116
117    sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
118    msg->setInt32("queueGeneration", getQueueGeneration(audio));
119    msg->setInt32("audio", static_cast<int32_t>(audio));
120    msg->setInt32("finalResult", finalResult);
121    msg->post();
122}
123
124void NuPlayer::Renderer::setPlaybackRate(float rate) {
125    sp<AMessage> msg = new AMessage(kWhatSetRate, this);
126    msg->setFloat("rate", rate);
127    msg->post();
128}
129
130void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
131    {
132        Mutex::Autolock autoLock(mLock);
133        if (audio) {
134            mNotifyCompleteAudio |= notifyComplete;
135            ++mAudioQueueGeneration;
136            ++mAudioDrainGeneration;
137        } else {
138            mNotifyCompleteVideo |= notifyComplete;
139            ++mVideoQueueGeneration;
140            ++mVideoDrainGeneration;
141        }
142
143        clearAnchorTime_l();
144        clearAudioFirstAnchorTime_l();
145        mVideoLateByUs = 0;
146        mSyncQueues = false;
147    }
148
149    sp<AMessage> msg = new AMessage(kWhatFlush, this);
150    msg->setInt32("audio", static_cast<int32_t>(audio));
151    msg->post();
152}
153
154void NuPlayer::Renderer::signalTimeDiscontinuity() {
155}
156
157void NuPlayer::Renderer::signalDisableOffloadAudio() {
158    (new AMessage(kWhatDisableOffloadAudio, this))->post();
159}
160
161void NuPlayer::Renderer::signalEnableOffloadAudio() {
162    (new AMessage(kWhatEnableOffloadAudio, this))->post();
163}
164
165void NuPlayer::Renderer::pause() {
166    (new AMessage(kWhatPause, this))->post();
167}
168
169void NuPlayer::Renderer::resume() {
170    (new AMessage(kWhatResume, this))->post();
171}
172
173void NuPlayer::Renderer::setVideoFrameRate(float fps) {
174    sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
175    msg->setFloat("frame-rate", fps);
176    msg->post();
177}
178
179// Called on any threads.
180status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
181    return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
182}
183
184void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
185    mAudioFirstAnchorTimeMediaUs = -1;
186    mMediaClock->setStartingTimeMedia(-1);
187}
188
189void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
190    if (mAudioFirstAnchorTimeMediaUs == -1) {
191        mAudioFirstAnchorTimeMediaUs = mediaUs;
192        mMediaClock->setStartingTimeMedia(mediaUs);
193    }
194}
195
196void NuPlayer::Renderer::clearAnchorTime_l() {
197    mMediaClock->clearAnchor();
198    mAnchorTimeMediaUs = -1;
199    mAnchorNumFramesWritten = -1;
200}
201
202void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
203    Mutex::Autolock autoLock(mLock);
204    mVideoLateByUs = lateUs;
205}
206
207int64_t NuPlayer::Renderer::getVideoLateByUs() {
208    Mutex::Autolock autoLock(mLock);
209    return mVideoLateByUs;
210}
211
212status_t NuPlayer::Renderer::openAudioSink(
213        const sp<AMessage> &format,
214        bool offloadOnly,
215        bool hasVideo,
216        uint32_t flags,
217        bool *isOffloaded) {
218    sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
219    msg->setMessage("format", format);
220    msg->setInt32("offload-only", offloadOnly);
221    msg->setInt32("has-video", hasVideo);
222    msg->setInt32("flags", flags);
223
224    sp<AMessage> response;
225    msg->postAndAwaitResponse(&response);
226
227    int32_t err;
228    if (!response->findInt32("err", &err)) {
229        err = INVALID_OPERATION;
230    } else if (err == OK && isOffloaded != NULL) {
231        int32_t offload;
232        CHECK(response->findInt32("offload", &offload));
233        *isOffloaded = (offload != 0);
234    }
235    return err;
236}
237
238void NuPlayer::Renderer::closeAudioSink() {
239    sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
240
241    sp<AMessage> response;
242    msg->postAndAwaitResponse(&response);
243}
244
245void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
246    switch (msg->what()) {
247        case kWhatOpenAudioSink:
248        {
249            sp<AMessage> format;
250            CHECK(msg->findMessage("format", &format));
251
252            int32_t offloadOnly;
253            CHECK(msg->findInt32("offload-only", &offloadOnly));
254
255            int32_t hasVideo;
256            CHECK(msg->findInt32("has-video", &hasVideo));
257
258            uint32_t flags;
259            CHECK(msg->findInt32("flags", (int32_t *)&flags));
260
261            status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
262
263            sp<AMessage> response = new AMessage;
264            response->setInt32("err", err);
265            response->setInt32("offload", offloadingAudio());
266
267            sp<AReplyToken> replyID;
268            CHECK(msg->senderAwaitsResponse(&replyID));
269            response->postReply(replyID);
270
271            break;
272        }
273
274        case kWhatCloseAudioSink:
275        {
276            sp<AReplyToken> replyID;
277            CHECK(msg->senderAwaitsResponse(&replyID));
278
279            onCloseAudioSink();
280
281            sp<AMessage> response = new AMessage;
282            response->postReply(replyID);
283            break;
284        }
285
286        case kWhatStopAudioSink:
287        {
288            mAudioSink->stop();
289            break;
290        }
291
292        case kWhatDrainAudioQueue:
293        {
294            int32_t generation;
295            CHECK(msg->findInt32("drainGeneration", &generation));
296            if (generation != getDrainGeneration(true /* audio */)) {
297                break;
298            }
299
300            mDrainAudioQueuePending = false;
301
302            if (onDrainAudioQueue()) {
303                uint32_t numFramesPlayed;
304                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
305                         (status_t)OK);
306
307                uint32_t numFramesPendingPlayout =
308                    mNumFramesWritten - numFramesPlayed;
309
310                // This is how long the audio sink will have data to
311                // play back.
312                int64_t delayUs =
313                    mAudioSink->msecsPerFrame()
314                        * numFramesPendingPlayout * 1000ll;
315                if (mPlaybackRate > 1.0f) {
316                    delayUs /= mPlaybackRate;
317                }
318
319                // Let's give it more data after about half that time
320                // has elapsed.
321                Mutex::Autolock autoLock(mLock);
322                postDrainAudioQueue_l(delayUs / 2);
323            }
324            break;
325        }
326
327        case kWhatDrainVideoQueue:
328        {
329            int32_t generation;
330            CHECK(msg->findInt32("drainGeneration", &generation));
331            if (generation != getDrainGeneration(false /* audio */)) {
332                break;
333            }
334
335            mDrainVideoQueuePending = false;
336
337            onDrainVideoQueue();
338
339            postDrainVideoQueue();
340            break;
341        }
342
343        case kWhatPostDrainVideoQueue:
344        {
345            int32_t generation;
346            CHECK(msg->findInt32("drainGeneration", &generation));
347            if (generation != getDrainGeneration(false /* audio */)) {
348                break;
349            }
350
351            mDrainVideoQueuePending = false;
352            postDrainVideoQueue();
353            break;
354        }
355
356        case kWhatQueueBuffer:
357        {
358            onQueueBuffer(msg);
359            break;
360        }
361
362        case kWhatQueueEOS:
363        {
364            onQueueEOS(msg);
365            break;
366        }
367
368        case kWhatSetRate:
369        {
370            CHECK(msg->findFloat("rate", &mPlaybackRate));
371            int32_t ratePermille = (int32_t)(0.5f + 1000 * mPlaybackRate);
372            mPlaybackRate = ratePermille / 1000.0f;
373            mMediaClock->setPlaybackRate(mPlaybackRate);
374            mAudioSink->setPlaybackRatePermille(ratePermille);
375            break;
376        }
377
378        case kWhatFlush:
379        {
380            onFlush(msg);
381            break;
382        }
383
384        case kWhatDisableOffloadAudio:
385        {
386            onDisableOffloadAudio();
387            break;
388        }
389
390        case kWhatEnableOffloadAudio:
391        {
392            onEnableOffloadAudio();
393            break;
394        }
395
396        case kWhatPause:
397        {
398            onPause();
399            break;
400        }
401
402        case kWhatResume:
403        {
404            onResume();
405            break;
406        }
407
408        case kWhatSetVideoFrameRate:
409        {
410            float fps;
411            CHECK(msg->findFloat("frame-rate", &fps));
412            onSetVideoFrameRate(fps);
413            break;
414        }
415
416        case kWhatAudioOffloadTearDown:
417        {
418            onAudioOffloadTearDown(kDueToError);
419            break;
420        }
421
422        case kWhatAudioOffloadPauseTimeout:
423        {
424            int32_t generation;
425            CHECK(msg->findInt32("drainGeneration", &generation));
426            if (generation != mAudioOffloadPauseTimeoutGeneration) {
427                break;
428            }
429            ALOGV("Audio Offload tear down due to pause timeout.");
430            onAudioOffloadTearDown(kDueToTimeout);
431            mWakeLock->release();
432            break;
433        }
434
435        default:
436            TRESPASS();
437            break;
438    }
439}
440
441void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
442    if (mDrainAudioQueuePending || mSyncQueues || mPaused
443            || offloadingAudio()) {
444        return;
445    }
446
447    if (mAudioQueue.empty()) {
448        return;
449    }
450
451    mDrainAudioQueuePending = true;
452    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
453    msg->setInt32("drainGeneration", mAudioDrainGeneration);
454    msg->post(delayUs);
455}
456
457void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
458    mAudioRenderingStartGeneration = mAudioDrainGeneration;
459    mVideoRenderingStartGeneration = mVideoDrainGeneration;
460}
461
462void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
463    if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
464        mAudioRenderingStartGeneration == mAudioDrainGeneration) {
465        mVideoRenderingStartGeneration = -1;
466        mAudioRenderingStartGeneration = -1;
467
468        sp<AMessage> notify = mNotify->dup();
469        notify->setInt32("what", kWhatMediaRenderingStart);
470        notify->post();
471    }
472}
473
474// static
475size_t NuPlayer::Renderer::AudioSinkCallback(
476        MediaPlayerBase::AudioSink * /* audioSink */,
477        void *buffer,
478        size_t size,
479        void *cookie,
480        MediaPlayerBase::AudioSink::cb_event_t event) {
481    NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
482
483    switch (event) {
484        case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
485        {
486            return me->fillAudioBuffer(buffer, size);
487            break;
488        }
489
490        case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
491        {
492            me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
493            break;
494        }
495
496        case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
497        {
498            me->notifyAudioOffloadTearDown();
499            break;
500        }
501    }
502
503    return 0;
504}
505
506size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
507    Mutex::Autolock autoLock(mLock);
508
509    if (!offloadingAudio() || mPaused) {
510        return 0;
511    }
512
513    bool hasEOS = false;
514
515    size_t sizeCopied = 0;
516    bool firstEntry = true;
517    while (sizeCopied < size && !mAudioQueue.empty()) {
518        QueueEntry *entry = &*mAudioQueue.begin();
519
520        if (entry->mBuffer == NULL) { // EOS
521            hasEOS = true;
522            mAudioQueue.erase(mAudioQueue.begin());
523            entry = NULL;
524            break;
525        }
526
527        if (firstEntry && entry->mOffset == 0) {
528            firstEntry = false;
529            int64_t mediaTimeUs;
530            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
531            ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
532            setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
533        }
534
535        size_t copy = entry->mBuffer->size() - entry->mOffset;
536        size_t sizeRemaining = size - sizeCopied;
537        if (copy > sizeRemaining) {
538            copy = sizeRemaining;
539        }
540
541        memcpy((char *)buffer + sizeCopied,
542               entry->mBuffer->data() + entry->mOffset,
543               copy);
544
545        entry->mOffset += copy;
546        if (entry->mOffset == entry->mBuffer->size()) {
547            entry->mNotifyConsumed->post();
548            mAudioQueue.erase(mAudioQueue.begin());
549            entry = NULL;
550        }
551        sizeCopied += copy;
552
553        notifyIfMediaRenderingStarted_l();
554    }
555
556    if (mAudioFirstAnchorTimeMediaUs >= 0) {
557        int64_t nowUs = ALooper::GetNowUs();
558        int64_t nowMediaUs =
559            mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs);
560        // we don't know how much data we are queueing for offloaded tracks.
561        mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
562    }
563
564    if (hasEOS) {
565        (new AMessage(kWhatStopAudioSink, this))->post();
566    }
567
568    return sizeCopied;
569}
570
571bool NuPlayer::Renderer::onDrainAudioQueue() {
572    // TODO: This call to getPosition checks if AudioTrack has been created
573    // in AudioSink before draining audio. If AudioTrack doesn't exist, then
574    // CHECKs on getPosition will fail.
575    // We still need to figure out why AudioTrack is not created when
576    // this function is called. One possible reason could be leftover
577    // audio. Another possible place is to check whether decoder
578    // has received INFO_FORMAT_CHANGED as the first buffer since
579    // AudioSink is opened there, and possible interactions with flush
580    // immediately after start. Investigate error message
581    // "vorbis_dsp_synthesis returned -135", along with RTSP.
582    uint32_t numFramesPlayed;
583    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
584        return false;
585    }
586
587#if 0
588    ssize_t numFramesAvailableToWrite =
589        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
590
591    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
592        ALOGI("audio sink underrun");
593    } else {
594        ALOGV("audio queue has %d frames left to play",
595             mAudioSink->frameCount() - numFramesAvailableToWrite);
596    }
597#endif
598
599    while (!mAudioQueue.empty()) {
600        QueueEntry *entry = &*mAudioQueue.begin();
601
602        mLastAudioBufferDrained = entry->mBufferOrdinal;
603
604        if (entry->mBuffer == NULL) {
605            // EOS
606            int64_t postEOSDelayUs = 0;
607            if (mAudioSink->needsTrailingPadding()) {
608                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
609            }
610            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
611
612            mAudioQueue.erase(mAudioQueue.begin());
613            entry = NULL;
614            if (mAudioSink->needsTrailingPadding()) {
615                // If we're not in gapless playback (i.e. through setNextPlayer), we
616                // need to stop the track here, because that will play out the last
617                // little bit at the end of the file. Otherwise short files won't play.
618                mAudioSink->stop();
619                mNumFramesWritten = 0;
620            }
621            return false;
622        }
623
624        if (entry->mOffset == 0) {
625            int64_t mediaTimeUs;
626            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
627            ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
628            onNewAudioMediaTime(mediaTimeUs);
629        }
630
631        size_t copy = entry->mBuffer->size() - entry->mOffset;
632
633        ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
634                                            copy, false /* blocking */);
635        if (written < 0) {
636            // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
637            if (written == WOULD_BLOCK) {
638                ALOGW("AudioSink write would block when writing %zu bytes", copy);
639            } else {
640                ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
641            }
642            break;
643        }
644
645        entry->mOffset += written;
646        if (entry->mOffset == entry->mBuffer->size()) {
647            entry->mNotifyConsumed->post();
648            mAudioQueue.erase(mAudioQueue.begin());
649
650            entry = NULL;
651        }
652
653        size_t copiedFrames = written / mAudioSink->frameSize();
654        mNumFramesWritten += copiedFrames;
655
656        {
657            Mutex::Autolock autoLock(mLock);
658            notifyIfMediaRenderingStarted_l();
659        }
660
661        if (written != (ssize_t)copy) {
662            // A short count was received from AudioSink::write()
663            //
664            // AudioSink write is called in non-blocking mode.
665            // It may return with a short count when:
666            //
667            // 1) Size to be copied is not a multiple of the frame size. We consider this fatal.
668            // 2) The data to be copied exceeds the available buffer in AudioSink.
669            // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
670            // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
671
672            // (Case 1)
673            // Must be a multiple of the frame size.  If it is not a multiple of a frame size, it
674            // needs to fail, as we should not carry over fractional frames between calls.
675            CHECK_EQ(copy % mAudioSink->frameSize(), 0);
676
677            // (Case 2, 3, 4)
678            // Return early to the caller.
679            // Beware of calling immediately again as this may busy-loop if you are not careful.
680            ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
681            break;
682        }
683    }
684    int64_t maxTimeMedia;
685    {
686        Mutex::Autolock autoLock(mLock);
687        maxTimeMedia =
688            mAnchorTimeMediaUs +
689                    (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
690                            * 1000LL * mAudioSink->msecsPerFrame());
691    }
692    mMediaClock->updateMaxTimeMedia(maxTimeMedia);
693
694    return !mAudioQueue.empty();
695}
696
697int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
698    int32_t sampleRate = offloadingAudio() ?
699            mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
700    // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
701    return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate);
702}
703
704// Calculate duration of pending samples if played at normal rate (i.e., 1.0).
705int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
706    int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
707    return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs);
708}
709
710int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
711    int64_t realUs;
712    if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
713        // If failed to get current position, e.g. due to audio clock is
714        // not ready, then just play out video immediately without delay.
715        return nowUs;
716    }
717    return realUs;
718}
719
720void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
721    Mutex::Autolock autoLock(mLock);
722    // TRICKY: vorbis decoder generates multiple frames with the same
723    // timestamp, so only update on the first frame with a given timestamp
724    if (mediaTimeUs == mAnchorTimeMediaUs) {
725        return;
726    }
727    setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
728    int64_t nowUs = ALooper::GetNowUs();
729    int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
730    mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
731    mAnchorNumFramesWritten = mNumFramesWritten;
732    mAnchorTimeMediaUs = mediaTimeUs;
733}
734
735// Called without mLock acquired.
736void NuPlayer::Renderer::postDrainVideoQueue() {
737    if (mDrainVideoQueuePending
738            || getSyncQueues()
739            || (mPaused && mVideoSampleReceived)) {
740        return;
741    }
742
743    if (mVideoQueue.empty()) {
744        return;
745    }
746
747    QueueEntry &entry = *mVideoQueue.begin();
748
749    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
750    msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
751
752    if (entry.mBuffer == NULL) {
753        // EOS doesn't carry a timestamp.
754        msg->post();
755        mDrainVideoQueuePending = true;
756        return;
757    }
758
759    int64_t delayUs;
760    int64_t nowUs = ALooper::GetNowUs();
761    int64_t realTimeUs;
762    if (mFlags & FLAG_REAL_TIME) {
763        int64_t mediaTimeUs;
764        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
765        realTimeUs = mediaTimeUs;
766    } else {
767        int64_t mediaTimeUs;
768        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
769
770        {
771            Mutex::Autolock autoLock(mLock);
772            if (mAnchorTimeMediaUs < 0) {
773                mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
774                mAnchorTimeMediaUs = mediaTimeUs;
775                realTimeUs = nowUs;
776            } else {
777                realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
778            }
779        }
780        if (!mHasAudio) {
781            // smooth out videos >= 10fps
782            mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
783        }
784
785        // Heuristics to handle situation when media time changed without a
786        // discontinuity. If we have not drained an audio buffer that was
787        // received after this buffer, repost in 10 msec. Otherwise repost
788        // in 500 msec.
789        delayUs = realTimeUs - nowUs;
790        if (delayUs > 500000) {
791            int64_t postDelayUs = 500000;
792            if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) {
793                postDelayUs = 10000;
794            }
795            msg->setWhat(kWhatPostDrainVideoQueue);
796            msg->post(postDelayUs);
797            mVideoScheduler->restart();
798            ALOGI("possible video time jump of %dms, retrying in %dms",
799                    (int)(delayUs / 1000), (int)(postDelayUs / 1000));
800            mDrainVideoQueuePending = true;
801            return;
802        }
803    }
804
805    realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
806    int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
807
808    delayUs = realTimeUs - nowUs;
809
810    ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
811    // post 2 display refreshes before rendering is due
812    msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
813
814    mDrainVideoQueuePending = true;
815}
816
817void NuPlayer::Renderer::onDrainVideoQueue() {
818    if (mVideoQueue.empty()) {
819        return;
820    }
821
822    QueueEntry *entry = &*mVideoQueue.begin();
823
824    if (entry->mBuffer == NULL) {
825        // EOS
826
827        notifyEOS(false /* audio */, entry->mFinalResult);
828
829        mVideoQueue.erase(mVideoQueue.begin());
830        entry = NULL;
831
832        setVideoLateByUs(0);
833        return;
834    }
835
836    int64_t nowUs = -1;
837    int64_t realTimeUs;
838    if (mFlags & FLAG_REAL_TIME) {
839        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
840    } else {
841        int64_t mediaTimeUs;
842        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
843
844        nowUs = ALooper::GetNowUs();
845        realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
846    }
847
848    bool tooLate = false;
849
850    if (!mPaused) {
851        if (nowUs == -1) {
852            nowUs = ALooper::GetNowUs();
853        }
854        setVideoLateByUs(nowUs - realTimeUs);
855        tooLate = (mVideoLateByUs > 40000);
856
857        if (tooLate) {
858            ALOGV("video late by %lld us (%.2f secs)",
859                 mVideoLateByUs, mVideoLateByUs / 1E6);
860        } else {
861            int64_t mediaUs = 0;
862            mMediaClock->getMediaTime(realTimeUs, &mediaUs);
863            ALOGV("rendering video at media time %.2f secs",
864                    (mFlags & FLAG_REAL_TIME ? realTimeUs :
865                    mediaUs) / 1E6);
866        }
867    } else {
868        setVideoLateByUs(0);
869        if (!mVideoSampleReceived && !mHasAudio) {
870            // This will ensure that the first frame after a flush won't be used as anchor
871            // when renderer is in paused state, because resume can happen any time after seek.
872            Mutex::Autolock autoLock(mLock);
873            clearAnchorTime_l();
874        }
875    }
876
877    entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
878    entry->mNotifyConsumed->setInt32("render", !tooLate);
879    entry->mNotifyConsumed->post();
880    mVideoQueue.erase(mVideoQueue.begin());
881    entry = NULL;
882
883    mVideoSampleReceived = true;
884
885    if (!mPaused) {
886        if (!mVideoRenderingStarted) {
887            mVideoRenderingStarted = true;
888            notifyVideoRenderingStart();
889        }
890        Mutex::Autolock autoLock(mLock);
891        notifyIfMediaRenderingStarted_l();
892    }
893}
894
895void NuPlayer::Renderer::notifyVideoRenderingStart() {
896    sp<AMessage> notify = mNotify->dup();
897    notify->setInt32("what", kWhatVideoRenderingStart);
898    notify->post();
899}
900
901void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
902    sp<AMessage> notify = mNotify->dup();
903    notify->setInt32("what", kWhatEOS);
904    notify->setInt32("audio", static_cast<int32_t>(audio));
905    notify->setInt32("finalResult", finalResult);
906    notify->post(delayUs);
907}
908
909void NuPlayer::Renderer::notifyAudioOffloadTearDown() {
910    (new AMessage(kWhatAudioOffloadTearDown, this))->post();
911}
912
913void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
914    int32_t audio;
915    CHECK(msg->findInt32("audio", &audio));
916
917    if (dropBufferIfStale(audio, msg)) {
918        return;
919    }
920
921    if (audio) {
922        mHasAudio = true;
923    } else {
924        mHasVideo = true;
925    }
926
927    if (mHasVideo) {
928        if (mVideoScheduler == NULL) {
929            mVideoScheduler = new VideoFrameScheduler();
930            mVideoScheduler->init();
931        }
932    }
933
934    sp<ABuffer> buffer;
935    CHECK(msg->findBuffer("buffer", &buffer));
936
937    sp<AMessage> notifyConsumed;
938    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
939
940    QueueEntry entry;
941    entry.mBuffer = buffer;
942    entry.mNotifyConsumed = notifyConsumed;
943    entry.mOffset = 0;
944    entry.mFinalResult = OK;
945    entry.mBufferOrdinal = ++mTotalBuffersQueued;
946
947    if (audio) {
948        Mutex::Autolock autoLock(mLock);
949        mAudioQueue.push_back(entry);
950        postDrainAudioQueue_l();
951    } else {
952        mVideoQueue.push_back(entry);
953        postDrainVideoQueue();
954    }
955
956    Mutex::Autolock autoLock(mLock);
957    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
958        return;
959    }
960
961    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
962    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
963
964    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
965        // EOS signalled on either queue.
966        syncQueuesDone_l();
967        return;
968    }
969
970    int64_t firstAudioTimeUs;
971    int64_t firstVideoTimeUs;
972    CHECK(firstAudioBuffer->meta()
973            ->findInt64("timeUs", &firstAudioTimeUs));
974    CHECK(firstVideoBuffer->meta()
975            ->findInt64("timeUs", &firstVideoTimeUs));
976
977    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
978
979    ALOGV("queueDiff = %.2f secs", diff / 1E6);
980
981    if (diff > 100000ll) {
982        // Audio data starts More than 0.1 secs before video.
983        // Drop some audio.
984
985        (*mAudioQueue.begin()).mNotifyConsumed->post();
986        mAudioQueue.erase(mAudioQueue.begin());
987        return;
988    }
989
990    syncQueuesDone_l();
991}
992
993void NuPlayer::Renderer::syncQueuesDone_l() {
994    if (!mSyncQueues) {
995        return;
996    }
997
998    mSyncQueues = false;
999
1000    if (!mAudioQueue.empty()) {
1001        postDrainAudioQueue_l();
1002    }
1003
1004    if (!mVideoQueue.empty()) {
1005        mLock.unlock();
1006        postDrainVideoQueue();
1007        mLock.lock();
1008    }
1009}
1010
1011void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1012    int32_t audio;
1013    CHECK(msg->findInt32("audio", &audio));
1014
1015    if (dropBufferIfStale(audio, msg)) {
1016        return;
1017    }
1018
1019    int32_t finalResult;
1020    CHECK(msg->findInt32("finalResult", &finalResult));
1021
1022    QueueEntry entry;
1023    entry.mOffset = 0;
1024    entry.mFinalResult = finalResult;
1025
1026    if (audio) {
1027        Mutex::Autolock autoLock(mLock);
1028        if (mAudioQueue.empty() && mSyncQueues) {
1029            syncQueuesDone_l();
1030        }
1031        mAudioQueue.push_back(entry);
1032        postDrainAudioQueue_l();
1033    } else {
1034        if (mVideoQueue.empty() && getSyncQueues()) {
1035            Mutex::Autolock autoLock(mLock);
1036            syncQueuesDone_l();
1037        }
1038        mVideoQueue.push_back(entry);
1039        postDrainVideoQueue();
1040    }
1041}
1042
1043void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
1044    int32_t audio, notifyComplete;
1045    CHECK(msg->findInt32("audio", &audio));
1046
1047    {
1048        Mutex::Autolock autoLock(mLock);
1049        if (audio) {
1050            notifyComplete = mNotifyCompleteAudio;
1051            mNotifyCompleteAudio = false;
1052        } else {
1053            notifyComplete = mNotifyCompleteVideo;
1054            mNotifyCompleteVideo = false;
1055        }
1056
1057        // If we're currently syncing the queues, i.e. dropping audio while
1058        // aligning the first audio/video buffer times and only one of the
1059        // two queues has data, we may starve that queue by not requesting
1060        // more buffers from the decoder. If the other source then encounters
1061        // a discontinuity that leads to flushing, we'll never find the
1062        // corresponding discontinuity on the other queue.
1063        // Therefore we'll stop syncing the queues if at least one of them
1064        // is flushed.
1065        syncQueuesDone_l();
1066        clearAnchorTime_l();
1067    }
1068
1069    ALOGV("flushing %s", audio ? "audio" : "video");
1070    if (audio) {
1071        {
1072            Mutex::Autolock autoLock(mLock);
1073            flushQueue(&mAudioQueue);
1074
1075            ++mAudioDrainGeneration;
1076            prepareForMediaRenderingStart_l();
1077
1078            if (offloadingAudio()) {
1079                clearAudioFirstAnchorTime_l();
1080            }
1081        }
1082
1083        mDrainAudioQueuePending = false;
1084
1085        if (offloadingAudio()) {
1086            mAudioSink->pause();
1087            mAudioSink->flush();
1088            mAudioSink->start();
1089        }
1090    } else {
1091        flushQueue(&mVideoQueue);
1092
1093        mDrainVideoQueuePending = false;
1094
1095        if (mVideoScheduler != NULL) {
1096            mVideoScheduler->restart();
1097        }
1098
1099        Mutex::Autolock autoLock(mLock);
1100        ++mVideoDrainGeneration;
1101        prepareForMediaRenderingStart_l();
1102    }
1103
1104    mVideoSampleReceived = false;
1105
1106    if (notifyComplete) {
1107        notifyFlushComplete(audio);
1108    }
1109}
1110
1111void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1112    while (!queue->empty()) {
1113        QueueEntry *entry = &*queue->begin();
1114
1115        if (entry->mBuffer != NULL) {
1116            entry->mNotifyConsumed->post();
1117        }
1118
1119        queue->erase(queue->begin());
1120        entry = NULL;
1121    }
1122}
1123
1124void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1125    sp<AMessage> notify = mNotify->dup();
1126    notify->setInt32("what", kWhatFlushComplete);
1127    notify->setInt32("audio", static_cast<int32_t>(audio));
1128    notify->post();
1129}
1130
1131bool NuPlayer::Renderer::dropBufferIfStale(
1132        bool audio, const sp<AMessage> &msg) {
1133    int32_t queueGeneration;
1134    CHECK(msg->findInt32("queueGeneration", &queueGeneration));
1135
1136    if (queueGeneration == getQueueGeneration(audio)) {
1137        return false;
1138    }
1139
1140    sp<AMessage> notifyConsumed;
1141    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1142        notifyConsumed->post();
1143    }
1144
1145    return true;
1146}
1147
1148void NuPlayer::Renderer::onAudioSinkChanged() {
1149    if (offloadingAudio()) {
1150        return;
1151    }
1152    CHECK(!mDrainAudioQueuePending);
1153    mNumFramesWritten = 0;
1154    {
1155        Mutex::Autolock autoLock(mLock);
1156        mAnchorNumFramesWritten = -1;
1157    }
1158    uint32_t written;
1159    if (mAudioSink->getFramesWritten(&written) == OK) {
1160        mNumFramesWritten = written;
1161    }
1162}
1163
1164void NuPlayer::Renderer::onDisableOffloadAudio() {
1165    Mutex::Autolock autoLock(mLock);
1166    mFlags &= ~FLAG_OFFLOAD_AUDIO;
1167    ++mAudioDrainGeneration;
1168}
1169
1170void NuPlayer::Renderer::onEnableOffloadAudio() {
1171    Mutex::Autolock autoLock(mLock);
1172    mFlags |= FLAG_OFFLOAD_AUDIO;
1173    ++mAudioDrainGeneration;
1174}
1175
1176void NuPlayer::Renderer::onPause() {
1177    if (mPaused) {
1178        ALOGW("Renderer::onPause() called while already paused!");
1179        return;
1180    }
1181    int64_t currentPositionUs;
1182    {
1183        Mutex::Autolock autoLock(mLock);
1184        ++mAudioDrainGeneration;
1185        ++mVideoDrainGeneration;
1186        prepareForMediaRenderingStart_l();
1187        mPaused = true;
1188        mMediaClock->setPlaybackRate(0.0);
1189    }
1190
1191    mDrainAudioQueuePending = false;
1192    mDrainVideoQueuePending = false;
1193
1194    if (mHasAudio) {
1195        mAudioSink->pause();
1196        startAudioOffloadPauseTimeout();
1197    }
1198
1199    ALOGV("now paused audio queue has %d entries, video has %d entries",
1200          mAudioQueue.size(), mVideoQueue.size());
1201}
1202
1203void NuPlayer::Renderer::onResume() {
1204    if (!mPaused) {
1205        return;
1206    }
1207
1208    if (mHasAudio) {
1209        cancelAudioOffloadPauseTimeout();
1210        mAudioSink->start();
1211    }
1212
1213    {
1214        Mutex::Autolock autoLock(mLock);
1215        mPaused = false;
1216        mMediaClock->setPlaybackRate(mPlaybackRate);
1217
1218        if (!mAudioQueue.empty()) {
1219            postDrainAudioQueue_l();
1220        }
1221    }
1222
1223    if (!mVideoQueue.empty()) {
1224        postDrainVideoQueue();
1225    }
1226}
1227
1228void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1229    if (mVideoScheduler == NULL) {
1230        mVideoScheduler = new VideoFrameScheduler();
1231    }
1232    mVideoScheduler->init(fps);
1233}
1234
1235int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
1236    Mutex::Autolock autoLock(mLock);
1237    return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
1238}
1239
1240int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
1241    Mutex::Autolock autoLock(mLock);
1242    return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
1243}
1244
1245bool NuPlayer::Renderer::getSyncQueues() {
1246    Mutex::Autolock autoLock(mLock);
1247    return mSyncQueues;
1248}
1249
1250// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs()
1251// as it acquires locks and may query the audio driver.
1252//
1253// Some calls could conceivably retrieve extrapolated data instead of
1254// accessing getTimestamp() or getPosition() every time a data buffer with
1255// a media time is received.
1256//
1257// Calculate duration of played samples if played at normal rate (i.e., 1.0).
1258int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) {
1259    uint32_t numFramesPlayed;
1260    int64_t numFramesPlayedAt;
1261    AudioTimestamp ts;
1262    static const int64_t kStaleTimestamp100ms = 100000;
1263
1264    status_t res = mAudioSink->getTimestamp(ts);
1265    if (res == OK) {                 // case 1: mixing audio tracks and offloaded tracks.
1266        numFramesPlayed = ts.mPosition;
1267        numFramesPlayedAt =
1268            ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
1269        const int64_t timestampAge = nowUs - numFramesPlayedAt;
1270        if (timestampAge > kStaleTimestamp100ms) {
1271            // This is an audio FIXME.
1272            // getTimestamp returns a timestamp which may come from audio mixing threads.
1273            // After pausing, the MixerThread may go idle, thus the mTime estimate may
1274            // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms,
1275            // the max latency should be about 25ms with an average around 12ms (to be verified).
1276            // For safety we use 100ms.
1277            ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)",
1278                    (long long)nowUs, (long long)numFramesPlayedAt);
1279            numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
1280        }
1281        //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1282    } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track
1283        numFramesPlayed = 0;
1284        numFramesPlayedAt = nowUs;
1285        //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
1286        //        numFramesPlayed, (long long)numFramesPlayedAt);
1287    } else {                         // case 3: transitory at new track or audio fast tracks.
1288        res = mAudioSink->getPosition(&numFramesPlayed);
1289        CHECK_EQ(res, (status_t)OK);
1290        numFramesPlayedAt = nowUs;
1291        numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */
1292        //ALOGD("getPosition: %d %lld", numFramesPlayed, numFramesPlayedAt);
1293    }
1294
1295    //CHECK_EQ(numFramesPlayed & (1 << 31), 0);  // can't be negative until 12.4 hrs, test
1296    int64_t durationUs = getDurationUsIfPlayedAtSampleRate(numFramesPlayed)
1297            + nowUs - numFramesPlayedAt;
1298    if (durationUs < 0) {
1299        // Occurs when numFramesPlayed position is very small and the following:
1300        // (1) In case 1, the time nowUs is computed before getTimestamp() is called and
1301        //     numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed.
1302        // (2) In case 3, using getPosition and adding mAudioSink->latency() to
1303        //     numFramesPlayedAt, by a time amount greater than numFramesPlayed.
1304        //
1305        // Both of these are transitory conditions.
1306        ALOGV("getPlayedOutAudioDurationUs: negative duration %lld set to zero", (long long)durationUs);
1307        durationUs = 0;
1308    }
1309    ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)",
1310            (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt);
1311    return durationUs;
1312}
1313
1314void NuPlayer::Renderer::onAudioOffloadTearDown(AudioOffloadTearDownReason reason) {
1315    if (mAudioOffloadTornDown) {
1316        return;
1317    }
1318    mAudioOffloadTornDown = true;
1319
1320    int64_t currentPositionUs;
1321    if (getCurrentPosition(&currentPositionUs) != OK) {
1322        currentPositionUs = 0;
1323    }
1324
1325    mAudioSink->stop();
1326    mAudioSink->flush();
1327
1328    sp<AMessage> notify = mNotify->dup();
1329    notify->setInt32("what", kWhatAudioOffloadTearDown);
1330    notify->setInt64("positionUs", currentPositionUs);
1331    notify->setInt32("reason", reason);
1332    notify->post();
1333}
1334
1335void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1336    if (offloadingAudio()) {
1337        mWakeLock->acquire();
1338        sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
1339        msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
1340        msg->post(kOffloadPauseMaxUs);
1341    }
1342}
1343
1344void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
1345    if (offloadingAudio()) {
1346        mWakeLock->release(true);
1347        ++mAudioOffloadPauseTimeoutGeneration;
1348    }
1349}
1350
1351status_t NuPlayer::Renderer::onOpenAudioSink(
1352        const sp<AMessage> &format,
1353        bool offloadOnly,
1354        bool hasVideo,
1355        uint32_t flags) {
1356    ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1357            offloadOnly, offloadingAudio());
1358    bool audioSinkChanged = false;
1359
1360    int32_t numChannels;
1361    CHECK(format->findInt32("channel-count", &numChannels));
1362
1363    int32_t channelMask;
1364    if (!format->findInt32("channel-mask", &channelMask)) {
1365        // signal to the AudioSink to derive the mask from count.
1366        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1367    }
1368
1369    int32_t sampleRate;
1370    CHECK(format->findInt32("sample-rate", &sampleRate));
1371
1372    if (offloadingAudio()) {
1373        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1374        AString mime;
1375        CHECK(format->findString("mime", &mime));
1376        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1377
1378        if (err != OK) {
1379            ALOGE("Couldn't map mime \"%s\" to a valid "
1380                    "audio_format", mime.c_str());
1381            onDisableOffloadAudio();
1382        } else {
1383            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1384                    mime.c_str(), audioFormat);
1385
1386            int avgBitRate = -1;
1387            format->findInt32("bit-rate", &avgBitRate);
1388
1389            int32_t aacProfile = -1;
1390            if (audioFormat == AUDIO_FORMAT_AAC
1391                    && format->findInt32("aac-profile", &aacProfile)) {
1392                // Redefine AAC format as per aac profile
1393                mapAACProfileToAudioFormat(
1394                        audioFormat,
1395                        aacProfile);
1396            }
1397
1398            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1399            offloadInfo.duration_us = -1;
1400            format->findInt64(
1401                    "durationUs", &offloadInfo.duration_us);
1402            offloadInfo.sample_rate = sampleRate;
1403            offloadInfo.channel_mask = channelMask;
1404            offloadInfo.format = audioFormat;
1405            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1406            offloadInfo.bit_rate = avgBitRate;
1407            offloadInfo.has_video = hasVideo;
1408            offloadInfo.is_streaming = true;
1409
1410            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1411                ALOGV("openAudioSink: no change in offload mode");
1412                // no change from previous configuration, everything ok.
1413                return OK;
1414            }
1415            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1416
1417            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1418            uint32_t offloadFlags = flags;
1419            offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1420            offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1421            audioSinkChanged = true;
1422            mAudioSink->close();
1423            err = mAudioSink->open(
1424                    sampleRate,
1425                    numChannels,
1426                    (audio_channel_mask_t)channelMask,
1427                    audioFormat,
1428                    8 /* bufferCount */,
1429                    &NuPlayer::Renderer::AudioSinkCallback,
1430                    this,
1431                    (audio_output_flags_t)offloadFlags,
1432                    &offloadInfo);
1433
1434            if (err == OK) {
1435                if (mPlaybackRate != 1.0) {
1436                    mAudioSink->setPlaybackRatePermille(
1437                            (int32_t)(mPlaybackRate * 1000 + 0.5f));
1438                }
1439                // If the playback is offloaded to h/w, we pass
1440                // the HAL some metadata information.
1441                // We don't want to do this for PCM because it
1442                // will be going through the AudioFlinger mixer
1443                // before reaching the hardware.
1444                // TODO
1445                mCurrentOffloadInfo = offloadInfo;
1446                err = mAudioSink->start();
1447                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1448            }
1449            if (err != OK) {
1450                // Clean up, fall back to non offload mode.
1451                mAudioSink->close();
1452                onDisableOffloadAudio();
1453                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1454                ALOGV("openAudioSink: offload failed");
1455            }
1456        }
1457    }
1458    if (!offloadOnly && !offloadingAudio()) {
1459        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1460        uint32_t pcmFlags = flags;
1461        pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1462
1463        const PcmInfo info = {
1464                (audio_channel_mask_t)channelMask,
1465                (audio_output_flags_t)pcmFlags,
1466                AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat
1467                numChannels,
1468                sampleRate
1469        };
1470        if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
1471            ALOGV("openAudioSink: no change in pcm mode");
1472            // no change from previous configuration, everything ok.
1473            return OK;
1474        }
1475
1476        audioSinkChanged = true;
1477        mAudioSink->close();
1478        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1479        status_t err = mAudioSink->open(
1480                    sampleRate,
1481                    numChannels,
1482                    (audio_channel_mask_t)channelMask,
1483                    AUDIO_FORMAT_PCM_16_BIT,
1484                    8 /* bufferCount */,
1485                    NULL,
1486                    NULL,
1487                    (audio_output_flags_t)pcmFlags);
1488        if (err != OK) {
1489            ALOGW("openAudioSink: non offloaded open failed status: %d", err);
1490            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1491            return err;
1492        }
1493        mCurrentPcmInfo = info;
1494        if (mPlaybackRate != 1.0) {
1495            mAudioSink->setPlaybackRatePermille(
1496                    (int32_t)(mPlaybackRate * 1000 + 0.5f));
1497        }
1498        mAudioSink->start();
1499    }
1500    if (audioSinkChanged) {
1501        onAudioSinkChanged();
1502    }
1503    if (offloadingAudio()) {
1504        mAudioOffloadTornDown = false;
1505    }
1506    return OK;
1507}
1508
1509void NuPlayer::Renderer::onCloseAudioSink() {
1510    mAudioSink->close();
1511    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1512    mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1513}
1514
1515}  // namespace android
1516
1517