NuPlayerRenderer.cpp revision 3491232a7c0d953fa021f6a81baee64c44f364f3 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26#include <media/stagefright/foundation/AUtils.h>
27#include <media/stagefright/MediaErrors.h>
28#include <media/stagefright/MetaData.h>
29#include <media/stagefright/Utils.h>
30
31#include <VideoFrameScheduler.h>
32
33#include <inttypes.h>
34
35namespace android {
36
37// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
38// is closed to allow the audio DSP to power down.
39static const int64_t kOffloadPauseMaxUs = 10000000ll;
40
41// static
42const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
43
44NuPlayer::Renderer::Renderer(
45        const sp<MediaPlayerBase::AudioSink> &sink,
46        const sp<AMessage> &notify,
47        uint32_t flags)
48    : mAudioSink(sink),
49      mNotify(notify),
50      mFlags(flags),
51      mNumFramesWritten(0),
52      mDrainAudioQueuePending(false),
53      mDrainVideoQueuePending(false),
54      mAudioQueueGeneration(0),
55      mVideoQueueGeneration(0),
56      mAudioFirstAnchorTimeMediaUs(-1),
57      mAnchorTimeMediaUs(-1),
58      mAnchorTimeRealUs(-1),
59      mAnchorNumFramesWritten(-1),
60      mAnchorMaxMediaUs(-1),
61      mVideoLateByUs(0ll),
62      mHasAudio(false),
63      mHasVideo(false),
64      mPauseStartedTimeRealUs(-1),
65      mFlushingAudio(false),
66      mFlushingVideo(false),
67      mNotifyCompleteAudio(false),
68      mNotifyCompleteVideo(false),
69      mSyncQueues(false),
70      mPaused(false),
71      mVideoSampleReceived(false),
72      mVideoRenderingStarted(false),
73      mVideoRenderingStartGeneration(0),
74      mAudioRenderingStartGeneration(0),
75      mAudioOffloadPauseTimeoutGeneration(0),
76      mAudioOffloadTornDown(false),
77      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
78      mTotalBuffersQueued(0),
79      mLastAudioBufferDrained(0) {
80}
81
82NuPlayer::Renderer::~Renderer() {
83    if (offloadingAudio()) {
84        mAudioSink->stop();
85        mAudioSink->flush();
86        mAudioSink->close();
87    }
88}
89
90void NuPlayer::Renderer::queueBuffer(
91        bool audio,
92        const sp<ABuffer> &buffer,
93        const sp<AMessage> &notifyConsumed) {
94    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
95    msg->setInt32("audio", static_cast<int32_t>(audio));
96    msg->setBuffer("buffer", buffer);
97    msg->setMessage("notifyConsumed", notifyConsumed);
98    msg->post();
99}
100
101void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
102    CHECK_NE(finalResult, (status_t)OK);
103
104    sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
105    msg->setInt32("audio", static_cast<int32_t>(audio));
106    msg->setInt32("finalResult", finalResult);
107    msg->post();
108}
109
110void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
111    {
112        Mutex::Autolock autoLock(mFlushLock);
113        if (audio) {
114            mNotifyCompleteAudio |= notifyComplete;
115            if (mFlushingAudio) {
116                return;
117            }
118            mFlushingAudio = true;
119        } else {
120            mNotifyCompleteVideo |= notifyComplete;
121            if (mFlushingVideo) {
122                return;
123            }
124            mFlushingVideo = true;
125        }
126    }
127
128    sp<AMessage> msg = new AMessage(kWhatFlush, id());
129    msg->setInt32("audio", static_cast<int32_t>(audio));
130    msg->post();
131}
132
133void NuPlayer::Renderer::signalTimeDiscontinuity() {
134    Mutex::Autolock autoLock(mLock);
135    // CHECK(mAudioQueue.empty());
136    // CHECK(mVideoQueue.empty());
137    setAudioFirstAnchorTime(-1);
138    setAnchorTime(-1, -1);
139    setVideoLateByUs(0);
140    mSyncQueues = false;
141}
142
143void NuPlayer::Renderer::signalAudioSinkChanged() {
144    (new AMessage(kWhatAudioSinkChanged, id()))->post();
145}
146
147void NuPlayer::Renderer::signalDisableOffloadAudio() {
148    (new AMessage(kWhatDisableOffloadAudio, id()))->post();
149}
150
151void NuPlayer::Renderer::signalEnableOffloadAudio() {
152    (new AMessage(kWhatEnableOffloadAudio, id()))->post();
153}
154
155void NuPlayer::Renderer::pause() {
156    (new AMessage(kWhatPause, id()))->post();
157}
158
159void NuPlayer::Renderer::resume() {
160    (new AMessage(kWhatResume, id()))->post();
161}
162
163void NuPlayer::Renderer::setVideoFrameRate(float fps) {
164    sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, id());
165    msg->setFloat("frame-rate", fps);
166    msg->post();
167}
168
169status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
170    return getCurrentPosition(mediaUs, ALooper::GetNowUs());
171}
172
173status_t NuPlayer::Renderer::getCurrentPosition(
174        int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo) {
175    Mutex::Autolock autoLock(mTimeLock);
176    if (!mHasAudio && !mHasVideo) {
177        return NO_INIT;
178    }
179
180    if (mAnchorTimeMediaUs < 0) {
181        return NO_INIT;
182    }
183
184    int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs;
185
186    if (mPauseStartedTimeRealUs != -1) {
187        positionUs -= (nowUs - mPauseStartedTimeRealUs);
188    }
189
190    // limit position to the last queued media time (for video only stream
191    // position will be discrete as we don't know how long each frame lasts)
192    if (mAnchorMaxMediaUs >= 0 && !allowPastQueuedVideo) {
193        if (positionUs > mAnchorMaxMediaUs) {
194            positionUs = mAnchorMaxMediaUs;
195        }
196    }
197
198    if (positionUs < mAudioFirstAnchorTimeMediaUs) {
199        positionUs = mAudioFirstAnchorTimeMediaUs;
200    }
201
202    *mediaUs = (positionUs <= 0) ? 0 : positionUs;
203    return OK;
204}
205
206void NuPlayer::Renderer::setHasMedia(bool audio) {
207    Mutex::Autolock autoLock(mTimeLock);
208    if (audio) {
209        mHasAudio = true;
210    } else {
211        mHasVideo = true;
212    }
213}
214
215void NuPlayer::Renderer::setAudioFirstAnchorTime(int64_t mediaUs) {
216    Mutex::Autolock autoLock(mTimeLock);
217    mAudioFirstAnchorTimeMediaUs = mediaUs;
218}
219
220void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded(int64_t mediaUs) {
221    Mutex::Autolock autoLock(mTimeLock);
222    if (mAudioFirstAnchorTimeMediaUs == -1) {
223        mAudioFirstAnchorTimeMediaUs = mediaUs;
224    }
225}
226
227void NuPlayer::Renderer::setAnchorTime(
228        int64_t mediaUs, int64_t realUs, int64_t numFramesWritten, bool resume) {
229    Mutex::Autolock autoLock(mTimeLock);
230    mAnchorTimeMediaUs = mediaUs;
231    mAnchorTimeRealUs = realUs;
232    mAnchorNumFramesWritten = numFramesWritten;
233    if (resume) {
234        mPauseStartedTimeRealUs = -1;
235    }
236}
237
238void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
239    Mutex::Autolock autoLock(mTimeLock);
240    mVideoLateByUs = lateUs;
241}
242
243int64_t NuPlayer::Renderer::getVideoLateByUs() {
244    Mutex::Autolock autoLock(mTimeLock);
245    return mVideoLateByUs;
246}
247
248void NuPlayer::Renderer::setPauseStartedTimeRealUs(int64_t realUs) {
249    Mutex::Autolock autoLock(mTimeLock);
250    mPauseStartedTimeRealUs = realUs;
251}
252
253status_t NuPlayer::Renderer::openAudioSink(
254        const sp<AMessage> &format,
255        bool offloadOnly,
256        bool hasVideo,
257        uint32_t flags,
258        bool *isOffloaded) {
259    sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, id());
260    msg->setMessage("format", format);
261    msg->setInt32("offload-only", offloadOnly);
262    msg->setInt32("has-video", hasVideo);
263    msg->setInt32("flags", flags);
264
265    sp<AMessage> response;
266    msg->postAndAwaitResponse(&response);
267
268    int32_t err;
269    if (!response->findInt32("err", &err)) {
270        err = INVALID_OPERATION;
271    } else if (err == OK && isOffloaded != NULL) {
272        int32_t offload;
273        CHECK(response->findInt32("offload", &offload));
274        *isOffloaded = (offload != 0);
275    }
276    return err;
277}
278
279void NuPlayer::Renderer::closeAudioSink() {
280    sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, id());
281
282    sp<AMessage> response;
283    msg->postAndAwaitResponse(&response);
284}
285
286void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
287    switch (msg->what()) {
288        case kWhatOpenAudioSink:
289        {
290            sp<AMessage> format;
291            CHECK(msg->findMessage("format", &format));
292
293            int32_t offloadOnly;
294            CHECK(msg->findInt32("offload-only", &offloadOnly));
295
296            int32_t hasVideo;
297            CHECK(msg->findInt32("has-video", &hasVideo));
298
299            uint32_t flags;
300            CHECK(msg->findInt32("flags", (int32_t *)&flags));
301
302            status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
303
304            sp<AMessage> response = new AMessage;
305            response->setInt32("err", err);
306            response->setInt32("offload", offloadingAudio());
307
308            uint32_t replyID;
309            CHECK(msg->senderAwaitsResponse(&replyID));
310            response->postReply(replyID);
311
312            break;
313        }
314
315        case kWhatCloseAudioSink:
316        {
317            uint32_t replyID;
318            CHECK(msg->senderAwaitsResponse(&replyID));
319
320            onCloseAudioSink();
321
322            sp<AMessage> response = new AMessage;
323            response->postReply(replyID);
324            break;
325        }
326
327        case kWhatStopAudioSink:
328        {
329            mAudioSink->stop();
330            break;
331        }
332
333        case kWhatDrainAudioQueue:
334        {
335            int32_t generation;
336            CHECK(msg->findInt32("generation", &generation));
337            if (generation != mAudioQueueGeneration) {
338                break;
339            }
340
341            mDrainAudioQueuePending = false;
342
343            if (onDrainAudioQueue()) {
344                uint32_t numFramesPlayed;
345                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
346                         (status_t)OK);
347
348                uint32_t numFramesPendingPlayout =
349                    mNumFramesWritten - numFramesPlayed;
350
351                // This is how long the audio sink will have data to
352                // play back.
353                int64_t delayUs =
354                    mAudioSink->msecsPerFrame()
355                        * numFramesPendingPlayout * 1000ll;
356
357                // Let's give it more data after about half that time
358                // has elapsed.
359                // kWhatDrainAudioQueue is used for non-offloading mode,
360                // and mLock is used only for offloading mode. Therefore,
361                // no need to acquire mLock here.
362                postDrainAudioQueue_l(delayUs / 2);
363            }
364            break;
365        }
366
367        case kWhatDrainVideoQueue:
368        {
369            int32_t generation;
370            CHECK(msg->findInt32("generation", &generation));
371            if (generation != mVideoQueueGeneration) {
372                break;
373            }
374
375            mDrainVideoQueuePending = false;
376
377            onDrainVideoQueue();
378
379            postDrainVideoQueue();
380            break;
381        }
382
383        case kWhatPostDrainVideoQueue:
384        {
385            int32_t generation;
386            CHECK(msg->findInt32("generation", &generation));
387            if (generation != mVideoQueueGeneration) {
388                break;
389            }
390
391            mDrainVideoQueuePending = false;
392            postDrainVideoQueue();
393            break;
394        }
395
396        case kWhatQueueBuffer:
397        {
398            onQueueBuffer(msg);
399            break;
400        }
401
402        case kWhatQueueEOS:
403        {
404            onQueueEOS(msg);
405            break;
406        }
407
408        case kWhatFlush:
409        {
410            onFlush(msg);
411            break;
412        }
413
414        case kWhatAudioSinkChanged:
415        {
416            onAudioSinkChanged();
417            break;
418        }
419
420        case kWhatDisableOffloadAudio:
421        {
422            onDisableOffloadAudio();
423            break;
424        }
425
426        case kWhatEnableOffloadAudio:
427        {
428            onEnableOffloadAudio();
429            break;
430        }
431
432        case kWhatPause:
433        {
434            onPause();
435            break;
436        }
437
438        case kWhatResume:
439        {
440            onResume();
441            break;
442        }
443
444        case kWhatSetVideoFrameRate:
445        {
446            float fps;
447            CHECK(msg->findFloat("frame-rate", &fps));
448            onSetVideoFrameRate(fps);
449            break;
450        }
451
452        case kWhatAudioOffloadTearDown:
453        {
454            onAudioOffloadTearDown(kDueToError);
455            break;
456        }
457
458        case kWhatAudioOffloadPauseTimeout:
459        {
460            int32_t generation;
461            CHECK(msg->findInt32("generation", &generation));
462            if (generation != mAudioOffloadPauseTimeoutGeneration) {
463                break;
464            }
465            ALOGV("Audio Offload tear down due to pause timeout.");
466            onAudioOffloadTearDown(kDueToTimeout);
467            break;
468        }
469
470        default:
471            TRESPASS();
472            break;
473    }
474}
475
476void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
477    if (mDrainAudioQueuePending || mSyncQueues || mPaused
478            || offloadingAudio()) {
479        return;
480    }
481
482    if (mAudioQueue.empty()) {
483        return;
484    }
485
486    mDrainAudioQueuePending = true;
487    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
488    msg->setInt32("generation", mAudioQueueGeneration);
489    msg->post(delayUs);
490}
491
492void NuPlayer::Renderer::prepareForMediaRenderingStart() {
493    mAudioRenderingStartGeneration = mAudioQueueGeneration;
494    mVideoRenderingStartGeneration = mVideoQueueGeneration;
495}
496
497void NuPlayer::Renderer::notifyIfMediaRenderingStarted() {
498    if (mVideoRenderingStartGeneration == mVideoQueueGeneration &&
499        mAudioRenderingStartGeneration == mAudioQueueGeneration) {
500        mVideoRenderingStartGeneration = -1;
501        mAudioRenderingStartGeneration = -1;
502
503        sp<AMessage> notify = mNotify->dup();
504        notify->setInt32("what", kWhatMediaRenderingStart);
505        notify->post();
506    }
507}
508
509// static
510size_t NuPlayer::Renderer::AudioSinkCallback(
511        MediaPlayerBase::AudioSink * /* audioSink */,
512        void *buffer,
513        size_t size,
514        void *cookie,
515        MediaPlayerBase::AudioSink::cb_event_t event) {
516    NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
517
518    switch (event) {
519        case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
520        {
521            return me->fillAudioBuffer(buffer, size);
522            break;
523        }
524
525        case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
526        {
527            me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
528            break;
529        }
530
531        case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
532        {
533            me->notifyAudioOffloadTearDown();
534            break;
535        }
536    }
537
538    return 0;
539}
540
541size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
542    Mutex::Autolock autoLock(mLock);
543
544    if (!offloadingAudio() || mPaused) {
545        return 0;
546    }
547
548    bool hasEOS = false;
549
550    size_t sizeCopied = 0;
551    bool firstEntry = true;
552    while (sizeCopied < size && !mAudioQueue.empty()) {
553        QueueEntry *entry = &*mAudioQueue.begin();
554
555        if (entry->mBuffer == NULL) { // EOS
556            hasEOS = true;
557            mAudioQueue.erase(mAudioQueue.begin());
558            entry = NULL;
559            break;
560        }
561
562        if (firstEntry && entry->mOffset == 0) {
563            firstEntry = false;
564            int64_t mediaTimeUs;
565            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
566            ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
567            setAudioFirstAnchorTimeIfNeeded(mediaTimeUs);
568        }
569
570        size_t copy = entry->mBuffer->size() - entry->mOffset;
571        size_t sizeRemaining = size - sizeCopied;
572        if (copy > sizeRemaining) {
573            copy = sizeRemaining;
574        }
575
576        memcpy((char *)buffer + sizeCopied,
577               entry->mBuffer->data() + entry->mOffset,
578               copy);
579
580        entry->mOffset += copy;
581        if (entry->mOffset == entry->mBuffer->size()) {
582            entry->mNotifyConsumed->post();
583            mAudioQueue.erase(mAudioQueue.begin());
584            entry = NULL;
585        }
586        sizeCopied += copy;
587        notifyIfMediaRenderingStarted();
588    }
589
590    if (mAudioFirstAnchorTimeMediaUs >= 0) {
591        int64_t nowUs = ALooper::GetNowUs();
592        setAnchorTime(mAudioFirstAnchorTimeMediaUs, nowUs - getPlayedOutAudioDurationUs(nowUs));
593    }
594
595    // we don't know how much data we are queueing for offloaded tracks
596    mAnchorMaxMediaUs = -1;
597
598    if (hasEOS) {
599        (new AMessage(kWhatStopAudioSink, id()))->post();
600    }
601
602    return sizeCopied;
603}
604
605bool NuPlayer::Renderer::onDrainAudioQueue() {
606    uint32_t numFramesPlayed;
607    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
608        return false;
609    }
610
611    ssize_t numFramesAvailableToWrite =
612        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
613
614#if 0
615    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
616        ALOGI("audio sink underrun");
617    } else {
618        ALOGV("audio queue has %d frames left to play",
619             mAudioSink->frameCount() - numFramesAvailableToWrite);
620    }
621#endif
622
623    size_t numBytesAvailableToWrite =
624        numFramesAvailableToWrite * mAudioSink->frameSize();
625
626    while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) {
627        QueueEntry *entry = &*mAudioQueue.begin();
628
629        mLastAudioBufferDrained = entry->mBufferOrdinal;
630
631        if (entry->mBuffer == NULL) {
632            // EOS
633            int64_t postEOSDelayUs = 0;
634            if (mAudioSink->needsTrailingPadding()) {
635                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
636            }
637            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
638
639            mAudioQueue.erase(mAudioQueue.begin());
640            entry = NULL;
641            if (mAudioSink->needsTrailingPadding()) {
642                // If we're not in gapless playback (i.e. through setNextPlayer), we
643                // need to stop the track here, because that will play out the last
644                // little bit at the end of the file. Otherwise short files won't play.
645                mAudioSink->stop();
646                mNumFramesWritten = 0;
647            }
648            return false;
649        }
650
651        if (entry->mOffset == 0) {
652            int64_t mediaTimeUs;
653            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
654            ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
655            onNewAudioMediaTime(mediaTimeUs);
656        }
657
658        size_t copy = entry->mBuffer->size() - entry->mOffset;
659        if (copy > numBytesAvailableToWrite) {
660            copy = numBytesAvailableToWrite;
661        }
662
663        ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, copy);
664        if (written < 0) {
665            // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
666            ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
667            break;
668        }
669
670        entry->mOffset += written;
671        if (entry->mOffset == entry->mBuffer->size()) {
672            entry->mNotifyConsumed->post();
673            mAudioQueue.erase(mAudioQueue.begin());
674
675            entry = NULL;
676        }
677
678        numBytesAvailableToWrite -= written;
679        size_t copiedFrames = written / mAudioSink->frameSize();
680        mNumFramesWritten += copiedFrames;
681
682        notifyIfMediaRenderingStarted();
683
684        if (written != (ssize_t)copy) {
685            // A short count was received from AudioSink::write()
686            //
687            // AudioSink write should block until exactly the number of bytes are delivered.
688            // But it may return with a short count (without an error) when:
689            //
690            // 1) Size to be copied is not a multiple of the frame size. We consider this fatal.
691            // 2) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
692
693            // (Case 1)
694            // Must be a multiple of the frame size.  If it is not a multiple of a frame size, it
695            // needs to fail, as we should not carry over fractional frames between calls.
696            CHECK_EQ(copy % mAudioSink->frameSize(), 0);
697
698            // (Case 2)
699            // Return early to the caller.
700            // Beware of calling immediately again as this may busy-loop if you are not careful.
701            ALOGW("AudioSink write short frame count %zd < %zu", written, copy);
702            break;
703        }
704    }
705    mAnchorMaxMediaUs =
706        mAnchorTimeMediaUs +
707                (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
708                        * 1000LL * mAudioSink->msecsPerFrame());
709
710    return !mAudioQueue.empty();
711}
712
713int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
714    int64_t writtenAudioDurationUs =
715        mNumFramesWritten * 1000LL * mAudioSink->msecsPerFrame();
716    return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs);
717}
718
719int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
720    int64_t currentPositionUs;
721    if (getCurrentPosition(&currentPositionUs, nowUs, true /* allowPastQueuedVideo */) != OK) {
722        // If failed to get current position, e.g. due to audio clock is not ready, then just
723        // play out video immediately without delay.
724        return nowUs;
725    }
726    return (mediaTimeUs - currentPositionUs) + nowUs;
727}
728
729void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
730    // TRICKY: vorbis decoder generates multiple frames with the same
731    // timestamp, so only update on the first frame with a given timestamp
732    if (mediaTimeUs == mAnchorTimeMediaUs) {
733        return;
734    }
735    setAudioFirstAnchorTimeIfNeeded(mediaTimeUs);
736    int64_t nowUs = ALooper::GetNowUs();
737    setAnchorTime(
738            mediaTimeUs, nowUs + getPendingAudioPlayoutDurationUs(nowUs), mNumFramesWritten);
739}
740
741void NuPlayer::Renderer::postDrainVideoQueue() {
742    if (mDrainVideoQueuePending
743            || mSyncQueues
744            || (mPaused && mVideoSampleReceived)) {
745        return;
746    }
747
748    if (mVideoQueue.empty()) {
749        return;
750    }
751
752    QueueEntry &entry = *mVideoQueue.begin();
753
754    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
755    msg->setInt32("generation", mVideoQueueGeneration);
756
757    if (entry.mBuffer == NULL) {
758        // EOS doesn't carry a timestamp.
759        msg->post();
760        mDrainVideoQueuePending = true;
761        return;
762    }
763
764    int64_t delayUs;
765    int64_t nowUs = ALooper::GetNowUs();
766    int64_t realTimeUs;
767    if (mFlags & FLAG_REAL_TIME) {
768        int64_t mediaTimeUs;
769        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
770        realTimeUs = mediaTimeUs;
771    } else {
772        int64_t mediaTimeUs;
773        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
774
775        if (mAnchorTimeMediaUs < 0) {
776            setAnchorTime(mediaTimeUs, nowUs);
777            mAnchorMaxMediaUs = mediaTimeUs;
778            realTimeUs = nowUs;
779        } else {
780            realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
781        }
782        if (!mHasAudio) {
783            mAnchorMaxMediaUs = mediaTimeUs + 100000; // smooth out videos >= 10fps
784        }
785
786        // Heuristics to handle situation when media time changed without a
787        // discontinuity. If we have not drained an audio buffer that was
788        // received after this buffer, repost in 10 msec. Otherwise repost
789        // in 500 msec.
790        delayUs = realTimeUs - nowUs;
791        if (delayUs > 500000) {
792            int64_t postDelayUs = 500000;
793            if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) {
794                postDelayUs = 10000;
795            }
796            msg->setWhat(kWhatPostDrainVideoQueue);
797            msg->post(postDelayUs);
798            mVideoScheduler->restart();
799            ALOGI("possible video time jump of %dms, retrying in %dms",
800                    (int)(delayUs / 1000), (int)(postDelayUs / 1000));
801            mDrainVideoQueuePending = true;
802            return;
803        }
804    }
805
806    realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
807    int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
808
809    delayUs = realTimeUs - nowUs;
810
811    ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
812    // post 2 display refreshes before rendering is due
813    msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
814
815    mDrainVideoQueuePending = true;
816}
817
818void NuPlayer::Renderer::onDrainVideoQueue() {
819    if (mVideoQueue.empty()) {
820        return;
821    }
822
823    QueueEntry *entry = &*mVideoQueue.begin();
824
825    if (entry->mBuffer == NULL) {
826        // EOS
827
828        notifyEOS(false /* audio */, entry->mFinalResult);
829
830        mVideoQueue.erase(mVideoQueue.begin());
831        entry = NULL;
832
833        setVideoLateByUs(0);
834        return;
835    }
836
837    int64_t nowUs = -1;
838    int64_t realTimeUs;
839    if (mFlags & FLAG_REAL_TIME) {
840        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
841    } else {
842        int64_t mediaTimeUs;
843        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
844
845        nowUs = ALooper::GetNowUs();
846        realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
847    }
848
849    bool tooLate = false;
850
851    if (!mPaused) {
852        if (nowUs == -1) {
853            nowUs = ALooper::GetNowUs();
854        }
855        setVideoLateByUs(nowUs - realTimeUs);
856        tooLate = (mVideoLateByUs > 40000);
857
858        if (tooLate) {
859            ALOGV("video late by %lld us (%.2f secs)",
860                 mVideoLateByUs, mVideoLateByUs / 1E6);
861        } else {
862            ALOGV("rendering video at media time %.2f secs",
863                    (mFlags & FLAG_REAL_TIME ? realTimeUs :
864                    (realTimeUs + mAnchorTimeMediaUs - mAnchorTimeRealUs)) / 1E6);
865        }
866    } else {
867        setVideoLateByUs(0);
868        if (!mVideoSampleReceived && !mHasAudio) {
869            // This will ensure that the first frame after a flush won't be used as anchor
870            // when renderer is in paused state, because resume can happen any time after seek.
871            setAnchorTime(-1, -1);
872        }
873    }
874
875    entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
876    entry->mNotifyConsumed->setInt32("render", !tooLate);
877    entry->mNotifyConsumed->post();
878    mVideoQueue.erase(mVideoQueue.begin());
879    entry = NULL;
880
881    mVideoSampleReceived = true;
882
883    if (!mPaused) {
884        if (!mVideoRenderingStarted) {
885            mVideoRenderingStarted = true;
886            notifyVideoRenderingStart();
887        }
888        notifyIfMediaRenderingStarted();
889    }
890}
891
892void NuPlayer::Renderer::notifyVideoRenderingStart() {
893    sp<AMessage> notify = mNotify->dup();
894    notify->setInt32("what", kWhatVideoRenderingStart);
895    notify->post();
896}
897
898void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
899    sp<AMessage> notify = mNotify->dup();
900    notify->setInt32("what", kWhatEOS);
901    notify->setInt32("audio", static_cast<int32_t>(audio));
902    notify->setInt32("finalResult", finalResult);
903    notify->post(delayUs);
904}
905
906void NuPlayer::Renderer::notifyAudioOffloadTearDown() {
907    (new AMessage(kWhatAudioOffloadTearDown, id()))->post();
908}
909
910void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
911    int32_t audio;
912    CHECK(msg->findInt32("audio", &audio));
913
914    setHasMedia(audio);
915
916    if (mHasVideo) {
917        if (mVideoScheduler == NULL) {
918            mVideoScheduler = new VideoFrameScheduler();
919            mVideoScheduler->init();
920        }
921    }
922
923    if (dropBufferWhileFlushing(audio, msg)) {
924        return;
925    }
926
927    sp<ABuffer> buffer;
928    CHECK(msg->findBuffer("buffer", &buffer));
929
930    sp<AMessage> notifyConsumed;
931    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
932
933    QueueEntry entry;
934    entry.mBuffer = buffer;
935    entry.mNotifyConsumed = notifyConsumed;
936    entry.mOffset = 0;
937    entry.mFinalResult = OK;
938    entry.mBufferOrdinal = ++mTotalBuffersQueued;
939
940    if (audio) {
941        Mutex::Autolock autoLock(mLock);
942        mAudioQueue.push_back(entry);
943        postDrainAudioQueue_l();
944    } else {
945        mVideoQueue.push_back(entry);
946        postDrainVideoQueue();
947    }
948
949    Mutex::Autolock autoLock(mLock);
950    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
951        return;
952    }
953
954    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
955    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
956
957    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
958        // EOS signalled on either queue.
959        syncQueuesDone_l();
960        return;
961    }
962
963    int64_t firstAudioTimeUs;
964    int64_t firstVideoTimeUs;
965    CHECK(firstAudioBuffer->meta()
966            ->findInt64("timeUs", &firstAudioTimeUs));
967    CHECK(firstVideoBuffer->meta()
968            ->findInt64("timeUs", &firstVideoTimeUs));
969
970    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
971
972    ALOGV("queueDiff = %.2f secs", diff / 1E6);
973
974    if (diff > 100000ll) {
975        // Audio data starts More than 0.1 secs before video.
976        // Drop some audio.
977
978        (*mAudioQueue.begin()).mNotifyConsumed->post();
979        mAudioQueue.erase(mAudioQueue.begin());
980        return;
981    }
982
983    syncQueuesDone_l();
984}
985
986void NuPlayer::Renderer::syncQueuesDone_l() {
987    if (!mSyncQueues) {
988        return;
989    }
990
991    mSyncQueues = false;
992
993    if (!mAudioQueue.empty()) {
994        postDrainAudioQueue_l();
995    }
996
997    if (!mVideoQueue.empty()) {
998        postDrainVideoQueue();
999    }
1000}
1001
1002void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1003    int32_t audio;
1004    CHECK(msg->findInt32("audio", &audio));
1005
1006    if (dropBufferWhileFlushing(audio, msg)) {
1007        return;
1008    }
1009
1010    int32_t finalResult;
1011    CHECK(msg->findInt32("finalResult", &finalResult));
1012
1013    QueueEntry entry;
1014    entry.mOffset = 0;
1015    entry.mFinalResult = finalResult;
1016
1017    if (audio) {
1018        Mutex::Autolock autoLock(mLock);
1019        if (mAudioQueue.empty() && mSyncQueues) {
1020            syncQueuesDone_l();
1021        }
1022        mAudioQueue.push_back(entry);
1023        postDrainAudioQueue_l();
1024    } else {
1025        if (mVideoQueue.empty() && mSyncQueues) {
1026            Mutex::Autolock autoLock(mLock);
1027            syncQueuesDone_l();
1028        }
1029        mVideoQueue.push_back(entry);
1030        postDrainVideoQueue();
1031    }
1032}
1033
1034void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
1035    int32_t audio, notifyComplete;
1036    CHECK(msg->findInt32("audio", &audio));
1037
1038    {
1039        Mutex::Autolock autoLock(mFlushLock);
1040        if (audio) {
1041            mFlushingAudio = false;
1042            notifyComplete = mNotifyCompleteAudio;
1043            mNotifyCompleteAudio = false;
1044        } else {
1045            mFlushingVideo = false;
1046            notifyComplete = mNotifyCompleteVideo;
1047            mNotifyCompleteVideo = false;
1048        }
1049    }
1050
1051    // If we're currently syncing the queues, i.e. dropping audio while
1052    // aligning the first audio/video buffer times and only one of the
1053    // two queues has data, we may starve that queue by not requesting
1054    // more buffers from the decoder. If the other source then encounters
1055    // a discontinuity that leads to flushing, we'll never find the
1056    // corresponding discontinuity on the other queue.
1057    // Therefore we'll stop syncing the queues if at least one of them
1058    // is flushed.
1059    {
1060         Mutex::Autolock autoLock(mLock);
1061         syncQueuesDone_l();
1062         setPauseStartedTimeRealUs(-1);
1063         setAnchorTime(-1, -1);
1064    }
1065
1066    ALOGV("flushing %s", audio ? "audio" : "video");
1067    if (audio) {
1068        {
1069            Mutex::Autolock autoLock(mLock);
1070            flushQueue(&mAudioQueue);
1071
1072            ++mAudioQueueGeneration;
1073            prepareForMediaRenderingStart();
1074
1075            if (offloadingAudio()) {
1076                setAudioFirstAnchorTime(-1);
1077            }
1078        }
1079
1080        mDrainAudioQueuePending = false;
1081
1082        if (offloadingAudio()) {
1083            mAudioSink->pause();
1084            mAudioSink->flush();
1085            mAudioSink->start();
1086        }
1087    } else {
1088        flushQueue(&mVideoQueue);
1089
1090        mDrainVideoQueuePending = false;
1091        ++mVideoQueueGeneration;
1092
1093        if (mVideoScheduler != NULL) {
1094            mVideoScheduler->restart();
1095        }
1096
1097        prepareForMediaRenderingStart();
1098    }
1099
1100    mVideoSampleReceived = false;
1101
1102    if (notifyComplete) {
1103        notifyFlushComplete(audio);
1104    }
1105}
1106
1107void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1108    while (!queue->empty()) {
1109        QueueEntry *entry = &*queue->begin();
1110
1111        if (entry->mBuffer != NULL) {
1112            entry->mNotifyConsumed->post();
1113        }
1114
1115        queue->erase(queue->begin());
1116        entry = NULL;
1117    }
1118}
1119
1120void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1121    sp<AMessage> notify = mNotify->dup();
1122    notify->setInt32("what", kWhatFlushComplete);
1123    notify->setInt32("audio", static_cast<int32_t>(audio));
1124    notify->post();
1125}
1126
1127bool NuPlayer::Renderer::dropBufferWhileFlushing(
1128        bool audio, const sp<AMessage> &msg) {
1129    bool flushing = false;
1130
1131    {
1132        Mutex::Autolock autoLock(mFlushLock);
1133        if (audio) {
1134            flushing = mFlushingAudio;
1135        } else {
1136            flushing = mFlushingVideo;
1137        }
1138    }
1139
1140    if (!flushing) {
1141        return false;
1142    }
1143
1144    sp<AMessage> notifyConsumed;
1145    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1146        notifyConsumed->post();
1147    }
1148
1149    return true;
1150}
1151
1152void NuPlayer::Renderer::onAudioSinkChanged() {
1153    if (offloadingAudio()) {
1154        return;
1155    }
1156    CHECK(!mDrainAudioQueuePending);
1157    mNumFramesWritten = 0;
1158    mAnchorNumFramesWritten = -1;
1159    uint32_t written;
1160    if (mAudioSink->getFramesWritten(&written) == OK) {
1161        mNumFramesWritten = written;
1162    }
1163}
1164
1165void NuPlayer::Renderer::onDisableOffloadAudio() {
1166    Mutex::Autolock autoLock(mLock);
1167    mFlags &= ~FLAG_OFFLOAD_AUDIO;
1168    ++mAudioQueueGeneration;
1169}
1170
1171void NuPlayer::Renderer::onEnableOffloadAudio() {
1172    Mutex::Autolock autoLock(mLock);
1173    mFlags |= FLAG_OFFLOAD_AUDIO;
1174    ++mAudioQueueGeneration;
1175}
1176
1177void NuPlayer::Renderer::onPause() {
1178    if (mPaused) {
1179        ALOGW("Renderer::onPause() called while already paused!");
1180        return;
1181    }
1182    {
1183        Mutex::Autolock autoLock(mLock);
1184        ++mAudioQueueGeneration;
1185        ++mVideoQueueGeneration;
1186        prepareForMediaRenderingStart();
1187        mPaused = true;
1188        setPauseStartedTimeRealUs(ALooper::GetNowUs());
1189    }
1190
1191    mDrainAudioQueuePending = false;
1192    mDrainVideoQueuePending = false;
1193
1194    if (mHasAudio) {
1195        mAudioSink->pause();
1196        startAudioOffloadPauseTimeout();
1197    }
1198
1199    ALOGV("now paused audio queue has %d entries, video has %d entries",
1200          mAudioQueue.size(), mVideoQueue.size());
1201}
1202
1203void NuPlayer::Renderer::onResume() {
1204    if (!mPaused) {
1205        return;
1206    }
1207
1208    if (mHasAudio) {
1209        cancelAudioOffloadPauseTimeout();
1210        mAudioSink->start();
1211    }
1212
1213    Mutex::Autolock autoLock(mLock);
1214    mPaused = false;
1215    if (mPauseStartedTimeRealUs != -1) {
1216        int64_t newAnchorRealUs =
1217            mAnchorTimeRealUs + ALooper::GetNowUs() - mPauseStartedTimeRealUs;
1218        setAnchorTime(
1219                mAnchorTimeMediaUs, newAnchorRealUs, mAnchorNumFramesWritten, true /* resume */);
1220    }
1221
1222    if (!mAudioQueue.empty()) {
1223        postDrainAudioQueue_l();
1224    }
1225
1226    if (!mVideoQueue.empty()) {
1227        postDrainVideoQueue();
1228    }
1229}
1230
1231void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1232    if (mVideoScheduler == NULL) {
1233        mVideoScheduler = new VideoFrameScheduler();
1234    }
1235    mVideoScheduler->init(fps);
1236}
1237
1238// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs()
1239// as it acquires locks and may query the audio driver.
1240//
1241// Some calls could conceivably retrieve extrapolated data instead of
1242// accessing getTimestamp() or getPosition() every time a data buffer with
1243// a media time is received.
1244//
1245int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) {
1246    uint32_t numFramesPlayed;
1247    int64_t numFramesPlayedAt;
1248    AudioTimestamp ts;
1249    static const int64_t kStaleTimestamp100ms = 100000;
1250
1251    status_t res = mAudioSink->getTimestamp(ts);
1252    if (res == OK) {                 // case 1: mixing audio tracks and offloaded tracks.
1253        numFramesPlayed = ts.mPosition;
1254        numFramesPlayedAt =
1255            ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
1256        const int64_t timestampAge = nowUs - numFramesPlayedAt;
1257        if (timestampAge > kStaleTimestamp100ms) {
1258            // This is an audio FIXME.
1259            // getTimestamp returns a timestamp which may come from audio mixing threads.
1260            // After pausing, the MixerThread may go idle, thus the mTime estimate may
1261            // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms,
1262            // the max latency should be about 25ms with an average around 12ms (to be verified).
1263            // For safety we use 100ms.
1264            ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)",
1265                    (long long)nowUs, (long long)numFramesPlayedAt);
1266            numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
1267        }
1268        //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1269    } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track
1270        numFramesPlayed = 0;
1271        numFramesPlayedAt = nowUs;
1272        //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
1273        //        numFramesPlayed, (long long)numFramesPlayedAt);
1274    } else {                         // case 3: transitory at new track or audio fast tracks.
1275        res = mAudioSink->getPosition(&numFramesPlayed);
1276        CHECK_EQ(res, (status_t)OK);
1277        numFramesPlayedAt = nowUs;
1278        numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */
1279        //ALOGD("getPosition: %d %lld", numFramesPlayed, numFramesPlayedAt);
1280    }
1281
1282    // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
1283    //CHECK_EQ(numFramesPlayed & (1 << 31), 0);  // can't be negative until 12.4 hrs, test
1284    int64_t durationUs = (int64_t)((int32_t)numFramesPlayed * 1000LL * mAudioSink->msecsPerFrame())
1285            + nowUs - numFramesPlayedAt;
1286    if (durationUs < 0) {
1287        // Occurs when numFramesPlayed position is very small and the following:
1288        // (1) In case 1, the time nowUs is computed before getTimestamp() is called and
1289        //     numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed.
1290        // (2) In case 3, using getPosition and adding mAudioSink->latency() to
1291        //     numFramesPlayedAt, by a time amount greater than numFramesPlayed.
1292        //
1293        // Both of these are transitory conditions.
1294        ALOGV("getPlayedOutAudioDurationUs: negative duration %lld set to zero", (long long)durationUs);
1295        durationUs = 0;
1296    }
1297    ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)",
1298            (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt);
1299    return durationUs;
1300}
1301
1302void NuPlayer::Renderer::onAudioOffloadTearDown(AudioOffloadTearDownReason reason) {
1303    if (mAudioOffloadTornDown) {
1304        return;
1305    }
1306    mAudioOffloadTornDown = true;
1307
1308    int64_t currentPositionUs;
1309    if (getCurrentPosition(&currentPositionUs) != OK) {
1310        currentPositionUs = 0;
1311    }
1312
1313    mAudioSink->stop();
1314    mAudioSink->flush();
1315
1316    sp<AMessage> notify = mNotify->dup();
1317    notify->setInt32("what", kWhatAudioOffloadTearDown);
1318    notify->setInt64("positionUs", currentPositionUs);
1319    notify->setInt32("reason", reason);
1320    notify->post();
1321}
1322
1323void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1324    if (offloadingAudio()) {
1325        sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, id());
1326        msg->setInt32("generation", mAudioOffloadPauseTimeoutGeneration);
1327        msg->post(kOffloadPauseMaxUs);
1328    }
1329}
1330
1331void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
1332    if (offloadingAudio()) {
1333        ++mAudioOffloadPauseTimeoutGeneration;
1334    }
1335}
1336
1337status_t NuPlayer::Renderer::onOpenAudioSink(
1338        const sp<AMessage> &format,
1339        bool offloadOnly,
1340        bool hasVideo,
1341        uint32_t flags) {
1342    ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1343            offloadOnly, offloadingAudio());
1344    bool audioSinkChanged = false;
1345
1346    int32_t numChannels;
1347    CHECK(format->findInt32("channel-count", &numChannels));
1348
1349    int32_t channelMask;
1350    if (!format->findInt32("channel-mask", &channelMask)) {
1351        // signal to the AudioSink to derive the mask from count.
1352        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1353    }
1354
1355    int32_t sampleRate;
1356    CHECK(format->findInt32("sample-rate", &sampleRate));
1357
1358    if (offloadingAudio()) {
1359        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1360        AString mime;
1361        CHECK(format->findString("mime", &mime));
1362        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1363
1364        if (err != OK) {
1365            ALOGE("Couldn't map mime \"%s\" to a valid "
1366                    "audio_format", mime.c_str());
1367            onDisableOffloadAudio();
1368        } else {
1369            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1370                    mime.c_str(), audioFormat);
1371
1372            int avgBitRate = -1;
1373            format->findInt32("bit-rate", &avgBitRate);
1374
1375            int32_t aacProfile = -1;
1376            if (audioFormat == AUDIO_FORMAT_AAC
1377                    && format->findInt32("aac-profile", &aacProfile)) {
1378                // Redefine AAC format as per aac profile
1379                mapAACProfileToAudioFormat(
1380                        audioFormat,
1381                        aacProfile);
1382            }
1383
1384            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1385            offloadInfo.duration_us = -1;
1386            format->findInt64(
1387                    "durationUs", &offloadInfo.duration_us);
1388            offloadInfo.sample_rate = sampleRate;
1389            offloadInfo.channel_mask = channelMask;
1390            offloadInfo.format = audioFormat;
1391            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1392            offloadInfo.bit_rate = avgBitRate;
1393            offloadInfo.has_video = hasVideo;
1394            offloadInfo.is_streaming = true;
1395
1396            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1397                ALOGV("openAudioSink: no change in offload mode");
1398                // no change from previous configuration, everything ok.
1399                return OK;
1400            }
1401            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1402            uint32_t offloadFlags = flags;
1403            offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1404            offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1405            audioSinkChanged = true;
1406            mAudioSink->close();
1407            err = mAudioSink->open(
1408                    sampleRate,
1409                    numChannels,
1410                    (audio_channel_mask_t)channelMask,
1411                    audioFormat,
1412                    8 /* bufferCount */,
1413                    &NuPlayer::Renderer::AudioSinkCallback,
1414                    this,
1415                    (audio_output_flags_t)offloadFlags,
1416                    &offloadInfo);
1417
1418            if (err == OK) {
1419                // If the playback is offloaded to h/w, we pass
1420                // the HAL some metadata information.
1421                // We don't want to do this for PCM because it
1422                // will be going through the AudioFlinger mixer
1423                // before reaching the hardware.
1424                // TODO
1425                mCurrentOffloadInfo = offloadInfo;
1426                err = mAudioSink->start();
1427                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1428            }
1429            if (err != OK) {
1430                // Clean up, fall back to non offload mode.
1431                mAudioSink->close();
1432                onDisableOffloadAudio();
1433                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1434                ALOGV("openAudioSink: offload failed");
1435            }
1436        }
1437    }
1438    if (!offloadOnly && !offloadingAudio()) {
1439        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1440        uint32_t pcmFlags = flags;
1441        pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1442        audioSinkChanged = true;
1443        mAudioSink->close();
1444        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1445        status_t err = mAudioSink->open(
1446                    sampleRate,
1447                    numChannels,
1448                    (audio_channel_mask_t)channelMask,
1449                    AUDIO_FORMAT_PCM_16_BIT,
1450                    8 /* bufferCount */,
1451                    NULL,
1452                    NULL,
1453                    (audio_output_flags_t)pcmFlags);
1454        if (err != OK) {
1455            ALOGW("openAudioSink: non offloaded open failed status: %d", err);
1456            return err;
1457        }
1458        mAudioSink->start();
1459    }
1460    if (audioSinkChanged) {
1461        onAudioSinkChanged();
1462    }
1463    if (offloadingAudio()) {
1464        mAudioOffloadTornDown = false;
1465    }
1466    return OK;
1467}
1468
1469void NuPlayer::Renderer::onCloseAudioSink() {
1470    mAudioSink->close();
1471    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1472}
1473
1474}  // namespace android
1475
1476