NuPlayerRenderer.cpp revision df809479696725faf5d3424b11fa8a07bd94cb5e 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26#include <media/stagefright/foundation/AUtils.h>
27#include <media/stagefright/foundation/AWakeLock.h>
28#include <media/stagefright/MediaClock.h>
29#include <media/stagefright/MediaErrors.h>
30#include <media/stagefright/MetaData.h>
31#include <media/stagefright/Utils.h>
32
33#include <VideoFrameScheduler.h>
34
35#include <inttypes.h>
36
37namespace android {
38
39// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
40// is closed to allow the audio DSP to power down.
41static const int64_t kOffloadPauseMaxUs = 10000000ll;
42
43// static
44const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
45        AUDIO_CHANNEL_NONE,
46        AUDIO_OUTPUT_FLAG_NONE,
47        AUDIO_FORMAT_INVALID,
48        0, // mNumChannels
49        0 // mSampleRate
50};
51
52// static
53const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
54
55NuPlayer::Renderer::Renderer(
56        const sp<MediaPlayerBase::AudioSink> &sink,
57        const sp<AMessage> &notify,
58        uint32_t flags)
59    : mAudioSink(sink),
60      mNotify(notify),
61      mFlags(flags),
62      mNumFramesWritten(0),
63      mDrainAudioQueuePending(false),
64      mDrainVideoQueuePending(false),
65      mAudioQueueGeneration(0),
66      mVideoQueueGeneration(0),
67      mAudioDrainGeneration(0),
68      mVideoDrainGeneration(0),
69      mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
70      mAudioFirstAnchorTimeMediaUs(-1),
71      mAnchorTimeMediaUs(-1),
72      mAnchorNumFramesWritten(-1),
73      mVideoLateByUs(0ll),
74      mHasAudio(false),
75      mHasVideo(false),
76      mNotifyCompleteAudio(false),
77      mNotifyCompleteVideo(false),
78      mSyncQueues(false),
79      mPaused(false),
80      mVideoSampleReceived(false),
81      mVideoRenderingStarted(false),
82      mVideoRenderingStartGeneration(0),
83      mAudioRenderingStartGeneration(0),
84      mAudioOffloadPauseTimeoutGeneration(0),
85      mAudioTornDown(false),
86      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
87      mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
88      mTotalBuffersQueued(0),
89      mLastAudioBufferDrained(0),
90      mWakeLock(new AWakeLock()) {
91    mMediaClock = new MediaClock;
92    mPlaybackRate = mPlaybackSettings.mSpeed;
93    mMediaClock->setPlaybackRate(mPlaybackRate);
94}
95
96NuPlayer::Renderer::~Renderer() {
97    if (offloadingAudio()) {
98        mAudioSink->stop();
99        mAudioSink->flush();
100        mAudioSink->close();
101    }
102}
103
104void NuPlayer::Renderer::queueBuffer(
105        bool audio,
106        const sp<ABuffer> &buffer,
107        const sp<AMessage> &notifyConsumed) {
108    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
109    msg->setInt32("queueGeneration", getQueueGeneration(audio));
110    msg->setInt32("audio", static_cast<int32_t>(audio));
111    msg->setBuffer("buffer", buffer);
112    msg->setMessage("notifyConsumed", notifyConsumed);
113    msg->post();
114}
115
116void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
117    CHECK_NE(finalResult, (status_t)OK);
118
119    sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
120    msg->setInt32("queueGeneration", getQueueGeneration(audio));
121    msg->setInt32("audio", static_cast<int32_t>(audio));
122    msg->setInt32("finalResult", finalResult);
123    msg->post();
124}
125
126status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) {
127    sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
128    writeToAMessage(msg, rate);
129    sp<AMessage> response;
130    status_t err = msg->postAndAwaitResponse(&response);
131    if (err == OK && response != NULL) {
132        CHECK(response->findInt32("err", &err));
133    }
134    return err;
135}
136
137status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) {
138    if (rate.mSpeed == 0.f) {
139        onPause();
140        // don't call audiosink's setPlaybackRate if pausing, as pitch does not
141        // have to correspond to the any non-0 speed (e.g old speed). Keep
142        // settings nonetheless, using the old speed, in case audiosink changes.
143        AudioPlaybackRate newRate = rate;
144        newRate.mSpeed = mPlaybackSettings.mSpeed;
145        mPlaybackSettings = newRate;
146        return OK;
147    }
148
149    if (mAudioSink != NULL && mAudioSink->ready()) {
150        status_t err = mAudioSink->setPlaybackRate(rate);
151        if (err != OK) {
152            return err;
153        }
154    }
155    mPlaybackSettings = rate;
156    mPlaybackRate = rate.mSpeed;
157    mMediaClock->setPlaybackRate(mPlaybackRate);
158    return OK;
159}
160
161status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
162    sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
163    sp<AMessage> response;
164    status_t err = msg->postAndAwaitResponse(&response);
165    if (err == OK && response != NULL) {
166        CHECK(response->findInt32("err", &err));
167        if (err == OK) {
168            readFromAMessage(response, rate);
169        }
170    }
171    return err;
172}
173
174status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
175    if (mAudioSink != NULL && mAudioSink->ready()) {
176        status_t err = mAudioSink->getPlaybackRate(rate);
177        if (err == OK) {
178            if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) {
179                ALOGW("correcting mismatch in internal/external playback rate");
180            }
181            // get playback settings used by audiosink, as it may be
182            // slightly off due to audiosink not taking small changes.
183            mPlaybackSettings = *rate;
184            if (mPaused) {
185                rate->mSpeed = 0.f;
186            }
187        }
188        return err;
189    }
190    *rate = mPlaybackSettings;
191    return OK;
192}
193
194status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
195    sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
196    writeToAMessage(msg, sync, videoFpsHint);
197    sp<AMessage> response;
198    status_t err = msg->postAndAwaitResponse(&response);
199    if (err == OK && response != NULL) {
200        CHECK(response->findInt32("err", &err));
201    }
202    return err;
203}
204
205status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) {
206    if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
207        return BAD_VALUE;
208    }
209    // TODO: support sync sources
210    return INVALID_OPERATION;
211}
212
213status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) {
214    sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
215    sp<AMessage> response;
216    status_t err = msg->postAndAwaitResponse(&response);
217    if (err == OK && response != NULL) {
218        CHECK(response->findInt32("err", &err));
219        if (err == OK) {
220            readFromAMessage(response, sync, videoFps);
221        }
222    }
223    return err;
224}
225
226status_t NuPlayer::Renderer::onGetSyncSettings(
227        AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
228    *sync = mSyncSettings;
229    *videoFps = -1.f;
230    return OK;
231}
232
233void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
234    {
235        Mutex::Autolock autoLock(mLock);
236        if (audio) {
237            mNotifyCompleteAudio |= notifyComplete;
238            ++mAudioQueueGeneration;
239            ++mAudioDrainGeneration;
240        } else {
241            mNotifyCompleteVideo |= notifyComplete;
242            ++mVideoQueueGeneration;
243            ++mVideoDrainGeneration;
244        }
245
246        clearAnchorTime_l();
247        clearAudioFirstAnchorTime_l();
248        mVideoLateByUs = 0;
249        mSyncQueues = false;
250    }
251
252    sp<AMessage> msg = new AMessage(kWhatFlush, this);
253    msg->setInt32("audio", static_cast<int32_t>(audio));
254    msg->post();
255}
256
257void NuPlayer::Renderer::signalTimeDiscontinuity() {
258}
259
260void NuPlayer::Renderer::signalDisableOffloadAudio() {
261    (new AMessage(kWhatDisableOffloadAudio, this))->post();
262}
263
264void NuPlayer::Renderer::signalEnableOffloadAudio() {
265    (new AMessage(kWhatEnableOffloadAudio, this))->post();
266}
267
268void NuPlayer::Renderer::pause() {
269    (new AMessage(kWhatPause, this))->post();
270}
271
272void NuPlayer::Renderer::resume() {
273    (new AMessage(kWhatResume, this))->post();
274}
275
276void NuPlayer::Renderer::setVideoFrameRate(float fps) {
277    sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
278    msg->setFloat("frame-rate", fps);
279    msg->post();
280}
281
282// Called on any threads.
283status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
284    return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
285}
286
287void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
288    mAudioFirstAnchorTimeMediaUs = -1;
289    mMediaClock->setStartingTimeMedia(-1);
290}
291
292void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
293    if (mAudioFirstAnchorTimeMediaUs == -1) {
294        mAudioFirstAnchorTimeMediaUs = mediaUs;
295        mMediaClock->setStartingTimeMedia(mediaUs);
296    }
297}
298
299void NuPlayer::Renderer::clearAnchorTime_l() {
300    mMediaClock->clearAnchor();
301    mAnchorTimeMediaUs = -1;
302    mAnchorNumFramesWritten = -1;
303}
304
305void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
306    Mutex::Autolock autoLock(mLock);
307    mVideoLateByUs = lateUs;
308}
309
310int64_t NuPlayer::Renderer::getVideoLateByUs() {
311    Mutex::Autolock autoLock(mLock);
312    return mVideoLateByUs;
313}
314
315status_t NuPlayer::Renderer::openAudioSink(
316        const sp<AMessage> &format,
317        bool offloadOnly,
318        bool hasVideo,
319        uint32_t flags,
320        bool *isOffloaded) {
321    sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
322    msg->setMessage("format", format);
323    msg->setInt32("offload-only", offloadOnly);
324    msg->setInt32("has-video", hasVideo);
325    msg->setInt32("flags", flags);
326
327    sp<AMessage> response;
328    msg->postAndAwaitResponse(&response);
329
330    int32_t err;
331    if (!response->findInt32("err", &err)) {
332        err = INVALID_OPERATION;
333    } else if (err == OK && isOffloaded != NULL) {
334        int32_t offload;
335        CHECK(response->findInt32("offload", &offload));
336        *isOffloaded = (offload != 0);
337    }
338    return err;
339}
340
341void NuPlayer::Renderer::closeAudioSink() {
342    sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
343
344    sp<AMessage> response;
345    msg->postAndAwaitResponse(&response);
346}
347
348void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
349    switch (msg->what()) {
350        case kWhatOpenAudioSink:
351        {
352            sp<AMessage> format;
353            CHECK(msg->findMessage("format", &format));
354
355            int32_t offloadOnly;
356            CHECK(msg->findInt32("offload-only", &offloadOnly));
357
358            int32_t hasVideo;
359            CHECK(msg->findInt32("has-video", &hasVideo));
360
361            uint32_t flags;
362            CHECK(msg->findInt32("flags", (int32_t *)&flags));
363
364            status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
365
366            sp<AMessage> response = new AMessage;
367            response->setInt32("err", err);
368            response->setInt32("offload", offloadingAudio());
369
370            sp<AReplyToken> replyID;
371            CHECK(msg->senderAwaitsResponse(&replyID));
372            response->postReply(replyID);
373
374            break;
375        }
376
377        case kWhatCloseAudioSink:
378        {
379            sp<AReplyToken> replyID;
380            CHECK(msg->senderAwaitsResponse(&replyID));
381
382            onCloseAudioSink();
383
384            sp<AMessage> response = new AMessage;
385            response->postReply(replyID);
386            break;
387        }
388
389        case kWhatStopAudioSink:
390        {
391            mAudioSink->stop();
392            break;
393        }
394
395        case kWhatDrainAudioQueue:
396        {
397            int32_t generation;
398            CHECK(msg->findInt32("drainGeneration", &generation));
399            if (generation != getDrainGeneration(true /* audio */)) {
400                break;
401            }
402
403            mDrainAudioQueuePending = false;
404
405            if (onDrainAudioQueue()) {
406                uint32_t numFramesPlayed;
407                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
408                         (status_t)OK);
409
410                uint32_t numFramesPendingPlayout =
411                    mNumFramesWritten - numFramesPlayed;
412
413                // This is how long the audio sink will have data to
414                // play back.
415                int64_t delayUs =
416                    mAudioSink->msecsPerFrame()
417                        * numFramesPendingPlayout * 1000ll;
418                if (mPlaybackRate > 1.0f) {
419                    delayUs /= mPlaybackRate;
420                }
421
422                // Let's give it more data after about half that time
423                // has elapsed.
424                Mutex::Autolock autoLock(mLock);
425                postDrainAudioQueue_l(delayUs / 2);
426            }
427            break;
428        }
429
430        case kWhatDrainVideoQueue:
431        {
432            int32_t generation;
433            CHECK(msg->findInt32("drainGeneration", &generation));
434            if (generation != getDrainGeneration(false /* audio */)) {
435                break;
436            }
437
438            mDrainVideoQueuePending = false;
439
440            onDrainVideoQueue();
441
442            postDrainVideoQueue();
443            break;
444        }
445
446        case kWhatPostDrainVideoQueue:
447        {
448            int32_t generation;
449            CHECK(msg->findInt32("drainGeneration", &generation));
450            if (generation != getDrainGeneration(false /* audio */)) {
451                break;
452            }
453
454            mDrainVideoQueuePending = false;
455            postDrainVideoQueue();
456            break;
457        }
458
459        case kWhatQueueBuffer:
460        {
461            onQueueBuffer(msg);
462            break;
463        }
464
465        case kWhatQueueEOS:
466        {
467            onQueueEOS(msg);
468            break;
469        }
470
471        case kWhatConfigPlayback:
472        {
473            sp<AReplyToken> replyID;
474            CHECK(msg->senderAwaitsResponse(&replyID));
475            AudioPlaybackRate rate;
476            readFromAMessage(msg, &rate);
477            status_t err = onConfigPlayback(rate);
478            sp<AMessage> response = new AMessage;
479            response->setInt32("err", err);
480            response->postReply(replyID);
481            break;
482        }
483
484        case kWhatGetPlaybackSettings:
485        {
486            sp<AReplyToken> replyID;
487            CHECK(msg->senderAwaitsResponse(&replyID));
488            AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT;
489            status_t err = onGetPlaybackSettings(&rate);
490            sp<AMessage> response = new AMessage;
491            if (err == OK) {
492                writeToAMessage(response, rate);
493            }
494            response->setInt32("err", err);
495            response->postReply(replyID);
496            break;
497        }
498
499        case kWhatConfigSync:
500        {
501            sp<AReplyToken> replyID;
502            CHECK(msg->senderAwaitsResponse(&replyID));
503            AVSyncSettings sync;
504            float videoFpsHint;
505            readFromAMessage(msg, &sync, &videoFpsHint);
506            status_t err = onConfigSync(sync, videoFpsHint);
507            sp<AMessage> response = new AMessage;
508            response->setInt32("err", err);
509            response->postReply(replyID);
510            break;
511        }
512
513        case kWhatGetSyncSettings:
514        {
515            sp<AReplyToken> replyID;
516            CHECK(msg->senderAwaitsResponse(&replyID));
517
518            ALOGV("kWhatGetSyncSettings");
519            AVSyncSettings sync;
520            float videoFps = -1.f;
521            status_t err = onGetSyncSettings(&sync, &videoFps);
522            sp<AMessage> response = new AMessage;
523            if (err == OK) {
524                writeToAMessage(response, sync, videoFps);
525            }
526            response->setInt32("err", err);
527            response->postReply(replyID);
528            break;
529        }
530
531        case kWhatFlush:
532        {
533            onFlush(msg);
534            break;
535        }
536
537        case kWhatDisableOffloadAudio:
538        {
539            onDisableOffloadAudio();
540            break;
541        }
542
543        case kWhatEnableOffloadAudio:
544        {
545            onEnableOffloadAudio();
546            break;
547        }
548
549        case kWhatPause:
550        {
551            onPause();
552            break;
553        }
554
555        case kWhatResume:
556        {
557            onResume();
558            break;
559        }
560
561        case kWhatSetVideoFrameRate:
562        {
563            float fps;
564            CHECK(msg->findFloat("frame-rate", &fps));
565            onSetVideoFrameRate(fps);
566            break;
567        }
568
569        case kWhatAudioTearDown:
570        {
571            onAudioTearDown(kDueToError);
572            break;
573        }
574
575        case kWhatAudioOffloadPauseTimeout:
576        {
577            int32_t generation;
578            CHECK(msg->findInt32("drainGeneration", &generation));
579            if (generation != mAudioOffloadPauseTimeoutGeneration) {
580                break;
581            }
582            ALOGV("Audio Offload tear down due to pause timeout.");
583            onAudioTearDown(kDueToTimeout);
584            mWakeLock->release();
585            break;
586        }
587
588        default:
589            TRESPASS();
590            break;
591    }
592}
593
594void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
595    if (mDrainAudioQueuePending || mSyncQueues || mPaused
596            || offloadingAudio()) {
597        return;
598    }
599
600    if (mAudioQueue.empty()) {
601        return;
602    }
603
604    mDrainAudioQueuePending = true;
605    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
606    msg->setInt32("drainGeneration", mAudioDrainGeneration);
607    msg->post(delayUs);
608}
609
610void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
611    mAudioRenderingStartGeneration = mAudioDrainGeneration;
612    mVideoRenderingStartGeneration = mVideoDrainGeneration;
613}
614
615void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
616    if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
617        mAudioRenderingStartGeneration == mAudioDrainGeneration) {
618        mVideoRenderingStartGeneration = -1;
619        mAudioRenderingStartGeneration = -1;
620
621        sp<AMessage> notify = mNotify->dup();
622        notify->setInt32("what", kWhatMediaRenderingStart);
623        notify->post();
624    }
625}
626
627// static
628size_t NuPlayer::Renderer::AudioSinkCallback(
629        MediaPlayerBase::AudioSink * /* audioSink */,
630        void *buffer,
631        size_t size,
632        void *cookie,
633        MediaPlayerBase::AudioSink::cb_event_t event) {
634    NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
635
636    switch (event) {
637        case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
638        {
639            return me->fillAudioBuffer(buffer, size);
640            break;
641        }
642
643        case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
644        {
645            me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
646            break;
647        }
648
649        case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
650        {
651            me->notifyAudioTearDown();
652            break;
653        }
654    }
655
656    return 0;
657}
658
659size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
660    Mutex::Autolock autoLock(mLock);
661
662    if (!offloadingAudio() || mPaused) {
663        return 0;
664    }
665
666    bool hasEOS = false;
667
668    size_t sizeCopied = 0;
669    bool firstEntry = true;
670    while (sizeCopied < size && !mAudioQueue.empty()) {
671        QueueEntry *entry = &*mAudioQueue.begin();
672
673        if (entry->mBuffer == NULL) { // EOS
674            hasEOS = true;
675            mAudioQueue.erase(mAudioQueue.begin());
676            entry = NULL;
677            break;
678        }
679
680        if (firstEntry && entry->mOffset == 0) {
681            firstEntry = false;
682            int64_t mediaTimeUs;
683            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
684            ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
685            setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
686        }
687
688        size_t copy = entry->mBuffer->size() - entry->mOffset;
689        size_t sizeRemaining = size - sizeCopied;
690        if (copy > sizeRemaining) {
691            copy = sizeRemaining;
692        }
693
694        memcpy((char *)buffer + sizeCopied,
695               entry->mBuffer->data() + entry->mOffset,
696               copy);
697
698        entry->mOffset += copy;
699        if (entry->mOffset == entry->mBuffer->size()) {
700            entry->mNotifyConsumed->post();
701            mAudioQueue.erase(mAudioQueue.begin());
702            entry = NULL;
703        }
704        sizeCopied += copy;
705
706        notifyIfMediaRenderingStarted_l();
707    }
708
709    if (mAudioFirstAnchorTimeMediaUs >= 0) {
710        int64_t nowUs = ALooper::GetNowUs();
711        int64_t nowMediaUs =
712            mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs);
713        // we don't know how much data we are queueing for offloaded tracks.
714        mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
715    }
716
717    if (hasEOS) {
718        (new AMessage(kWhatStopAudioSink, this))->post();
719    }
720
721    return sizeCopied;
722}
723
724bool NuPlayer::Renderer::onDrainAudioQueue() {
725    // TODO: This call to getPosition checks if AudioTrack has been created
726    // in AudioSink before draining audio. If AudioTrack doesn't exist, then
727    // CHECKs on getPosition will fail.
728    // We still need to figure out why AudioTrack is not created when
729    // this function is called. One possible reason could be leftover
730    // audio. Another possible place is to check whether decoder
731    // has received INFO_FORMAT_CHANGED as the first buffer since
732    // AudioSink is opened there, and possible interactions with flush
733    // immediately after start. Investigate error message
734    // "vorbis_dsp_synthesis returned -135", along with RTSP.
735    uint32_t numFramesPlayed;
736    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
737        return false;
738    }
739
740#if 0
741    ssize_t numFramesAvailableToWrite =
742        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
743
744    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
745        ALOGI("audio sink underrun");
746    } else {
747        ALOGV("audio queue has %d frames left to play",
748             mAudioSink->frameCount() - numFramesAvailableToWrite);
749    }
750#endif
751
752    while (!mAudioQueue.empty()) {
753        QueueEntry *entry = &*mAudioQueue.begin();
754
755        mLastAudioBufferDrained = entry->mBufferOrdinal;
756
757        if (entry->mBuffer == NULL) {
758            // EOS
759            int64_t postEOSDelayUs = 0;
760            if (mAudioSink->needsTrailingPadding()) {
761                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
762            }
763            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
764
765            mAudioQueue.erase(mAudioQueue.begin());
766            entry = NULL;
767            if (mAudioSink->needsTrailingPadding()) {
768                // If we're not in gapless playback (i.e. through setNextPlayer), we
769                // need to stop the track here, because that will play out the last
770                // little bit at the end of the file. Otherwise short files won't play.
771                mAudioSink->stop();
772                mNumFramesWritten = 0;
773            }
774            return false;
775        }
776
777        // ignore 0-sized buffer which could be EOS marker with no data
778        if (entry->mOffset == 0 && entry->mBuffer->size() > 0) {
779            int64_t mediaTimeUs;
780            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
781            ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
782            onNewAudioMediaTime(mediaTimeUs);
783        }
784
785        size_t copy = entry->mBuffer->size() - entry->mOffset;
786
787        ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
788                                            copy, false /* blocking */);
789        if (written < 0) {
790            // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
791            if (written == WOULD_BLOCK) {
792                ALOGV("AudioSink write would block when writing %zu bytes", copy);
793            } else {
794                ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
795                notifyAudioTearDown();
796            }
797            break;
798        }
799
800        entry->mOffset += written;
801        if (entry->mOffset == entry->mBuffer->size()) {
802            entry->mNotifyConsumed->post();
803            mAudioQueue.erase(mAudioQueue.begin());
804
805            entry = NULL;
806        }
807
808        size_t copiedFrames = written / mAudioSink->frameSize();
809        mNumFramesWritten += copiedFrames;
810
811        {
812            Mutex::Autolock autoLock(mLock);
813            notifyIfMediaRenderingStarted_l();
814        }
815
816        if (written != (ssize_t)copy) {
817            // A short count was received from AudioSink::write()
818            //
819            // AudioSink write is called in non-blocking mode.
820            // It may return with a short count when:
821            //
822            // 1) Size to be copied is not a multiple of the frame size. We consider this fatal.
823            // 2) The data to be copied exceeds the available buffer in AudioSink.
824            // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
825            // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
826
827            // (Case 1)
828            // Must be a multiple of the frame size.  If it is not a multiple of a frame size, it
829            // needs to fail, as we should not carry over fractional frames between calls.
830            CHECK_EQ(copy % mAudioSink->frameSize(), 0);
831
832            // (Case 2, 3, 4)
833            // Return early to the caller.
834            // Beware of calling immediately again as this may busy-loop if you are not careful.
835            ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
836            break;
837        }
838    }
839    int64_t maxTimeMedia;
840    {
841        Mutex::Autolock autoLock(mLock);
842        maxTimeMedia =
843            mAnchorTimeMediaUs +
844                    (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
845                            * 1000LL * mAudioSink->msecsPerFrame());
846    }
847    mMediaClock->updateMaxTimeMedia(maxTimeMedia);
848
849    return !mAudioQueue.empty();
850}
851
852int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
853    int32_t sampleRate = offloadingAudio() ?
854            mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
855    // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
856    return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate);
857}
858
859// Calculate duration of pending samples if played at normal rate (i.e., 1.0).
860int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
861    int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
862    return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs);
863}
864
865int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
866    int64_t realUs;
867    if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
868        // If failed to get current position, e.g. due to audio clock is
869        // not ready, then just play out video immediately without delay.
870        return nowUs;
871    }
872    return realUs;
873}
874
875void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
876    Mutex::Autolock autoLock(mLock);
877    // TRICKY: vorbis decoder generates multiple frames with the same
878    // timestamp, so only update on the first frame with a given timestamp
879    if (mediaTimeUs == mAnchorTimeMediaUs) {
880        return;
881    }
882    setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
883    int64_t nowUs = ALooper::GetNowUs();
884    int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
885    mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
886    mAnchorNumFramesWritten = mNumFramesWritten;
887    mAnchorTimeMediaUs = mediaTimeUs;
888}
889
890// Called without mLock acquired.
891void NuPlayer::Renderer::postDrainVideoQueue() {
892    if (mDrainVideoQueuePending
893            || getSyncQueues()
894            || (mPaused && mVideoSampleReceived)) {
895        return;
896    }
897
898    if (mVideoQueue.empty()) {
899        return;
900    }
901
902    QueueEntry &entry = *mVideoQueue.begin();
903
904    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
905    msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
906
907    if (entry.mBuffer == NULL) {
908        // EOS doesn't carry a timestamp.
909        msg->post();
910        mDrainVideoQueuePending = true;
911        return;
912    }
913
914    int64_t delayUs;
915    int64_t nowUs = ALooper::GetNowUs();
916    int64_t realTimeUs;
917    if (mFlags & FLAG_REAL_TIME) {
918        int64_t mediaTimeUs;
919        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
920        realTimeUs = mediaTimeUs;
921    } else {
922        int64_t mediaTimeUs;
923        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
924
925        {
926            Mutex::Autolock autoLock(mLock);
927            if (mAnchorTimeMediaUs < 0) {
928                mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
929                mAnchorTimeMediaUs = mediaTimeUs;
930                realTimeUs = nowUs;
931            } else {
932                realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
933            }
934        }
935        if (!mHasAudio) {
936            // smooth out videos >= 10fps
937            mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
938        }
939
940        // Heuristics to handle situation when media time changed without a
941        // discontinuity. If we have not drained an audio buffer that was
942        // received after this buffer, repost in 10 msec. Otherwise repost
943        // in 500 msec.
944        delayUs = realTimeUs - nowUs;
945        if (delayUs > 500000) {
946            int64_t postDelayUs = 500000;
947            if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) {
948                postDelayUs = 10000;
949            }
950            msg->setWhat(kWhatPostDrainVideoQueue);
951            msg->post(postDelayUs);
952            mVideoScheduler->restart();
953            ALOGI("possible video time jump of %dms, retrying in %dms",
954                    (int)(delayUs / 1000), (int)(postDelayUs / 1000));
955            mDrainVideoQueuePending = true;
956            return;
957        }
958    }
959
960    realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
961    int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
962
963    delayUs = realTimeUs - nowUs;
964
965    ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
966    // post 2 display refreshes before rendering is due
967    msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
968
969    mDrainVideoQueuePending = true;
970}
971
972void NuPlayer::Renderer::onDrainVideoQueue() {
973    if (mVideoQueue.empty()) {
974        return;
975    }
976
977    QueueEntry *entry = &*mVideoQueue.begin();
978
979    if (entry->mBuffer == NULL) {
980        // EOS
981
982        notifyEOS(false /* audio */, entry->mFinalResult);
983
984        mVideoQueue.erase(mVideoQueue.begin());
985        entry = NULL;
986
987        setVideoLateByUs(0);
988        return;
989    }
990
991    int64_t nowUs = -1;
992    int64_t realTimeUs;
993    if (mFlags & FLAG_REAL_TIME) {
994        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
995    } else {
996        int64_t mediaTimeUs;
997        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
998
999        nowUs = ALooper::GetNowUs();
1000        realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1001    }
1002
1003    bool tooLate = false;
1004
1005    if (!mPaused) {
1006        if (nowUs == -1) {
1007            nowUs = ALooper::GetNowUs();
1008        }
1009        setVideoLateByUs(nowUs - realTimeUs);
1010        tooLate = (mVideoLateByUs > 40000);
1011
1012        if (tooLate) {
1013            ALOGV("video late by %lld us (%.2f secs)",
1014                 (long long)mVideoLateByUs, mVideoLateByUs / 1E6);
1015        } else {
1016            int64_t mediaUs = 0;
1017            mMediaClock->getMediaTime(realTimeUs, &mediaUs);
1018            ALOGV("rendering video at media time %.2f secs",
1019                    (mFlags & FLAG_REAL_TIME ? realTimeUs :
1020                    mediaUs) / 1E6);
1021        }
1022    } else {
1023        setVideoLateByUs(0);
1024        if (!mVideoSampleReceived && !mHasAudio) {
1025            // This will ensure that the first frame after a flush won't be used as anchor
1026            // when renderer is in paused state, because resume can happen any time after seek.
1027            Mutex::Autolock autoLock(mLock);
1028            clearAnchorTime_l();
1029        }
1030    }
1031
1032    entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
1033    entry->mNotifyConsumed->setInt32("render", !tooLate);
1034    entry->mNotifyConsumed->post();
1035    mVideoQueue.erase(mVideoQueue.begin());
1036    entry = NULL;
1037
1038    mVideoSampleReceived = true;
1039
1040    if (!mPaused) {
1041        if (!mVideoRenderingStarted) {
1042            mVideoRenderingStarted = true;
1043            notifyVideoRenderingStart();
1044        }
1045        Mutex::Autolock autoLock(mLock);
1046        notifyIfMediaRenderingStarted_l();
1047    }
1048}
1049
1050void NuPlayer::Renderer::notifyVideoRenderingStart() {
1051    sp<AMessage> notify = mNotify->dup();
1052    notify->setInt32("what", kWhatVideoRenderingStart);
1053    notify->post();
1054}
1055
1056void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
1057    sp<AMessage> notify = mNotify->dup();
1058    notify->setInt32("what", kWhatEOS);
1059    notify->setInt32("audio", static_cast<int32_t>(audio));
1060    notify->setInt32("finalResult", finalResult);
1061    notify->post(delayUs);
1062}
1063
1064void NuPlayer::Renderer::notifyAudioTearDown() {
1065    (new AMessage(kWhatAudioTearDown, this))->post();
1066}
1067
1068void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
1069    int32_t audio;
1070    CHECK(msg->findInt32("audio", &audio));
1071
1072    if (dropBufferIfStale(audio, msg)) {
1073        return;
1074    }
1075
1076    if (audio) {
1077        mHasAudio = true;
1078    } else {
1079        mHasVideo = true;
1080    }
1081
1082    if (mHasVideo) {
1083        if (mVideoScheduler == NULL) {
1084            mVideoScheduler = new VideoFrameScheduler();
1085            mVideoScheduler->init();
1086        }
1087    }
1088
1089    sp<ABuffer> buffer;
1090    CHECK(msg->findBuffer("buffer", &buffer));
1091
1092    sp<AMessage> notifyConsumed;
1093    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
1094
1095    QueueEntry entry;
1096    entry.mBuffer = buffer;
1097    entry.mNotifyConsumed = notifyConsumed;
1098    entry.mOffset = 0;
1099    entry.mFinalResult = OK;
1100    entry.mBufferOrdinal = ++mTotalBuffersQueued;
1101
1102    if (audio) {
1103        Mutex::Autolock autoLock(mLock);
1104        mAudioQueue.push_back(entry);
1105        postDrainAudioQueue_l();
1106    } else {
1107        mVideoQueue.push_back(entry);
1108        postDrainVideoQueue();
1109    }
1110
1111    Mutex::Autolock autoLock(mLock);
1112    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
1113        return;
1114    }
1115
1116    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
1117    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
1118
1119    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
1120        // EOS signalled on either queue.
1121        syncQueuesDone_l();
1122        return;
1123    }
1124
1125    int64_t firstAudioTimeUs;
1126    int64_t firstVideoTimeUs;
1127    CHECK(firstAudioBuffer->meta()
1128            ->findInt64("timeUs", &firstAudioTimeUs));
1129    CHECK(firstVideoBuffer->meta()
1130            ->findInt64("timeUs", &firstVideoTimeUs));
1131
1132    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
1133
1134    ALOGV("queueDiff = %.2f secs", diff / 1E6);
1135
1136    if (diff > 100000ll) {
1137        // Audio data starts More than 0.1 secs before video.
1138        // Drop some audio.
1139
1140        (*mAudioQueue.begin()).mNotifyConsumed->post();
1141        mAudioQueue.erase(mAudioQueue.begin());
1142        return;
1143    }
1144
1145    syncQueuesDone_l();
1146}
1147
1148void NuPlayer::Renderer::syncQueuesDone_l() {
1149    if (!mSyncQueues) {
1150        return;
1151    }
1152
1153    mSyncQueues = false;
1154
1155    if (!mAudioQueue.empty()) {
1156        postDrainAudioQueue_l();
1157    }
1158
1159    if (!mVideoQueue.empty()) {
1160        mLock.unlock();
1161        postDrainVideoQueue();
1162        mLock.lock();
1163    }
1164}
1165
1166void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1167    int32_t audio;
1168    CHECK(msg->findInt32("audio", &audio));
1169
1170    if (dropBufferIfStale(audio, msg)) {
1171        return;
1172    }
1173
1174    int32_t finalResult;
1175    CHECK(msg->findInt32("finalResult", &finalResult));
1176
1177    QueueEntry entry;
1178    entry.mOffset = 0;
1179    entry.mFinalResult = finalResult;
1180
1181    if (audio) {
1182        Mutex::Autolock autoLock(mLock);
1183        if (mAudioQueue.empty() && mSyncQueues) {
1184            syncQueuesDone_l();
1185        }
1186        mAudioQueue.push_back(entry);
1187        postDrainAudioQueue_l();
1188    } else {
1189        if (mVideoQueue.empty() && getSyncQueues()) {
1190            Mutex::Autolock autoLock(mLock);
1191            syncQueuesDone_l();
1192        }
1193        mVideoQueue.push_back(entry);
1194        postDrainVideoQueue();
1195    }
1196}
1197
1198void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
1199    int32_t audio, notifyComplete;
1200    CHECK(msg->findInt32("audio", &audio));
1201
1202    {
1203        Mutex::Autolock autoLock(mLock);
1204        if (audio) {
1205            notifyComplete = mNotifyCompleteAudio;
1206            mNotifyCompleteAudio = false;
1207        } else {
1208            notifyComplete = mNotifyCompleteVideo;
1209            mNotifyCompleteVideo = false;
1210        }
1211
1212        // If we're currently syncing the queues, i.e. dropping audio while
1213        // aligning the first audio/video buffer times and only one of the
1214        // two queues has data, we may starve that queue by not requesting
1215        // more buffers from the decoder. If the other source then encounters
1216        // a discontinuity that leads to flushing, we'll never find the
1217        // corresponding discontinuity on the other queue.
1218        // Therefore we'll stop syncing the queues if at least one of them
1219        // is flushed.
1220        syncQueuesDone_l();
1221        clearAnchorTime_l();
1222    }
1223
1224    ALOGV("flushing %s", audio ? "audio" : "video");
1225    if (audio) {
1226        {
1227            Mutex::Autolock autoLock(mLock);
1228            flushQueue(&mAudioQueue);
1229
1230            ++mAudioDrainGeneration;
1231            prepareForMediaRenderingStart_l();
1232
1233            if (offloadingAudio()) {
1234                clearAudioFirstAnchorTime_l();
1235            }
1236        }
1237
1238        mDrainAudioQueuePending = false;
1239
1240        if (offloadingAudio()) {
1241            mAudioSink->pause();
1242            mAudioSink->flush();
1243            mAudioSink->start();
1244        } else {
1245            mAudioSink->pause();
1246            mAudioSink->flush();
1247            // Call stop() to signal to the AudioSink to completely fill the
1248            // internal buffer before resuming playback.
1249            mAudioSink->stop();
1250            if (!mPaused) {
1251                mAudioSink->start();
1252            }
1253            mNumFramesWritten = 0;
1254        }
1255    } else {
1256        flushQueue(&mVideoQueue);
1257
1258        mDrainVideoQueuePending = false;
1259
1260        if (mVideoScheduler != NULL) {
1261            mVideoScheduler->restart();
1262        }
1263
1264        Mutex::Autolock autoLock(mLock);
1265        ++mVideoDrainGeneration;
1266        prepareForMediaRenderingStart_l();
1267    }
1268
1269    mVideoSampleReceived = false;
1270
1271    if (notifyComplete) {
1272        notifyFlushComplete(audio);
1273    }
1274}
1275
1276void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1277    while (!queue->empty()) {
1278        QueueEntry *entry = &*queue->begin();
1279
1280        if (entry->mBuffer != NULL) {
1281            entry->mNotifyConsumed->post();
1282        }
1283
1284        queue->erase(queue->begin());
1285        entry = NULL;
1286    }
1287}
1288
1289void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1290    sp<AMessage> notify = mNotify->dup();
1291    notify->setInt32("what", kWhatFlushComplete);
1292    notify->setInt32("audio", static_cast<int32_t>(audio));
1293    notify->post();
1294}
1295
1296bool NuPlayer::Renderer::dropBufferIfStale(
1297        bool audio, const sp<AMessage> &msg) {
1298    int32_t queueGeneration;
1299    CHECK(msg->findInt32("queueGeneration", &queueGeneration));
1300
1301    if (queueGeneration == getQueueGeneration(audio)) {
1302        return false;
1303    }
1304
1305    sp<AMessage> notifyConsumed;
1306    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1307        notifyConsumed->post();
1308    }
1309
1310    return true;
1311}
1312
1313void NuPlayer::Renderer::onAudioSinkChanged() {
1314    if (offloadingAudio()) {
1315        return;
1316    }
1317    CHECK(!mDrainAudioQueuePending);
1318    mNumFramesWritten = 0;
1319    {
1320        Mutex::Autolock autoLock(mLock);
1321        mAnchorNumFramesWritten = -1;
1322    }
1323    uint32_t written;
1324    if (mAudioSink->getFramesWritten(&written) == OK) {
1325        mNumFramesWritten = written;
1326    }
1327}
1328
1329void NuPlayer::Renderer::onDisableOffloadAudio() {
1330    Mutex::Autolock autoLock(mLock);
1331    mFlags &= ~FLAG_OFFLOAD_AUDIO;
1332    ++mAudioDrainGeneration;
1333}
1334
1335void NuPlayer::Renderer::onEnableOffloadAudio() {
1336    Mutex::Autolock autoLock(mLock);
1337    mFlags |= FLAG_OFFLOAD_AUDIO;
1338    ++mAudioDrainGeneration;
1339}
1340
1341void NuPlayer::Renderer::onPause() {
1342    if (mPaused) {
1343        return;
1344    }
1345
1346    {
1347        Mutex::Autolock autoLock(mLock);
1348        ++mAudioDrainGeneration;
1349        ++mVideoDrainGeneration;
1350        prepareForMediaRenderingStart_l();
1351        mPaused = true;
1352        mMediaClock->setPlaybackRate(0.0);
1353    }
1354
1355    mDrainAudioQueuePending = false;
1356    mDrainVideoQueuePending = false;
1357
1358    if (mHasAudio) {
1359        mAudioSink->pause();
1360        startAudioOffloadPauseTimeout();
1361    }
1362
1363    ALOGV("now paused audio queue has %zu entries, video has %zu entries",
1364          mAudioQueue.size(), mVideoQueue.size());
1365}
1366
1367void NuPlayer::Renderer::onResume() {
1368    if (!mPaused) {
1369        return;
1370    }
1371
1372    if (mHasAudio) {
1373        cancelAudioOffloadPauseTimeout();
1374        mAudioSink->start();
1375    }
1376
1377    {
1378        Mutex::Autolock autoLock(mLock);
1379        mPaused = false;
1380
1381        // configure audiosink as we did not do it when pausing
1382        if (mAudioSink != NULL && mAudioSink->ready()) {
1383            mAudioSink->setPlaybackRate(mPlaybackSettings);
1384        }
1385
1386        mMediaClock->setPlaybackRate(mPlaybackRate);
1387
1388        if (!mAudioQueue.empty()) {
1389            postDrainAudioQueue_l();
1390        }
1391    }
1392
1393    if (!mVideoQueue.empty()) {
1394        postDrainVideoQueue();
1395    }
1396}
1397
1398void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1399    if (mVideoScheduler == NULL) {
1400        mVideoScheduler = new VideoFrameScheduler();
1401    }
1402    mVideoScheduler->init(fps);
1403}
1404
1405int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
1406    Mutex::Autolock autoLock(mLock);
1407    return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
1408}
1409
1410int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
1411    Mutex::Autolock autoLock(mLock);
1412    return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
1413}
1414
1415bool NuPlayer::Renderer::getSyncQueues() {
1416    Mutex::Autolock autoLock(mLock);
1417    return mSyncQueues;
1418}
1419
1420// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs()
1421// as it acquires locks and may query the audio driver.
1422//
1423// Some calls could conceivably retrieve extrapolated data instead of
1424// accessing getTimestamp() or getPosition() every time a data buffer with
1425// a media time is received.
1426//
1427// Calculate duration of played samples if played at normal rate (i.e., 1.0).
1428int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) {
1429    uint32_t numFramesPlayed;
1430    int64_t numFramesPlayedAt;
1431    AudioTimestamp ts;
1432    static const int64_t kStaleTimestamp100ms = 100000;
1433
1434    status_t res = mAudioSink->getTimestamp(ts);
1435    if (res == OK) {                 // case 1: mixing audio tracks and offloaded tracks.
1436        numFramesPlayed = ts.mPosition;
1437        numFramesPlayedAt =
1438            ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
1439        const int64_t timestampAge = nowUs - numFramesPlayedAt;
1440        if (timestampAge > kStaleTimestamp100ms) {
1441            // This is an audio FIXME.
1442            // getTimestamp returns a timestamp which may come from audio mixing threads.
1443            // After pausing, the MixerThread may go idle, thus the mTime estimate may
1444            // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms,
1445            // the max latency should be about 25ms with an average around 12ms (to be verified).
1446            // For safety we use 100ms.
1447            ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)",
1448                    (long long)nowUs, (long long)numFramesPlayedAt);
1449            numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
1450        }
1451        //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1452    } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track
1453        numFramesPlayed = 0;
1454        numFramesPlayedAt = nowUs;
1455        //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
1456        //        numFramesPlayed, (long long)numFramesPlayedAt);
1457    } else {                         // case 3: transitory at new track or audio fast tracks.
1458        res = mAudioSink->getPosition(&numFramesPlayed);
1459        CHECK_EQ(res, (status_t)OK);
1460        numFramesPlayedAt = nowUs;
1461        numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */
1462        //ALOGD("getPosition: %u %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1463    }
1464
1465    //CHECK_EQ(numFramesPlayed & (1 << 31), 0);  // can't be negative until 12.4 hrs, test
1466    int64_t durationUs = getDurationUsIfPlayedAtSampleRate(numFramesPlayed)
1467            + nowUs - numFramesPlayedAt;
1468    if (durationUs < 0) {
1469        // Occurs when numFramesPlayed position is very small and the following:
1470        // (1) In case 1, the time nowUs is computed before getTimestamp() is called and
1471        //     numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed.
1472        // (2) In case 3, using getPosition and adding mAudioSink->latency() to
1473        //     numFramesPlayedAt, by a time amount greater than numFramesPlayed.
1474        //
1475        // Both of these are transitory conditions.
1476        ALOGV("getPlayedOutAudioDurationUs: negative duration %lld set to zero", (long long)durationUs);
1477        durationUs = 0;
1478    }
1479    ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)",
1480            (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt);
1481    return durationUs;
1482}
1483
1484void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) {
1485    if (mAudioTornDown) {
1486        return;
1487    }
1488    mAudioTornDown = true;
1489
1490    int64_t currentPositionUs;
1491    if (getCurrentPosition(&currentPositionUs) != OK) {
1492        currentPositionUs = 0;
1493    }
1494
1495    mAudioSink->stop();
1496    mAudioSink->flush();
1497
1498    sp<AMessage> notify = mNotify->dup();
1499    notify->setInt32("what", kWhatAudioTearDown);
1500    notify->setInt64("positionUs", currentPositionUs);
1501    notify->setInt32("reason", reason);
1502    notify->post();
1503}
1504
1505void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1506    if (offloadingAudio()) {
1507        mWakeLock->acquire();
1508        sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
1509        msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
1510        msg->post(kOffloadPauseMaxUs);
1511    }
1512}
1513
1514void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
1515    if (offloadingAudio()) {
1516        mWakeLock->release(true);
1517        ++mAudioOffloadPauseTimeoutGeneration;
1518    }
1519}
1520
1521status_t NuPlayer::Renderer::onOpenAudioSink(
1522        const sp<AMessage> &format,
1523        bool offloadOnly,
1524        bool hasVideo,
1525        uint32_t flags) {
1526    ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1527            offloadOnly, offloadingAudio());
1528    bool audioSinkChanged = false;
1529
1530    int32_t numChannels;
1531    CHECK(format->findInt32("channel-count", &numChannels));
1532
1533    int32_t channelMask;
1534    if (!format->findInt32("channel-mask", &channelMask)) {
1535        // signal to the AudioSink to derive the mask from count.
1536        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1537    }
1538
1539    int32_t sampleRate;
1540    CHECK(format->findInt32("sample-rate", &sampleRate));
1541
1542    if (offloadingAudio()) {
1543        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1544        AString mime;
1545        CHECK(format->findString("mime", &mime));
1546        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1547
1548        if (err != OK) {
1549            ALOGE("Couldn't map mime \"%s\" to a valid "
1550                    "audio_format", mime.c_str());
1551            onDisableOffloadAudio();
1552        } else {
1553            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1554                    mime.c_str(), audioFormat);
1555
1556            int avgBitRate = -1;
1557            format->findInt32("bit-rate", &avgBitRate);
1558
1559            int32_t aacProfile = -1;
1560            if (audioFormat == AUDIO_FORMAT_AAC
1561                    && format->findInt32("aac-profile", &aacProfile)) {
1562                // Redefine AAC format as per aac profile
1563                mapAACProfileToAudioFormat(
1564                        audioFormat,
1565                        aacProfile);
1566            }
1567
1568            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1569            offloadInfo.duration_us = -1;
1570            format->findInt64(
1571                    "durationUs", &offloadInfo.duration_us);
1572            offloadInfo.sample_rate = sampleRate;
1573            offloadInfo.channel_mask = channelMask;
1574            offloadInfo.format = audioFormat;
1575            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1576            offloadInfo.bit_rate = avgBitRate;
1577            offloadInfo.has_video = hasVideo;
1578            offloadInfo.is_streaming = true;
1579
1580            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1581                ALOGV("openAudioSink: no change in offload mode");
1582                // no change from previous configuration, everything ok.
1583                return OK;
1584            }
1585            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1586
1587            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1588            uint32_t offloadFlags = flags;
1589            offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1590            offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1591            audioSinkChanged = true;
1592            mAudioSink->close();
1593            err = mAudioSink->open(
1594                    sampleRate,
1595                    numChannels,
1596                    (audio_channel_mask_t)channelMask,
1597                    audioFormat,
1598                    8 /* bufferCount */,
1599                    &NuPlayer::Renderer::AudioSinkCallback,
1600                    this,
1601                    (audio_output_flags_t)offloadFlags,
1602                    &offloadInfo);
1603
1604            if (err == OK) {
1605                err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1606            }
1607
1608            if (err == OK) {
1609                // If the playback is offloaded to h/w, we pass
1610                // the HAL some metadata information.
1611                // We don't want to do this for PCM because it
1612                // will be going through the AudioFlinger mixer
1613                // before reaching the hardware.
1614                // TODO
1615                mCurrentOffloadInfo = offloadInfo;
1616                err = mAudioSink->start();
1617                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1618            }
1619            if (err != OK) {
1620                // Clean up, fall back to non offload mode.
1621                mAudioSink->close();
1622                onDisableOffloadAudio();
1623                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1624                ALOGV("openAudioSink: offload failed");
1625            }
1626        }
1627    }
1628    if (!offloadOnly && !offloadingAudio()) {
1629        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1630        uint32_t pcmFlags = flags;
1631        pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1632
1633        const PcmInfo info = {
1634                (audio_channel_mask_t)channelMask,
1635                (audio_output_flags_t)pcmFlags,
1636                AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat
1637                numChannels,
1638                sampleRate
1639        };
1640        if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
1641            ALOGV("openAudioSink: no change in pcm mode");
1642            // no change from previous configuration, everything ok.
1643            return OK;
1644        }
1645
1646        audioSinkChanged = true;
1647        mAudioSink->close();
1648        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1649        status_t err = mAudioSink->open(
1650                    sampleRate,
1651                    numChannels,
1652                    (audio_channel_mask_t)channelMask,
1653                    AUDIO_FORMAT_PCM_16_BIT,
1654                    8 /* bufferCount */,
1655                    NULL,
1656                    NULL,
1657                    (audio_output_flags_t)pcmFlags,
1658                    NULL,
1659                    true /* doNotReconnect */);
1660        if (err == OK) {
1661            err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1662        }
1663        if (err != OK) {
1664            ALOGW("openAudioSink: non offloaded open failed status: %d", err);
1665            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1666            return err;
1667        }
1668        mCurrentPcmInfo = info;
1669        mAudioSink->start();
1670    }
1671    if (audioSinkChanged) {
1672        onAudioSinkChanged();
1673    }
1674    mAudioTornDown = false;
1675    return OK;
1676}
1677
1678void NuPlayer::Renderer::onCloseAudioSink() {
1679    mAudioSink->close();
1680    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1681    mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1682}
1683
1684}  // namespace android
1685
1686