NuPlayerRenderer.cpp revision 7c8d0e0a896eb736ca4cec7fb796704a3c14ab66 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22#include <cutils/properties.h>
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26#include <media/stagefright/foundation/AUtils.h>
27#include <media/stagefright/foundation/AWakeLock.h>
28#include <media/stagefright/MediaClock.h>
29#include <media/stagefright/MediaErrors.h>
30#include <media/stagefright/MetaData.h>
31#include <media/stagefright/Utils.h>
32#include <media/stagefright/VideoFrameScheduler.h>
33
34#include <inttypes.h>
35
36namespace android {
37
38/*
39 * Example of common configuration settings in shell script form
40
41   #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager
42   adb shell setprop audio.offload.disable 1
43
44   #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager
45   adb shell setprop audio.offload.video 1
46
47   #Use audio callbacks for PCM data
48   adb shell setprop media.stagefright.audio.cbk 1
49
50   #Use deep buffer for PCM data with video (it is generally enabled for audio-only)
51   adb shell setprop media.stagefright.audio.deep 1
52
53   #Set size of buffers for pcm audio sink in msec (example: 1000 msec)
54   adb shell setprop media.stagefright.audio.sink 1000
55
56 * These configurations take effect for the next track played (not the current track).
57 */
58
59static inline bool getUseAudioCallbackSetting() {
60    return property_get_bool("media.stagefright.audio.cbk", false /* default_value */);
61}
62
63static inline int32_t getAudioSinkPcmMsSetting() {
64    return property_get_int32(
65            "media.stagefright.audio.sink", 500 /* default_value */);
66}
67
68// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
69// is closed to allow the audio DSP to power down.
70static const int64_t kOffloadPauseMaxUs = 10000000ll;
71
72// static
73const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
74        AUDIO_CHANNEL_NONE,
75        AUDIO_OUTPUT_FLAG_NONE,
76        AUDIO_FORMAT_INVALID,
77        0, // mNumChannels
78        0 // mSampleRate
79};
80
81// static
82const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
83
84NuPlayer::Renderer::Renderer(
85        const sp<MediaPlayerBase::AudioSink> &sink,
86        const sp<AMessage> &notify,
87        uint32_t flags)
88    : mAudioSink(sink),
89      mNotify(notify),
90      mFlags(flags),
91      mNumFramesWritten(0),
92      mDrainAudioQueuePending(false),
93      mDrainVideoQueuePending(false),
94      mAudioQueueGeneration(0),
95      mVideoQueueGeneration(0),
96      mAudioDrainGeneration(0),
97      mVideoDrainGeneration(0),
98      mAudioEOSGeneration(0),
99      mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
100      mAudioFirstAnchorTimeMediaUs(-1),
101      mAnchorTimeMediaUs(-1),
102      mAnchorNumFramesWritten(-1),
103      mVideoLateByUs(0ll),
104      mHasAudio(false),
105      mHasVideo(false),
106      mNotifyCompleteAudio(false),
107      mNotifyCompleteVideo(false),
108      mSyncQueues(false),
109      mPaused(false),
110      mVideoSampleReceived(false),
111      mVideoRenderingStarted(false),
112      mVideoRenderingStartGeneration(0),
113      mAudioRenderingStartGeneration(0),
114      mLastAudioMediaTimeUs(-1),
115      mAudioOffloadPauseTimeoutGeneration(0),
116      mAudioTornDown(false),
117      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
118      mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
119      mTotalBuffersQueued(0),
120      mLastAudioBufferDrained(0),
121      mUseAudioCallback(false),
122      mWakeLock(new AWakeLock()) {
123    mMediaClock = new MediaClock;
124    mPlaybackRate = mPlaybackSettings.mSpeed;
125    mMediaClock->setPlaybackRate(mPlaybackRate);
126}
127
128NuPlayer::Renderer::~Renderer() {
129    if (offloadingAudio()) {
130        mAudioSink->stop();
131        mAudioSink->flush();
132        mAudioSink->close();
133    }
134}
135
136void NuPlayer::Renderer::queueBuffer(
137        bool audio,
138        const sp<ABuffer> &buffer,
139        const sp<AMessage> &notifyConsumed) {
140    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
141    msg->setInt32("queueGeneration", getQueueGeneration(audio));
142    msg->setInt32("audio", static_cast<int32_t>(audio));
143    msg->setBuffer("buffer", buffer);
144    msg->setMessage("notifyConsumed", notifyConsumed);
145    msg->post();
146}
147
148void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
149    CHECK_NE(finalResult, (status_t)OK);
150
151    sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
152    msg->setInt32("queueGeneration", getQueueGeneration(audio));
153    msg->setInt32("audio", static_cast<int32_t>(audio));
154    msg->setInt32("finalResult", finalResult);
155    msg->post();
156}
157
158status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) {
159    sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
160    writeToAMessage(msg, rate);
161    sp<AMessage> response;
162    status_t err = msg->postAndAwaitResponse(&response);
163    if (err == OK && response != NULL) {
164        CHECK(response->findInt32("err", &err));
165    }
166    return err;
167}
168
169status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) {
170    if (rate.mSpeed == 0.f) {
171        onPause();
172        // don't call audiosink's setPlaybackRate if pausing, as pitch does not
173        // have to correspond to the any non-0 speed (e.g old speed). Keep
174        // settings nonetheless, using the old speed, in case audiosink changes.
175        AudioPlaybackRate newRate = rate;
176        newRate.mSpeed = mPlaybackSettings.mSpeed;
177        mPlaybackSettings = newRate;
178        return OK;
179    }
180
181    if (mAudioSink != NULL && mAudioSink->ready()) {
182        status_t err = mAudioSink->setPlaybackRate(rate);
183        if (err != OK) {
184            return err;
185        }
186    }
187    mPlaybackSettings = rate;
188    mPlaybackRate = rate.mSpeed;
189    mMediaClock->setPlaybackRate(mPlaybackRate);
190    return OK;
191}
192
193status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
194    sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
195    sp<AMessage> response;
196    status_t err = msg->postAndAwaitResponse(&response);
197    if (err == OK && response != NULL) {
198        CHECK(response->findInt32("err", &err));
199        if (err == OK) {
200            readFromAMessage(response, rate);
201        }
202    }
203    return err;
204}
205
206status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
207    if (mAudioSink != NULL && mAudioSink->ready()) {
208        status_t err = mAudioSink->getPlaybackRate(rate);
209        if (err == OK) {
210            if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) {
211                ALOGW("correcting mismatch in internal/external playback rate");
212            }
213            // get playback settings used by audiosink, as it may be
214            // slightly off due to audiosink not taking small changes.
215            mPlaybackSettings = *rate;
216            if (mPaused) {
217                rate->mSpeed = 0.f;
218            }
219        }
220        return err;
221    }
222    *rate = mPlaybackSettings;
223    return OK;
224}
225
226status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
227    sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
228    writeToAMessage(msg, sync, videoFpsHint);
229    sp<AMessage> response;
230    status_t err = msg->postAndAwaitResponse(&response);
231    if (err == OK && response != NULL) {
232        CHECK(response->findInt32("err", &err));
233    }
234    return err;
235}
236
237status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) {
238    if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
239        return BAD_VALUE;
240    }
241    // TODO: support sync sources
242    return INVALID_OPERATION;
243}
244
245status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) {
246    sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
247    sp<AMessage> response;
248    status_t err = msg->postAndAwaitResponse(&response);
249    if (err == OK && response != NULL) {
250        CHECK(response->findInt32("err", &err));
251        if (err == OK) {
252            readFromAMessage(response, sync, videoFps);
253        }
254    }
255    return err;
256}
257
258status_t NuPlayer::Renderer::onGetSyncSettings(
259        AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
260    *sync = mSyncSettings;
261    *videoFps = -1.f;
262    return OK;
263}
264
265void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
266    {
267        Mutex::Autolock autoLock(mLock);
268        if (audio) {
269            mNotifyCompleteAudio |= notifyComplete;
270            clearAudioFirstAnchorTime_l();
271            ++mAudioQueueGeneration;
272            ++mAudioDrainGeneration;
273        } else {
274            mNotifyCompleteVideo |= notifyComplete;
275            ++mVideoQueueGeneration;
276            ++mVideoDrainGeneration;
277        }
278
279        clearAnchorTime_l();
280        mVideoLateByUs = 0;
281        mSyncQueues = false;
282    }
283
284    sp<AMessage> msg = new AMessage(kWhatFlush, this);
285    msg->setInt32("audio", static_cast<int32_t>(audio));
286    msg->post();
287}
288
289void NuPlayer::Renderer::signalTimeDiscontinuity() {
290}
291
292void NuPlayer::Renderer::signalDisableOffloadAudio() {
293    (new AMessage(kWhatDisableOffloadAudio, this))->post();
294}
295
296void NuPlayer::Renderer::signalEnableOffloadAudio() {
297    (new AMessage(kWhatEnableOffloadAudio, this))->post();
298}
299
300void NuPlayer::Renderer::pause() {
301    (new AMessage(kWhatPause, this))->post();
302}
303
304void NuPlayer::Renderer::resume() {
305    (new AMessage(kWhatResume, this))->post();
306}
307
308void NuPlayer::Renderer::setVideoFrameRate(float fps) {
309    sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
310    msg->setFloat("frame-rate", fps);
311    msg->post();
312}
313
314// Called on any threads.
315status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
316    return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
317}
318
319void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
320    mAudioFirstAnchorTimeMediaUs = -1;
321    mMediaClock->setStartingTimeMedia(-1);
322}
323
324void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
325    if (mAudioFirstAnchorTimeMediaUs == -1) {
326        mAudioFirstAnchorTimeMediaUs = mediaUs;
327        mMediaClock->setStartingTimeMedia(mediaUs);
328    }
329}
330
331void NuPlayer::Renderer::clearAnchorTime_l() {
332    mMediaClock->clearAnchor();
333    mAnchorTimeMediaUs = -1;
334    mAnchorNumFramesWritten = -1;
335}
336
337void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
338    Mutex::Autolock autoLock(mLock);
339    mVideoLateByUs = lateUs;
340}
341
342int64_t NuPlayer::Renderer::getVideoLateByUs() {
343    Mutex::Autolock autoLock(mLock);
344    return mVideoLateByUs;
345}
346
347status_t NuPlayer::Renderer::openAudioSink(
348        const sp<AMessage> &format,
349        bool offloadOnly,
350        bool hasVideo,
351        uint32_t flags,
352        bool *isOffloaded) {
353    sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
354    msg->setMessage("format", format);
355    msg->setInt32("offload-only", offloadOnly);
356    msg->setInt32("has-video", hasVideo);
357    msg->setInt32("flags", flags);
358
359    sp<AMessage> response;
360    msg->postAndAwaitResponse(&response);
361
362    int32_t err;
363    if (!response->findInt32("err", &err)) {
364        err = INVALID_OPERATION;
365    } else if (err == OK && isOffloaded != NULL) {
366        int32_t offload;
367        CHECK(response->findInt32("offload", &offload));
368        *isOffloaded = (offload != 0);
369    }
370    return err;
371}
372
373void NuPlayer::Renderer::closeAudioSink() {
374    sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
375
376    sp<AMessage> response;
377    msg->postAndAwaitResponse(&response);
378}
379
380void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
381    switch (msg->what()) {
382        case kWhatOpenAudioSink:
383        {
384            sp<AMessage> format;
385            CHECK(msg->findMessage("format", &format));
386
387            int32_t offloadOnly;
388            CHECK(msg->findInt32("offload-only", &offloadOnly));
389
390            int32_t hasVideo;
391            CHECK(msg->findInt32("has-video", &hasVideo));
392
393            uint32_t flags;
394            CHECK(msg->findInt32("flags", (int32_t *)&flags));
395
396            status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
397
398            sp<AMessage> response = new AMessage;
399            response->setInt32("err", err);
400            response->setInt32("offload", offloadingAudio());
401
402            sp<AReplyToken> replyID;
403            CHECK(msg->senderAwaitsResponse(&replyID));
404            response->postReply(replyID);
405
406            break;
407        }
408
409        case kWhatCloseAudioSink:
410        {
411            sp<AReplyToken> replyID;
412            CHECK(msg->senderAwaitsResponse(&replyID));
413
414            onCloseAudioSink();
415
416            sp<AMessage> response = new AMessage;
417            response->postReply(replyID);
418            break;
419        }
420
421        case kWhatStopAudioSink:
422        {
423            mAudioSink->stop();
424            break;
425        }
426
427        case kWhatDrainAudioQueue:
428        {
429            mDrainAudioQueuePending = false;
430
431            int32_t generation;
432            CHECK(msg->findInt32("drainGeneration", &generation));
433            if (generation != getDrainGeneration(true /* audio */)) {
434                break;
435            }
436
437            if (onDrainAudioQueue()) {
438                uint32_t numFramesPlayed;
439                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
440                         (status_t)OK);
441
442                uint32_t numFramesPendingPlayout =
443                    mNumFramesWritten - numFramesPlayed;
444
445                // This is how long the audio sink will have data to
446                // play back.
447                int64_t delayUs =
448                    mAudioSink->msecsPerFrame()
449                        * numFramesPendingPlayout * 1000ll;
450                if (mPlaybackRate > 1.0f) {
451                    delayUs /= mPlaybackRate;
452                }
453
454                // Let's give it more data after about half that time
455                // has elapsed.
456                Mutex::Autolock autoLock(mLock);
457                postDrainAudioQueue_l(delayUs / 2);
458            }
459            break;
460        }
461
462        case kWhatDrainVideoQueue:
463        {
464            int32_t generation;
465            CHECK(msg->findInt32("drainGeneration", &generation));
466            if (generation != getDrainGeneration(false /* audio */)) {
467                break;
468            }
469
470            mDrainVideoQueuePending = false;
471
472            onDrainVideoQueue();
473
474            postDrainVideoQueue();
475            break;
476        }
477
478        case kWhatPostDrainVideoQueue:
479        {
480            int32_t generation;
481            CHECK(msg->findInt32("drainGeneration", &generation));
482            if (generation != getDrainGeneration(false /* audio */)) {
483                break;
484            }
485
486            mDrainVideoQueuePending = false;
487            postDrainVideoQueue();
488            break;
489        }
490
491        case kWhatQueueBuffer:
492        {
493            onQueueBuffer(msg);
494            break;
495        }
496
497        case kWhatQueueEOS:
498        {
499            onQueueEOS(msg);
500            break;
501        }
502
503        case kWhatEOS:
504        {
505            int32_t generation;
506            CHECK(msg->findInt32("audioEOSGeneration", &generation));
507            if (generation != mAudioEOSGeneration) {
508                break;
509            }
510            status_t finalResult;
511            CHECK(msg->findInt32("finalResult", &finalResult));
512            notifyEOS(true /* audio */, finalResult);
513            break;
514        }
515
516        case kWhatConfigPlayback:
517        {
518            sp<AReplyToken> replyID;
519            CHECK(msg->senderAwaitsResponse(&replyID));
520            AudioPlaybackRate rate;
521            readFromAMessage(msg, &rate);
522            status_t err = onConfigPlayback(rate);
523            sp<AMessage> response = new AMessage;
524            response->setInt32("err", err);
525            response->postReply(replyID);
526            break;
527        }
528
529        case kWhatGetPlaybackSettings:
530        {
531            sp<AReplyToken> replyID;
532            CHECK(msg->senderAwaitsResponse(&replyID));
533            AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT;
534            status_t err = onGetPlaybackSettings(&rate);
535            sp<AMessage> response = new AMessage;
536            if (err == OK) {
537                writeToAMessage(response, rate);
538            }
539            response->setInt32("err", err);
540            response->postReply(replyID);
541            break;
542        }
543
544        case kWhatConfigSync:
545        {
546            sp<AReplyToken> replyID;
547            CHECK(msg->senderAwaitsResponse(&replyID));
548            AVSyncSettings sync;
549            float videoFpsHint;
550            readFromAMessage(msg, &sync, &videoFpsHint);
551            status_t err = onConfigSync(sync, videoFpsHint);
552            sp<AMessage> response = new AMessage;
553            response->setInt32("err", err);
554            response->postReply(replyID);
555            break;
556        }
557
558        case kWhatGetSyncSettings:
559        {
560            sp<AReplyToken> replyID;
561            CHECK(msg->senderAwaitsResponse(&replyID));
562
563            ALOGV("kWhatGetSyncSettings");
564            AVSyncSettings sync;
565            float videoFps = -1.f;
566            status_t err = onGetSyncSettings(&sync, &videoFps);
567            sp<AMessage> response = new AMessage;
568            if (err == OK) {
569                writeToAMessage(response, sync, videoFps);
570            }
571            response->setInt32("err", err);
572            response->postReply(replyID);
573            break;
574        }
575
576        case kWhatFlush:
577        {
578            onFlush(msg);
579            break;
580        }
581
582        case kWhatDisableOffloadAudio:
583        {
584            onDisableOffloadAudio();
585            break;
586        }
587
588        case kWhatEnableOffloadAudio:
589        {
590            onEnableOffloadAudio();
591            break;
592        }
593
594        case kWhatPause:
595        {
596            onPause();
597            break;
598        }
599
600        case kWhatResume:
601        {
602            onResume();
603            break;
604        }
605
606        case kWhatSetVideoFrameRate:
607        {
608            float fps;
609            CHECK(msg->findFloat("frame-rate", &fps));
610            onSetVideoFrameRate(fps);
611            break;
612        }
613
614        case kWhatAudioTearDown:
615        {
616            onAudioTearDown(kDueToError);
617            break;
618        }
619
620        case kWhatAudioOffloadPauseTimeout:
621        {
622            int32_t generation;
623            CHECK(msg->findInt32("drainGeneration", &generation));
624            if (generation != mAudioOffloadPauseTimeoutGeneration) {
625                break;
626            }
627            ALOGV("Audio Offload tear down due to pause timeout.");
628            onAudioTearDown(kDueToTimeout);
629            mWakeLock->release();
630            break;
631        }
632
633        default:
634            TRESPASS();
635            break;
636    }
637}
638
639void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
640    if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) {
641        return;
642    }
643
644    if (mAudioQueue.empty()) {
645        return;
646    }
647
648    mDrainAudioQueuePending = true;
649    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
650    msg->setInt32("drainGeneration", mAudioDrainGeneration);
651    msg->post(delayUs);
652}
653
654void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
655    mAudioRenderingStartGeneration = mAudioDrainGeneration;
656    mVideoRenderingStartGeneration = mVideoDrainGeneration;
657}
658
659void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
660    if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
661        mAudioRenderingStartGeneration == mAudioDrainGeneration) {
662        mVideoRenderingStartGeneration = -1;
663        mAudioRenderingStartGeneration = -1;
664
665        sp<AMessage> notify = mNotify->dup();
666        notify->setInt32("what", kWhatMediaRenderingStart);
667        notify->post();
668    }
669}
670
671// static
672size_t NuPlayer::Renderer::AudioSinkCallback(
673        MediaPlayerBase::AudioSink * /* audioSink */,
674        void *buffer,
675        size_t size,
676        void *cookie,
677        MediaPlayerBase::AudioSink::cb_event_t event) {
678    NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
679
680    switch (event) {
681        case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
682        {
683            return me->fillAudioBuffer(buffer, size);
684            break;
685        }
686
687        case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
688        {
689            ALOGV("AudioSink::CB_EVENT_STREAM_END");
690            me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
691            break;
692        }
693
694        case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
695        {
696            ALOGV("AudioSink::CB_EVENT_TEAR_DOWN");
697            me->notifyAudioTearDown();
698            break;
699        }
700    }
701
702    return 0;
703}
704
705size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
706    Mutex::Autolock autoLock(mLock);
707
708    if (!mUseAudioCallback) {
709        return 0;
710    }
711
712    bool hasEOS = false;
713
714    size_t sizeCopied = 0;
715    bool firstEntry = true;
716    QueueEntry *entry;  // will be valid after while loop if hasEOS is set.
717    while (sizeCopied < size && !mAudioQueue.empty()) {
718        entry = &*mAudioQueue.begin();
719
720        if (entry->mBuffer == NULL) { // EOS
721            hasEOS = true;
722            mAudioQueue.erase(mAudioQueue.begin());
723            break;
724        }
725
726        if (firstEntry && entry->mOffset == 0) {
727            firstEntry = false;
728            int64_t mediaTimeUs;
729            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
730            ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
731            setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
732        }
733
734        size_t copy = entry->mBuffer->size() - entry->mOffset;
735        size_t sizeRemaining = size - sizeCopied;
736        if (copy > sizeRemaining) {
737            copy = sizeRemaining;
738        }
739
740        memcpy((char *)buffer + sizeCopied,
741               entry->mBuffer->data() + entry->mOffset,
742               copy);
743
744        entry->mOffset += copy;
745        if (entry->mOffset == entry->mBuffer->size()) {
746            entry->mNotifyConsumed->post();
747            mAudioQueue.erase(mAudioQueue.begin());
748            entry = NULL;
749        }
750        sizeCopied += copy;
751
752        notifyIfMediaRenderingStarted_l();
753    }
754
755    if (mAudioFirstAnchorTimeMediaUs >= 0) {
756        int64_t nowUs = ALooper::GetNowUs();
757        int64_t nowMediaUs =
758            mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs);
759        // we don't know how much data we are queueing for offloaded tracks.
760        mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
761    }
762
763    // for non-offloaded audio, we need to compute the frames written because
764    // there is no EVENT_STREAM_END notification. The frames written gives
765    // an estimate on the pending played out duration.
766    if (!offloadingAudio()) {
767        mNumFramesWritten += sizeCopied / mAudioSink->frameSize();
768    }
769
770    if (hasEOS) {
771        (new AMessage(kWhatStopAudioSink, this))->post();
772        // As there is currently no EVENT_STREAM_END callback notification for
773        // non-offloaded audio tracks, we need to post the EOS ourselves.
774        if (!offloadingAudio()) {
775            int64_t postEOSDelayUs = 0;
776            if (mAudioSink->needsTrailingPadding()) {
777                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
778            }
779            ALOGV("fillAudioBuffer: notifyEOS "
780                    "mNumFramesWritten:%u  finalResult:%d  postEOSDelay:%lld",
781                    mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs);
782            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
783        }
784    }
785    return sizeCopied;
786}
787
788void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() {
789    List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it;
790    bool foundEOS = false;
791    while (it != mAudioQueue.end()) {
792        int32_t eos;
793        QueueEntry *entry = &*it++;
794        if (entry->mBuffer == NULL
795                || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) {
796            itEOS = it;
797            foundEOS = true;
798        }
799    }
800
801    if (foundEOS) {
802        // post all replies before EOS and drop the samples
803        for (it = mAudioQueue.begin(); it != itEOS; it++) {
804            if (it->mBuffer == NULL) {
805                // delay doesn't matter as we don't even have an AudioTrack
806                notifyEOS(true /* audio */, it->mFinalResult);
807            } else {
808                it->mNotifyConsumed->post();
809            }
810        }
811        mAudioQueue.erase(mAudioQueue.begin(), itEOS);
812    }
813}
814
815bool NuPlayer::Renderer::onDrainAudioQueue() {
816    // TODO: This call to getPosition checks if AudioTrack has been created
817    // in AudioSink before draining audio. If AudioTrack doesn't exist, then
818    // CHECKs on getPosition will fail.
819    // We still need to figure out why AudioTrack is not created when
820    // this function is called. One possible reason could be leftover
821    // audio. Another possible place is to check whether decoder
822    // has received INFO_FORMAT_CHANGED as the first buffer since
823    // AudioSink is opened there, and possible interactions with flush
824    // immediately after start. Investigate error message
825    // "vorbis_dsp_synthesis returned -135", along with RTSP.
826    uint32_t numFramesPlayed;
827    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
828        // When getPosition fails, renderer will not reschedule the draining
829        // unless new samples are queued.
830        // If we have pending EOS (or "eos" marker for discontinuities), we need
831        // to post these now as NuPlayerDecoder might be waiting for it.
832        drainAudioQueueUntilLastEOS();
833
834        ALOGW("onDrainAudioQueue(): audio sink is not ready");
835        return false;
836    }
837
838#if 0
839    ssize_t numFramesAvailableToWrite =
840        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
841
842    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
843        ALOGI("audio sink underrun");
844    } else {
845        ALOGV("audio queue has %d frames left to play",
846             mAudioSink->frameCount() - numFramesAvailableToWrite);
847    }
848#endif
849
850    uint32_t prevFramesWritten = mNumFramesWritten;
851    while (!mAudioQueue.empty()) {
852        QueueEntry *entry = &*mAudioQueue.begin();
853
854        mLastAudioBufferDrained = entry->mBufferOrdinal;
855
856        if (entry->mBuffer == NULL) {
857            // EOS
858            int64_t postEOSDelayUs = 0;
859            if (mAudioSink->needsTrailingPadding()) {
860                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
861            }
862            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
863            mLastAudioMediaTimeUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
864
865            mAudioQueue.erase(mAudioQueue.begin());
866            entry = NULL;
867            if (mAudioSink->needsTrailingPadding()) {
868                // If we're not in gapless playback (i.e. through setNextPlayer), we
869                // need to stop the track here, because that will play out the last
870                // little bit at the end of the file. Otherwise short files won't play.
871                mAudioSink->stop();
872                mNumFramesWritten = 0;
873            }
874            return false;
875        }
876
877        // ignore 0-sized buffer which could be EOS marker with no data
878        if (entry->mOffset == 0 && entry->mBuffer->size() > 0) {
879            int64_t mediaTimeUs;
880            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
881            ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs",
882                    mediaTimeUs / 1E6);
883            onNewAudioMediaTime(mediaTimeUs);
884        }
885
886        size_t copy = entry->mBuffer->size() - entry->mOffset;
887
888        ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
889                                            copy, false /* blocking */);
890        if (written < 0) {
891            // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
892            if (written == WOULD_BLOCK) {
893                ALOGV("AudioSink write would block when writing %zu bytes", copy);
894            } else {
895                ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
896                notifyAudioTearDown();
897            }
898            break;
899        }
900
901        entry->mOffset += written;
902        if (entry->mOffset == entry->mBuffer->size()) {
903            entry->mNotifyConsumed->post();
904            mAudioQueue.erase(mAudioQueue.begin());
905
906            entry = NULL;
907        }
908
909        size_t copiedFrames = written / mAudioSink->frameSize();
910        mNumFramesWritten += copiedFrames;
911
912        {
913            Mutex::Autolock autoLock(mLock);
914            notifyIfMediaRenderingStarted_l();
915        }
916
917        if (written != (ssize_t)copy) {
918            // A short count was received from AudioSink::write()
919            //
920            // AudioSink write is called in non-blocking mode.
921            // It may return with a short count when:
922            //
923            // 1) Size to be copied is not a multiple of the frame size. We consider this fatal.
924            // 2) The data to be copied exceeds the available buffer in AudioSink.
925            // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
926            // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
927
928            // (Case 1)
929            // Must be a multiple of the frame size.  If it is not a multiple of a frame size, it
930            // needs to fail, as we should not carry over fractional frames between calls.
931            CHECK_EQ(copy % mAudioSink->frameSize(), 0);
932
933            // (Case 2, 3, 4)
934            // Return early to the caller.
935            // Beware of calling immediately again as this may busy-loop if you are not careful.
936            ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
937            break;
938        }
939    }
940    int64_t maxTimeMedia;
941    {
942        Mutex::Autolock autoLock(mLock);
943        maxTimeMedia =
944            mAnchorTimeMediaUs +
945                    (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
946                            * 1000LL * mAudioSink->msecsPerFrame());
947    }
948    mMediaClock->updateMaxTimeMedia(maxTimeMedia);
949
950    // calculate whether we need to reschedule another write.
951    bool reschedule = !mAudioQueue.empty()
952            && (!mPaused
953                || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers
954    //ALOGD("reschedule:%d  empty:%d  mPaused:%d  prevFramesWritten:%u  mNumFramesWritten:%u",
955    //        reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten);
956    return reschedule;
957}
958
959int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
960    int32_t sampleRate = offloadingAudio() ?
961            mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
962    // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
963    return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate);
964}
965
966// Calculate duration of pending samples if played at normal rate (i.e., 1.0).
967int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
968    int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
969    return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs);
970}
971
972int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
973    int64_t realUs;
974    if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
975        // If failed to get current position, e.g. due to audio clock is
976        // not ready, then just play out video immediately without delay.
977        return nowUs;
978    }
979    return realUs;
980}
981
982void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
983    Mutex::Autolock autoLock(mLock);
984    // TRICKY: vorbis decoder generates multiple frames with the same
985    // timestamp, so only update on the first frame with a given timestamp
986    if (mediaTimeUs == mAnchorTimeMediaUs) {
987        return;
988    }
989    setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
990    int64_t nowUs = ALooper::GetNowUs();
991    int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
992    mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
993    mAnchorNumFramesWritten = mNumFramesWritten;
994    mAnchorTimeMediaUs = mediaTimeUs;
995}
996
997// Called without mLock acquired.
998void NuPlayer::Renderer::postDrainVideoQueue() {
999    if (mDrainVideoQueuePending
1000            || getSyncQueues()
1001            || (mPaused && mVideoSampleReceived)) {
1002        return;
1003    }
1004
1005    if (mVideoQueue.empty()) {
1006        return;
1007    }
1008
1009    QueueEntry &entry = *mVideoQueue.begin();
1010
1011    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
1012    msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
1013
1014    if (entry.mBuffer == NULL) {
1015        // EOS doesn't carry a timestamp.
1016        msg->post();
1017        mDrainVideoQueuePending = true;
1018        return;
1019    }
1020
1021    int64_t delayUs;
1022    int64_t nowUs = ALooper::GetNowUs();
1023    int64_t realTimeUs;
1024    if (mFlags & FLAG_REAL_TIME) {
1025        int64_t mediaTimeUs;
1026        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1027        realTimeUs = mediaTimeUs;
1028    } else {
1029        int64_t mediaTimeUs;
1030        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1031
1032        {
1033            Mutex::Autolock autoLock(mLock);
1034            if (mAnchorTimeMediaUs < 0) {
1035                mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
1036                mAnchorTimeMediaUs = mediaTimeUs;
1037                realTimeUs = nowUs;
1038            } else {
1039                realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1040            }
1041        }
1042        if (!mHasAudio) {
1043            // smooth out videos >= 10fps
1044            mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
1045        }
1046
1047        // Heuristics to handle situation when media time changed without a
1048        // discontinuity. If we have not drained an audio buffer that was
1049        // received after this buffer, repost in 10 msec. Otherwise repost
1050        // in 500 msec.
1051        delayUs = realTimeUs - nowUs;
1052        if (delayUs > 500000) {
1053            int64_t postDelayUs = 500000;
1054            if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) {
1055                postDelayUs = 10000;
1056            }
1057            msg->setWhat(kWhatPostDrainVideoQueue);
1058            msg->post(postDelayUs);
1059            mVideoScheduler->restart();
1060            ALOGI("possible video time jump of %dms, retrying in %dms",
1061                    (int)(delayUs / 1000), (int)(postDelayUs / 1000));
1062            mDrainVideoQueuePending = true;
1063            return;
1064        }
1065    }
1066
1067    realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
1068    int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
1069
1070    delayUs = realTimeUs - nowUs;
1071
1072    ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
1073    // post 2 display refreshes before rendering is due
1074    msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
1075
1076    mDrainVideoQueuePending = true;
1077}
1078
1079void NuPlayer::Renderer::onDrainVideoQueue() {
1080    if (mVideoQueue.empty()) {
1081        return;
1082    }
1083
1084    QueueEntry *entry = &*mVideoQueue.begin();
1085
1086    if (entry->mBuffer == NULL) {
1087        // EOS
1088
1089        notifyEOS(false /* audio */, entry->mFinalResult);
1090
1091        mVideoQueue.erase(mVideoQueue.begin());
1092        entry = NULL;
1093
1094        setVideoLateByUs(0);
1095        return;
1096    }
1097
1098    int64_t nowUs = -1;
1099    int64_t realTimeUs;
1100    int64_t mediaTimeUs = -1;
1101    if (mFlags & FLAG_REAL_TIME) {
1102        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
1103    } else {
1104        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1105
1106        nowUs = ALooper::GetNowUs();
1107        realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1108    }
1109
1110    bool tooLate = false;
1111
1112    if (!mPaused) {
1113        if (nowUs == -1) {
1114            nowUs = ALooper::GetNowUs();
1115        }
1116        setVideoLateByUs(nowUs - realTimeUs);
1117        tooLate = (mVideoLateByUs > 40000);
1118
1119        if (tooLate) {
1120            ALOGV("video late by %lld us (%.2f secs)",
1121                 (long long)mVideoLateByUs, mVideoLateByUs / 1E6);
1122        } else {
1123            int64_t mediaUs = 0;
1124            mMediaClock->getMediaTime(realTimeUs, &mediaUs);
1125            ALOGV("rendering video at media time %.2f secs",
1126                    (mFlags & FLAG_REAL_TIME ? realTimeUs :
1127                    mediaUs) / 1E6);
1128
1129            if (!(mFlags & FLAG_REAL_TIME)
1130                    && mLastAudioMediaTimeUs != -1
1131                    && mediaTimeUs > mLastAudioMediaTimeUs) {
1132                // If audio ends before video, video continues to drive media clock.
1133                // Also smooth out videos >= 10fps.
1134                mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
1135            }
1136        }
1137    } else {
1138        setVideoLateByUs(0);
1139        if (!mVideoSampleReceived && !mHasAudio) {
1140            // This will ensure that the first frame after a flush won't be used as anchor
1141            // when renderer is in paused state, because resume can happen any time after seek.
1142            Mutex::Autolock autoLock(mLock);
1143            clearAnchorTime_l();
1144        }
1145    }
1146
1147    entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
1148    entry->mNotifyConsumed->setInt32("render", !tooLate);
1149    entry->mNotifyConsumed->post();
1150    mVideoQueue.erase(mVideoQueue.begin());
1151    entry = NULL;
1152
1153    mVideoSampleReceived = true;
1154
1155    if (!mPaused) {
1156        if (!mVideoRenderingStarted) {
1157            mVideoRenderingStarted = true;
1158            notifyVideoRenderingStart();
1159        }
1160        Mutex::Autolock autoLock(mLock);
1161        notifyIfMediaRenderingStarted_l();
1162    }
1163}
1164
1165void NuPlayer::Renderer::notifyVideoRenderingStart() {
1166    sp<AMessage> notify = mNotify->dup();
1167    notify->setInt32("what", kWhatVideoRenderingStart);
1168    notify->post();
1169}
1170
1171void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
1172    if (audio && delayUs > 0) {
1173        sp<AMessage> msg = new AMessage(kWhatEOS, this);
1174        msg->setInt32("audioEOSGeneration", mAudioEOSGeneration);
1175        msg->setInt32("finalResult", finalResult);
1176        msg->post(delayUs);
1177        return;
1178    }
1179    sp<AMessage> notify = mNotify->dup();
1180    notify->setInt32("what", kWhatEOS);
1181    notify->setInt32("audio", static_cast<int32_t>(audio));
1182    notify->setInt32("finalResult", finalResult);
1183    notify->post(delayUs);
1184}
1185
1186void NuPlayer::Renderer::notifyAudioTearDown() {
1187    (new AMessage(kWhatAudioTearDown, this))->post();
1188}
1189
1190void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
1191    int32_t audio;
1192    CHECK(msg->findInt32("audio", &audio));
1193
1194    if (dropBufferIfStale(audio, msg)) {
1195        return;
1196    }
1197
1198    if (audio) {
1199        mHasAudio = true;
1200    } else {
1201        mHasVideo = true;
1202    }
1203
1204    if (mHasVideo) {
1205        if (mVideoScheduler == NULL) {
1206            mVideoScheduler = new VideoFrameScheduler();
1207            mVideoScheduler->init();
1208        }
1209    }
1210
1211    sp<ABuffer> buffer;
1212    CHECK(msg->findBuffer("buffer", &buffer));
1213
1214    sp<AMessage> notifyConsumed;
1215    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
1216
1217    QueueEntry entry;
1218    entry.mBuffer = buffer;
1219    entry.mNotifyConsumed = notifyConsumed;
1220    entry.mOffset = 0;
1221    entry.mFinalResult = OK;
1222    entry.mBufferOrdinal = ++mTotalBuffersQueued;
1223
1224    if (audio) {
1225        Mutex::Autolock autoLock(mLock);
1226        mAudioQueue.push_back(entry);
1227        postDrainAudioQueue_l();
1228    } else {
1229        mVideoQueue.push_back(entry);
1230        postDrainVideoQueue();
1231    }
1232
1233    Mutex::Autolock autoLock(mLock);
1234    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
1235        return;
1236    }
1237
1238    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
1239    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
1240
1241    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
1242        // EOS signalled on either queue.
1243        syncQueuesDone_l();
1244        return;
1245    }
1246
1247    int64_t firstAudioTimeUs;
1248    int64_t firstVideoTimeUs;
1249    CHECK(firstAudioBuffer->meta()
1250            ->findInt64("timeUs", &firstAudioTimeUs));
1251    CHECK(firstVideoBuffer->meta()
1252            ->findInt64("timeUs", &firstVideoTimeUs));
1253
1254    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
1255
1256    ALOGV("queueDiff = %.2f secs", diff / 1E6);
1257
1258    if (diff > 100000ll) {
1259        // Audio data starts More than 0.1 secs before video.
1260        // Drop some audio.
1261
1262        (*mAudioQueue.begin()).mNotifyConsumed->post();
1263        mAudioQueue.erase(mAudioQueue.begin());
1264        return;
1265    }
1266
1267    syncQueuesDone_l();
1268}
1269
1270void NuPlayer::Renderer::syncQueuesDone_l() {
1271    if (!mSyncQueues) {
1272        return;
1273    }
1274
1275    mSyncQueues = false;
1276
1277    if (!mAudioQueue.empty()) {
1278        postDrainAudioQueue_l();
1279    }
1280
1281    if (!mVideoQueue.empty()) {
1282        mLock.unlock();
1283        postDrainVideoQueue();
1284        mLock.lock();
1285    }
1286}
1287
1288void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1289    int32_t audio;
1290    CHECK(msg->findInt32("audio", &audio));
1291
1292    if (dropBufferIfStale(audio, msg)) {
1293        return;
1294    }
1295
1296    int32_t finalResult;
1297    CHECK(msg->findInt32("finalResult", &finalResult));
1298
1299    QueueEntry entry;
1300    entry.mOffset = 0;
1301    entry.mFinalResult = finalResult;
1302
1303    if (audio) {
1304        Mutex::Autolock autoLock(mLock);
1305        if (mAudioQueue.empty() && mSyncQueues) {
1306            syncQueuesDone_l();
1307        }
1308        mAudioQueue.push_back(entry);
1309        postDrainAudioQueue_l();
1310    } else {
1311        if (mVideoQueue.empty() && getSyncQueues()) {
1312            Mutex::Autolock autoLock(mLock);
1313            syncQueuesDone_l();
1314        }
1315        mVideoQueue.push_back(entry);
1316        postDrainVideoQueue();
1317    }
1318}
1319
1320void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
1321    int32_t audio, notifyComplete;
1322    CHECK(msg->findInt32("audio", &audio));
1323
1324    {
1325        Mutex::Autolock autoLock(mLock);
1326        if (audio) {
1327            notifyComplete = mNotifyCompleteAudio;
1328            mNotifyCompleteAudio = false;
1329            mLastAudioMediaTimeUs = -1;
1330        } else {
1331            notifyComplete = mNotifyCompleteVideo;
1332            mNotifyCompleteVideo = false;
1333        }
1334
1335        // If we're currently syncing the queues, i.e. dropping audio while
1336        // aligning the first audio/video buffer times and only one of the
1337        // two queues has data, we may starve that queue by not requesting
1338        // more buffers from the decoder. If the other source then encounters
1339        // a discontinuity that leads to flushing, we'll never find the
1340        // corresponding discontinuity on the other queue.
1341        // Therefore we'll stop syncing the queues if at least one of them
1342        // is flushed.
1343        syncQueuesDone_l();
1344        clearAnchorTime_l();
1345    }
1346
1347    ALOGV("flushing %s", audio ? "audio" : "video");
1348    if (audio) {
1349        {
1350            Mutex::Autolock autoLock(mLock);
1351            flushQueue(&mAudioQueue);
1352
1353            ++mAudioDrainGeneration;
1354            ++mAudioEOSGeneration;
1355            prepareForMediaRenderingStart_l();
1356
1357            // the frame count will be reset after flush.
1358            clearAudioFirstAnchorTime_l();
1359        }
1360
1361        mDrainAudioQueuePending = false;
1362
1363        if (offloadingAudio()) {
1364            mAudioSink->pause();
1365            mAudioSink->flush();
1366            if (!mPaused) {
1367                mAudioSink->start();
1368            }
1369        } else {
1370            mAudioSink->pause();
1371            mAudioSink->flush();
1372            // Call stop() to signal to the AudioSink to completely fill the
1373            // internal buffer before resuming playback.
1374            mAudioSink->stop();
1375            if (!mPaused) {
1376                mAudioSink->start();
1377            }
1378            mNumFramesWritten = 0;
1379        }
1380    } else {
1381        flushQueue(&mVideoQueue);
1382
1383        mDrainVideoQueuePending = false;
1384
1385        if (mVideoScheduler != NULL) {
1386            mVideoScheduler->restart();
1387        }
1388
1389        Mutex::Autolock autoLock(mLock);
1390        ++mVideoDrainGeneration;
1391        prepareForMediaRenderingStart_l();
1392    }
1393
1394    mVideoSampleReceived = false;
1395
1396    if (notifyComplete) {
1397        notifyFlushComplete(audio);
1398    }
1399}
1400
1401void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1402    while (!queue->empty()) {
1403        QueueEntry *entry = &*queue->begin();
1404
1405        if (entry->mBuffer != NULL) {
1406            entry->mNotifyConsumed->post();
1407        }
1408
1409        queue->erase(queue->begin());
1410        entry = NULL;
1411    }
1412}
1413
1414void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1415    sp<AMessage> notify = mNotify->dup();
1416    notify->setInt32("what", kWhatFlushComplete);
1417    notify->setInt32("audio", static_cast<int32_t>(audio));
1418    notify->post();
1419}
1420
1421bool NuPlayer::Renderer::dropBufferIfStale(
1422        bool audio, const sp<AMessage> &msg) {
1423    int32_t queueGeneration;
1424    CHECK(msg->findInt32("queueGeneration", &queueGeneration));
1425
1426    if (queueGeneration == getQueueGeneration(audio)) {
1427        return false;
1428    }
1429
1430    sp<AMessage> notifyConsumed;
1431    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1432        notifyConsumed->post();
1433    }
1434
1435    return true;
1436}
1437
1438void NuPlayer::Renderer::onAudioSinkChanged() {
1439    if (offloadingAudio()) {
1440        return;
1441    }
1442    CHECK(!mDrainAudioQueuePending);
1443    mNumFramesWritten = 0;
1444    {
1445        Mutex::Autolock autoLock(mLock);
1446        mAnchorNumFramesWritten = -1;
1447    }
1448    uint32_t written;
1449    if (mAudioSink->getFramesWritten(&written) == OK) {
1450        mNumFramesWritten = written;
1451    }
1452}
1453
1454void NuPlayer::Renderer::onDisableOffloadAudio() {
1455    Mutex::Autolock autoLock(mLock);
1456    mFlags &= ~FLAG_OFFLOAD_AUDIO;
1457    ++mAudioDrainGeneration;
1458    if (mAudioRenderingStartGeneration != -1) {
1459        prepareForMediaRenderingStart_l();
1460    }
1461}
1462
1463void NuPlayer::Renderer::onEnableOffloadAudio() {
1464    Mutex::Autolock autoLock(mLock);
1465    mFlags |= FLAG_OFFLOAD_AUDIO;
1466    ++mAudioDrainGeneration;
1467    if (mAudioRenderingStartGeneration != -1) {
1468        prepareForMediaRenderingStart_l();
1469    }
1470}
1471
1472void NuPlayer::Renderer::onPause() {
1473    if (mPaused) {
1474        return;
1475    }
1476
1477    {
1478        Mutex::Autolock autoLock(mLock);
1479        // we do not increment audio drain generation so that we fill audio buffer during pause.
1480        ++mVideoDrainGeneration;
1481        prepareForMediaRenderingStart_l();
1482        mPaused = true;
1483        mMediaClock->setPlaybackRate(0.0);
1484    }
1485
1486    mDrainAudioQueuePending = false;
1487    mDrainVideoQueuePending = false;
1488
1489    if (mHasAudio) {
1490        mAudioSink->pause();
1491        startAudioOffloadPauseTimeout();
1492    }
1493
1494    ALOGV("now paused audio queue has %zu entries, video has %zu entries",
1495          mAudioQueue.size(), mVideoQueue.size());
1496}
1497
1498void NuPlayer::Renderer::onResume() {
1499    if (!mPaused) {
1500        return;
1501    }
1502
1503    if (mHasAudio) {
1504        cancelAudioOffloadPauseTimeout();
1505        status_t err = mAudioSink->start();
1506        if (err != OK) {
1507            notifyAudioTearDown();
1508        }
1509    }
1510
1511    {
1512        Mutex::Autolock autoLock(mLock);
1513        mPaused = false;
1514
1515        // configure audiosink as we did not do it when pausing
1516        if (mAudioSink != NULL && mAudioSink->ready()) {
1517            mAudioSink->setPlaybackRate(mPlaybackSettings);
1518        }
1519
1520        mMediaClock->setPlaybackRate(mPlaybackRate);
1521
1522        if (!mAudioQueue.empty()) {
1523            postDrainAudioQueue_l();
1524        }
1525    }
1526
1527    if (!mVideoQueue.empty()) {
1528        postDrainVideoQueue();
1529    }
1530}
1531
1532void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1533    if (mVideoScheduler == NULL) {
1534        mVideoScheduler = new VideoFrameScheduler();
1535    }
1536    mVideoScheduler->init(fps);
1537}
1538
1539int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
1540    Mutex::Autolock autoLock(mLock);
1541    return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
1542}
1543
1544int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
1545    Mutex::Autolock autoLock(mLock);
1546    return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
1547}
1548
1549bool NuPlayer::Renderer::getSyncQueues() {
1550    Mutex::Autolock autoLock(mLock);
1551    return mSyncQueues;
1552}
1553
1554// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs()
1555// as it acquires locks and may query the audio driver.
1556//
1557// Some calls could conceivably retrieve extrapolated data instead of
1558// accessing getTimestamp() or getPosition() every time a data buffer with
1559// a media time is received.
1560//
1561// Calculate duration of played samples if played at normal rate (i.e., 1.0).
1562int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) {
1563    uint32_t numFramesPlayed;
1564    int64_t numFramesPlayedAt;
1565    AudioTimestamp ts;
1566    static const int64_t kStaleTimestamp100ms = 100000;
1567
1568    status_t res = mAudioSink->getTimestamp(ts);
1569    if (res == OK) {                 // case 1: mixing audio tracks and offloaded tracks.
1570        numFramesPlayed = ts.mPosition;
1571        numFramesPlayedAt =
1572            ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
1573        const int64_t timestampAge = nowUs - numFramesPlayedAt;
1574        if (timestampAge > kStaleTimestamp100ms) {
1575            // This is an audio FIXME.
1576            // getTimestamp returns a timestamp which may come from audio mixing threads.
1577            // After pausing, the MixerThread may go idle, thus the mTime estimate may
1578            // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms,
1579            // the max latency should be about 25ms with an average around 12ms (to be verified).
1580            // For safety we use 100ms.
1581            ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)",
1582                    (long long)nowUs, (long long)numFramesPlayedAt);
1583            numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
1584        }
1585        //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1586    } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track
1587        numFramesPlayed = 0;
1588        numFramesPlayedAt = nowUs;
1589        //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
1590        //        numFramesPlayed, (long long)numFramesPlayedAt);
1591    } else {                         // case 3: transitory at new track or audio fast tracks.
1592        res = mAudioSink->getPosition(&numFramesPlayed);
1593        CHECK_EQ(res, (status_t)OK);
1594        numFramesPlayedAt = nowUs;
1595        numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */
1596        //ALOGD("getPosition: %u %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1597    }
1598
1599    //CHECK_EQ(numFramesPlayed & (1 << 31), 0);  // can't be negative until 12.4 hrs, test
1600    int64_t durationUs = getDurationUsIfPlayedAtSampleRate(numFramesPlayed)
1601            + nowUs - numFramesPlayedAt;
1602    if (durationUs < 0) {
1603        // Occurs when numFramesPlayed position is very small and the following:
1604        // (1) In case 1, the time nowUs is computed before getTimestamp() is called and
1605        //     numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed.
1606        // (2) In case 3, using getPosition and adding mAudioSink->latency() to
1607        //     numFramesPlayedAt, by a time amount greater than numFramesPlayed.
1608        //
1609        // Both of these are transitory conditions.
1610        ALOGV("getPlayedOutAudioDurationUs: negative duration %lld set to zero", (long long)durationUs);
1611        durationUs = 0;
1612    }
1613    ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)",
1614            (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt);
1615    return durationUs;
1616}
1617
1618void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) {
1619    if (mAudioTornDown) {
1620        return;
1621    }
1622    mAudioTornDown = true;
1623
1624    int64_t currentPositionUs;
1625    sp<AMessage> notify = mNotify->dup();
1626    if (getCurrentPosition(&currentPositionUs) == OK) {
1627        notify->setInt64("positionUs", currentPositionUs);
1628    }
1629
1630    mAudioSink->stop();
1631    mAudioSink->flush();
1632
1633    notify->setInt32("what", kWhatAudioTearDown);
1634    notify->setInt32("reason", reason);
1635    notify->post();
1636}
1637
1638void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1639    if (offloadingAudio()) {
1640        mWakeLock->acquire();
1641        sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
1642        msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
1643        msg->post(kOffloadPauseMaxUs);
1644    }
1645}
1646
1647void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
1648    if (offloadingAudio()) {
1649        mWakeLock->release(true);
1650        ++mAudioOffloadPauseTimeoutGeneration;
1651    }
1652}
1653
1654status_t NuPlayer::Renderer::onOpenAudioSink(
1655        const sp<AMessage> &format,
1656        bool offloadOnly,
1657        bool hasVideo,
1658        uint32_t flags) {
1659    ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1660            offloadOnly, offloadingAudio());
1661    bool audioSinkChanged = false;
1662
1663    int32_t numChannels;
1664    CHECK(format->findInt32("channel-count", &numChannels));
1665
1666    int32_t channelMask;
1667    if (!format->findInt32("channel-mask", &channelMask)) {
1668        // signal to the AudioSink to derive the mask from count.
1669        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1670    }
1671
1672    int32_t sampleRate;
1673    CHECK(format->findInt32("sample-rate", &sampleRate));
1674
1675    if (offloadingAudio()) {
1676        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1677        AString mime;
1678        CHECK(format->findString("mime", &mime));
1679        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1680
1681        if (err != OK) {
1682            ALOGE("Couldn't map mime \"%s\" to a valid "
1683                    "audio_format", mime.c_str());
1684            onDisableOffloadAudio();
1685        } else {
1686            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1687                    mime.c_str(), audioFormat);
1688
1689            int avgBitRate = -1;
1690            format->findInt32("bit-rate", &avgBitRate);
1691
1692            int32_t aacProfile = -1;
1693            if (audioFormat == AUDIO_FORMAT_AAC
1694                    && format->findInt32("aac-profile", &aacProfile)) {
1695                // Redefine AAC format as per aac profile
1696                mapAACProfileToAudioFormat(
1697                        audioFormat,
1698                        aacProfile);
1699            }
1700
1701            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1702            offloadInfo.duration_us = -1;
1703            format->findInt64(
1704                    "durationUs", &offloadInfo.duration_us);
1705            offloadInfo.sample_rate = sampleRate;
1706            offloadInfo.channel_mask = channelMask;
1707            offloadInfo.format = audioFormat;
1708            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1709            offloadInfo.bit_rate = avgBitRate;
1710            offloadInfo.has_video = hasVideo;
1711            offloadInfo.is_streaming = true;
1712
1713            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1714                ALOGV("openAudioSink: no change in offload mode");
1715                // no change from previous configuration, everything ok.
1716                return OK;
1717            }
1718            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1719
1720            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1721            uint32_t offloadFlags = flags;
1722            offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1723            offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1724            audioSinkChanged = true;
1725            mAudioSink->close();
1726
1727            err = mAudioSink->open(
1728                    sampleRate,
1729                    numChannels,
1730                    (audio_channel_mask_t)channelMask,
1731                    audioFormat,
1732                    0 /* bufferCount - unused */,
1733                    &NuPlayer::Renderer::AudioSinkCallback,
1734                    this,
1735                    (audio_output_flags_t)offloadFlags,
1736                    &offloadInfo);
1737
1738            if (err == OK) {
1739                err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1740            }
1741
1742            if (err == OK) {
1743                // If the playback is offloaded to h/w, we pass
1744                // the HAL some metadata information.
1745                // We don't want to do this for PCM because it
1746                // will be going through the AudioFlinger mixer
1747                // before reaching the hardware.
1748                // TODO
1749                mCurrentOffloadInfo = offloadInfo;
1750                if (!mPaused) { // for preview mode, don't start if paused
1751                    err = mAudioSink->start();
1752                }
1753                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1754            }
1755            if (err != OK) {
1756                // Clean up, fall back to non offload mode.
1757                mAudioSink->close();
1758                onDisableOffloadAudio();
1759                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1760                ALOGV("openAudioSink: offload failed");
1761            } else {
1762                mUseAudioCallback = true;  // offload mode transfers data through callback
1763                ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
1764            }
1765        }
1766    }
1767    if (!offloadOnly && !offloadingAudio()) {
1768        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1769        uint32_t pcmFlags = flags;
1770        pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1771
1772        const PcmInfo info = {
1773                (audio_channel_mask_t)channelMask,
1774                (audio_output_flags_t)pcmFlags,
1775                AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat
1776                numChannels,
1777                sampleRate
1778        };
1779        if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
1780            ALOGV("openAudioSink: no change in pcm mode");
1781            // no change from previous configuration, everything ok.
1782            return OK;
1783        }
1784
1785        audioSinkChanged = true;
1786        mAudioSink->close();
1787        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1788        // Note: It is possible to set up the callback, but not use it to send audio data.
1789        // This requires a fix in AudioSink to explicitly specify the transfer mode.
1790        mUseAudioCallback = getUseAudioCallbackSetting();
1791        if (mUseAudioCallback) {
1792            ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
1793        }
1794
1795        // Compute the desired buffer size.
1796        // For callback mode, the amount of time before wakeup is about half the buffer size.
1797        const uint32_t frameCount =
1798                (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000;
1799
1800        status_t err = mAudioSink->open(
1801                    sampleRate,
1802                    numChannels,
1803                    (audio_channel_mask_t)channelMask,
1804                    AUDIO_FORMAT_PCM_16_BIT,
1805                    0 /* bufferCount - unused */,
1806                    mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL,
1807                    mUseAudioCallback ? this : NULL,
1808                    (audio_output_flags_t)pcmFlags,
1809                    NULL,
1810                    true /* doNotReconnect */,
1811                    frameCount);
1812        if (err == OK) {
1813            err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1814        }
1815        if (err != OK) {
1816            ALOGW("openAudioSink: non offloaded open failed status: %d", err);
1817            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1818            return err;
1819        }
1820        mCurrentPcmInfo = info;
1821        if (!mPaused) { // for preview mode, don't start if paused
1822            mAudioSink->start();
1823        }
1824    }
1825    if (audioSinkChanged) {
1826        onAudioSinkChanged();
1827    }
1828    mAudioTornDown = false;
1829    return OK;
1830}
1831
1832void NuPlayer::Renderer::onCloseAudioSink() {
1833    mAudioSink->close();
1834    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1835    mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1836}
1837
1838}  // namespace android
1839
1840