NuPlayerRenderer.cpp revision b12ea0bbda453769584efcea69054b41d9b4c4c7 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22#include <cutils/properties.h>
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26#include <media/stagefright/foundation/AUtils.h>
27#include <media/stagefright/foundation/AWakeLock.h>
28#include <media/stagefright/MediaClock.h>
29#include <media/stagefright/MediaErrors.h>
30#include <media/stagefright/MetaData.h>
31#include <media/stagefright/Utils.h>
32#include <media/stagefright/VideoFrameScheduler.h>
33
34#include <inttypes.h>
35
36namespace android {
37
38/*
39 * Example of common configuration settings in shell script form
40
41   #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager
42   adb shell setprop audio.offload.disable 1
43
44   #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager
45   adb shell setprop audio.offload.video 1
46
47   #Use audio callbacks for PCM data
48   adb shell setprop media.stagefright.audio.cbk 1
49
50   #Use deep buffer for PCM data with video (it is generally enabled for audio-only)
51   adb shell setprop media.stagefright.audio.deep 1
52
53   #Set size of buffers for pcm audio sink in msec (example: 1000 msec)
54   adb shell setprop media.stagefright.audio.sink 1000
55
56 * These configurations take effect for the next track played (not the current track).
57 */
58
59static inline bool getUseAudioCallbackSetting() {
60    return property_get_bool("media.stagefright.audio.cbk", false /* default_value */);
61}
62
63static inline int32_t getAudioSinkPcmMsSetting() {
64    return property_get_int32(
65            "media.stagefright.audio.sink", 500 /* default_value */);
66}
67
68// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
69// is closed to allow the audio DSP to power down.
70static const int64_t kOffloadPauseMaxUs = 10000000ll;
71
72// static
73const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
74        AUDIO_CHANNEL_NONE,
75        AUDIO_OUTPUT_FLAG_NONE,
76        AUDIO_FORMAT_INVALID,
77        0, // mNumChannels
78        0 // mSampleRate
79};
80
81// static
82const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
83
84NuPlayer::Renderer::Renderer(
85        const sp<MediaPlayerBase::AudioSink> &sink,
86        const sp<AMessage> &notify,
87        uint32_t flags)
88    : mAudioSink(sink),
89      mNotify(notify),
90      mFlags(flags),
91      mNumFramesWritten(0),
92      mDrainAudioQueuePending(false),
93      mDrainVideoQueuePending(false),
94      mAudioQueueGeneration(0),
95      mVideoQueueGeneration(0),
96      mAudioDrainGeneration(0),
97      mVideoDrainGeneration(0),
98      mAudioEOSGeneration(0),
99      mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
100      mAudioFirstAnchorTimeMediaUs(-1),
101      mAnchorTimeMediaUs(-1),
102      mAnchorNumFramesWritten(-1),
103      mVideoLateByUs(0ll),
104      mHasAudio(false),
105      mHasVideo(false),
106      mNotifyCompleteAudio(false),
107      mNotifyCompleteVideo(false),
108      mSyncQueues(false),
109      mPaused(false),
110      mPauseDrainAudioAllowedUs(0),
111      mVideoSampleReceived(false),
112      mVideoRenderingStarted(false),
113      mVideoRenderingStartGeneration(0),
114      mAudioRenderingStartGeneration(0),
115      mRenderingDataDelivered(false),
116      mLastAudioMediaTimeUs(-1),
117      mAudioOffloadPauseTimeoutGeneration(0),
118      mAudioTornDown(false),
119      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
120      mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
121      mTotalBuffersQueued(0),
122      mLastAudioBufferDrained(0),
123      mUseAudioCallback(false),
124      mWakeLock(new AWakeLock()) {
125    mMediaClock = new MediaClock;
126    mPlaybackRate = mPlaybackSettings.mSpeed;
127    mMediaClock->setPlaybackRate(mPlaybackRate);
128}
129
130NuPlayer::Renderer::~Renderer() {
131    if (offloadingAudio()) {
132        mAudioSink->stop();
133        mAudioSink->flush();
134        mAudioSink->close();
135    }
136}
137
138void NuPlayer::Renderer::queueBuffer(
139        bool audio,
140        const sp<ABuffer> &buffer,
141        const sp<AMessage> &notifyConsumed) {
142    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
143    msg->setInt32("queueGeneration", getQueueGeneration(audio));
144    msg->setInt32("audio", static_cast<int32_t>(audio));
145    msg->setBuffer("buffer", buffer);
146    msg->setMessage("notifyConsumed", notifyConsumed);
147    msg->post();
148}
149
150void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
151    CHECK_NE(finalResult, (status_t)OK);
152
153    sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
154    msg->setInt32("queueGeneration", getQueueGeneration(audio));
155    msg->setInt32("audio", static_cast<int32_t>(audio));
156    msg->setInt32("finalResult", finalResult);
157    msg->post();
158}
159
160status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) {
161    sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
162    writeToAMessage(msg, rate);
163    sp<AMessage> response;
164    status_t err = msg->postAndAwaitResponse(&response);
165    if (err == OK && response != NULL) {
166        CHECK(response->findInt32("err", &err));
167    }
168    return err;
169}
170
171status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) {
172    if (rate.mSpeed == 0.f) {
173        onPause();
174        // don't call audiosink's setPlaybackRate if pausing, as pitch does not
175        // have to correspond to the any non-0 speed (e.g old speed). Keep
176        // settings nonetheless, using the old speed, in case audiosink changes.
177        AudioPlaybackRate newRate = rate;
178        newRate.mSpeed = mPlaybackSettings.mSpeed;
179        mPlaybackSettings = newRate;
180        return OK;
181    }
182
183    if (mAudioSink != NULL && mAudioSink->ready()) {
184        status_t err = mAudioSink->setPlaybackRate(rate);
185        if (err != OK) {
186            return err;
187        }
188    }
189    mPlaybackSettings = rate;
190    mPlaybackRate = rate.mSpeed;
191    mMediaClock->setPlaybackRate(mPlaybackRate);
192    return OK;
193}
194
195status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
196    sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
197    sp<AMessage> response;
198    status_t err = msg->postAndAwaitResponse(&response);
199    if (err == OK && response != NULL) {
200        CHECK(response->findInt32("err", &err));
201        if (err == OK) {
202            readFromAMessage(response, rate);
203        }
204    }
205    return err;
206}
207
208status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
209    if (mAudioSink != NULL && mAudioSink->ready()) {
210        status_t err = mAudioSink->getPlaybackRate(rate);
211        if (err == OK) {
212            if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) {
213                ALOGW("correcting mismatch in internal/external playback rate");
214            }
215            // get playback settings used by audiosink, as it may be
216            // slightly off due to audiosink not taking small changes.
217            mPlaybackSettings = *rate;
218            if (mPaused) {
219                rate->mSpeed = 0.f;
220            }
221        }
222        return err;
223    }
224    *rate = mPlaybackSettings;
225    return OK;
226}
227
228status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
229    sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
230    writeToAMessage(msg, sync, videoFpsHint);
231    sp<AMessage> response;
232    status_t err = msg->postAndAwaitResponse(&response);
233    if (err == OK && response != NULL) {
234        CHECK(response->findInt32("err", &err));
235    }
236    return err;
237}
238
239status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) {
240    if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
241        return BAD_VALUE;
242    }
243    // TODO: support sync sources
244    return INVALID_OPERATION;
245}
246
247status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) {
248    sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
249    sp<AMessage> response;
250    status_t err = msg->postAndAwaitResponse(&response);
251    if (err == OK && response != NULL) {
252        CHECK(response->findInt32("err", &err));
253        if (err == OK) {
254            readFromAMessage(response, sync, videoFps);
255        }
256    }
257    return err;
258}
259
260status_t NuPlayer::Renderer::onGetSyncSettings(
261        AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
262    *sync = mSyncSettings;
263    *videoFps = -1.f;
264    return OK;
265}
266
267void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
268    {
269        Mutex::Autolock autoLock(mLock);
270        if (audio) {
271            mNotifyCompleteAudio |= notifyComplete;
272            clearAudioFirstAnchorTime_l();
273            ++mAudioQueueGeneration;
274            ++mAudioDrainGeneration;
275        } else {
276            mNotifyCompleteVideo |= notifyComplete;
277            ++mVideoQueueGeneration;
278            ++mVideoDrainGeneration;
279        }
280
281        clearAnchorTime_l();
282        mVideoLateByUs = 0;
283        mSyncQueues = false;
284    }
285
286    sp<AMessage> msg = new AMessage(kWhatFlush, this);
287    msg->setInt32("audio", static_cast<int32_t>(audio));
288    msg->post();
289}
290
291void NuPlayer::Renderer::signalTimeDiscontinuity() {
292}
293
294void NuPlayer::Renderer::signalDisableOffloadAudio() {
295    (new AMessage(kWhatDisableOffloadAudio, this))->post();
296}
297
298void NuPlayer::Renderer::signalEnableOffloadAudio() {
299    (new AMessage(kWhatEnableOffloadAudio, this))->post();
300}
301
302void NuPlayer::Renderer::pause() {
303    (new AMessage(kWhatPause, this))->post();
304}
305
306void NuPlayer::Renderer::resume() {
307    (new AMessage(kWhatResume, this))->post();
308}
309
310void NuPlayer::Renderer::setVideoFrameRate(float fps) {
311    sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
312    msg->setFloat("frame-rate", fps);
313    msg->post();
314}
315
316// Called on any threads.
317status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
318    return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
319}
320
321void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
322    mAudioFirstAnchorTimeMediaUs = -1;
323    mMediaClock->setStartingTimeMedia(-1);
324}
325
326void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
327    if (mAudioFirstAnchorTimeMediaUs == -1) {
328        mAudioFirstAnchorTimeMediaUs = mediaUs;
329        mMediaClock->setStartingTimeMedia(mediaUs);
330    }
331}
332
333void NuPlayer::Renderer::clearAnchorTime_l() {
334    mMediaClock->clearAnchor();
335    mAnchorTimeMediaUs = -1;
336    mAnchorNumFramesWritten = -1;
337}
338
339void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
340    Mutex::Autolock autoLock(mLock);
341    mVideoLateByUs = lateUs;
342}
343
344int64_t NuPlayer::Renderer::getVideoLateByUs() {
345    Mutex::Autolock autoLock(mLock);
346    return mVideoLateByUs;
347}
348
349status_t NuPlayer::Renderer::openAudioSink(
350        const sp<AMessage> &format,
351        bool offloadOnly,
352        bool hasVideo,
353        uint32_t flags,
354        bool *isOffloaded) {
355    sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
356    msg->setMessage("format", format);
357    msg->setInt32("offload-only", offloadOnly);
358    msg->setInt32("has-video", hasVideo);
359    msg->setInt32("flags", flags);
360
361    sp<AMessage> response;
362    msg->postAndAwaitResponse(&response);
363
364    int32_t err;
365    if (!response->findInt32("err", &err)) {
366        err = INVALID_OPERATION;
367    } else if (err == OK && isOffloaded != NULL) {
368        int32_t offload;
369        CHECK(response->findInt32("offload", &offload));
370        *isOffloaded = (offload != 0);
371    }
372    return err;
373}
374
375void NuPlayer::Renderer::closeAudioSink() {
376    sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
377
378    sp<AMessage> response;
379    msg->postAndAwaitResponse(&response);
380}
381
382void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
383    switch (msg->what()) {
384        case kWhatOpenAudioSink:
385        {
386            sp<AMessage> format;
387            CHECK(msg->findMessage("format", &format));
388
389            int32_t offloadOnly;
390            CHECK(msg->findInt32("offload-only", &offloadOnly));
391
392            int32_t hasVideo;
393            CHECK(msg->findInt32("has-video", &hasVideo));
394
395            uint32_t flags;
396            CHECK(msg->findInt32("flags", (int32_t *)&flags));
397
398            status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
399
400            sp<AMessage> response = new AMessage;
401            response->setInt32("err", err);
402            response->setInt32("offload", offloadingAudio());
403
404            sp<AReplyToken> replyID;
405            CHECK(msg->senderAwaitsResponse(&replyID));
406            response->postReply(replyID);
407
408            break;
409        }
410
411        case kWhatCloseAudioSink:
412        {
413            sp<AReplyToken> replyID;
414            CHECK(msg->senderAwaitsResponse(&replyID));
415
416            onCloseAudioSink();
417
418            sp<AMessage> response = new AMessage;
419            response->postReply(replyID);
420            break;
421        }
422
423        case kWhatStopAudioSink:
424        {
425            mAudioSink->stop();
426            break;
427        }
428
429        case kWhatDrainAudioQueue:
430        {
431            mDrainAudioQueuePending = false;
432
433            int32_t generation;
434            CHECK(msg->findInt32("drainGeneration", &generation));
435            if (generation != getDrainGeneration(true /* audio */)) {
436                break;
437            }
438
439            if (onDrainAudioQueue()) {
440                uint32_t numFramesPlayed;
441                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
442                         (status_t)OK);
443
444                uint32_t numFramesPendingPlayout =
445                    mNumFramesWritten - numFramesPlayed;
446
447                // This is how long the audio sink will have data to
448                // play back.
449                int64_t delayUs =
450                    mAudioSink->msecsPerFrame()
451                        * numFramesPendingPlayout * 1000ll;
452                if (mPlaybackRate > 1.0f) {
453                    delayUs /= mPlaybackRate;
454                }
455
456                // Let's give it more data after about half that time
457                // has elapsed.
458                Mutex::Autolock autoLock(mLock);
459                postDrainAudioQueue_l(delayUs / 2);
460            }
461            break;
462        }
463
464        case kWhatDrainVideoQueue:
465        {
466            int32_t generation;
467            CHECK(msg->findInt32("drainGeneration", &generation));
468            if (generation != getDrainGeneration(false /* audio */)) {
469                break;
470            }
471
472            mDrainVideoQueuePending = false;
473
474            onDrainVideoQueue();
475
476            postDrainVideoQueue();
477            break;
478        }
479
480        case kWhatPostDrainVideoQueue:
481        {
482            int32_t generation;
483            CHECK(msg->findInt32("drainGeneration", &generation));
484            if (generation != getDrainGeneration(false /* audio */)) {
485                break;
486            }
487
488            mDrainVideoQueuePending = false;
489            postDrainVideoQueue();
490            break;
491        }
492
493        case kWhatQueueBuffer:
494        {
495            onQueueBuffer(msg);
496            break;
497        }
498
499        case kWhatQueueEOS:
500        {
501            onQueueEOS(msg);
502            break;
503        }
504
505        case kWhatEOS:
506        {
507            int32_t generation;
508            CHECK(msg->findInt32("audioEOSGeneration", &generation));
509            if (generation != mAudioEOSGeneration) {
510                break;
511            }
512            status_t finalResult;
513            CHECK(msg->findInt32("finalResult", &finalResult));
514            notifyEOS(true /* audio */, finalResult);
515            break;
516        }
517
518        case kWhatConfigPlayback:
519        {
520            sp<AReplyToken> replyID;
521            CHECK(msg->senderAwaitsResponse(&replyID));
522            AudioPlaybackRate rate;
523            readFromAMessage(msg, &rate);
524            status_t err = onConfigPlayback(rate);
525            sp<AMessage> response = new AMessage;
526            response->setInt32("err", err);
527            response->postReply(replyID);
528            break;
529        }
530
531        case kWhatGetPlaybackSettings:
532        {
533            sp<AReplyToken> replyID;
534            CHECK(msg->senderAwaitsResponse(&replyID));
535            AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT;
536            status_t err = onGetPlaybackSettings(&rate);
537            sp<AMessage> response = new AMessage;
538            if (err == OK) {
539                writeToAMessage(response, rate);
540            }
541            response->setInt32("err", err);
542            response->postReply(replyID);
543            break;
544        }
545
546        case kWhatConfigSync:
547        {
548            sp<AReplyToken> replyID;
549            CHECK(msg->senderAwaitsResponse(&replyID));
550            AVSyncSettings sync;
551            float videoFpsHint;
552            readFromAMessage(msg, &sync, &videoFpsHint);
553            status_t err = onConfigSync(sync, videoFpsHint);
554            sp<AMessage> response = new AMessage;
555            response->setInt32("err", err);
556            response->postReply(replyID);
557            break;
558        }
559
560        case kWhatGetSyncSettings:
561        {
562            sp<AReplyToken> replyID;
563            CHECK(msg->senderAwaitsResponse(&replyID));
564
565            ALOGV("kWhatGetSyncSettings");
566            AVSyncSettings sync;
567            float videoFps = -1.f;
568            status_t err = onGetSyncSettings(&sync, &videoFps);
569            sp<AMessage> response = new AMessage;
570            if (err == OK) {
571                writeToAMessage(response, sync, videoFps);
572            }
573            response->setInt32("err", err);
574            response->postReply(replyID);
575            break;
576        }
577
578        case kWhatFlush:
579        {
580            onFlush(msg);
581            break;
582        }
583
584        case kWhatDisableOffloadAudio:
585        {
586            onDisableOffloadAudio();
587            break;
588        }
589
590        case kWhatEnableOffloadAudio:
591        {
592            onEnableOffloadAudio();
593            break;
594        }
595
596        case kWhatPause:
597        {
598            onPause();
599            break;
600        }
601
602        case kWhatResume:
603        {
604            onResume();
605            break;
606        }
607
608        case kWhatSetVideoFrameRate:
609        {
610            float fps;
611            CHECK(msg->findFloat("frame-rate", &fps));
612            onSetVideoFrameRate(fps);
613            break;
614        }
615
616        case kWhatAudioTearDown:
617        {
618            onAudioTearDown(kDueToError);
619            break;
620        }
621
622        case kWhatAudioOffloadPauseTimeout:
623        {
624            int32_t generation;
625            CHECK(msg->findInt32("drainGeneration", &generation));
626            if (generation != mAudioOffloadPauseTimeoutGeneration) {
627                break;
628            }
629            ALOGV("Audio Offload tear down due to pause timeout.");
630            onAudioTearDown(kDueToTimeout);
631            mWakeLock->release();
632            break;
633        }
634
635        default:
636            TRESPASS();
637            break;
638    }
639}
640
641void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
642    if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) {
643        return;
644    }
645
646    if (mAudioQueue.empty()) {
647        return;
648    }
649
650    // FIXME: if paused, wait until AudioTrack stop() is complete before delivering data.
651    if (mPaused) {
652        const int64_t diffUs = mPauseDrainAudioAllowedUs - ALooper::GetNowUs();
653        if (diffUs > delayUs) {
654            delayUs = diffUs;
655        }
656    }
657
658    mDrainAudioQueuePending = true;
659    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
660    msg->setInt32("drainGeneration", mAudioDrainGeneration);
661    msg->post(delayUs);
662}
663
664void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
665    mAudioRenderingStartGeneration = mAudioDrainGeneration;
666    mVideoRenderingStartGeneration = mVideoDrainGeneration;
667    mRenderingDataDelivered = false;
668}
669
670void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
671    if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
672        mAudioRenderingStartGeneration == mAudioDrainGeneration) {
673        mRenderingDataDelivered = true;
674        if (mPaused) {
675            return;
676        }
677        mVideoRenderingStartGeneration = -1;
678        mAudioRenderingStartGeneration = -1;
679
680        sp<AMessage> notify = mNotify->dup();
681        notify->setInt32("what", kWhatMediaRenderingStart);
682        notify->post();
683    }
684}
685
686// static
687size_t NuPlayer::Renderer::AudioSinkCallback(
688        MediaPlayerBase::AudioSink * /* audioSink */,
689        void *buffer,
690        size_t size,
691        void *cookie,
692        MediaPlayerBase::AudioSink::cb_event_t event) {
693    NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
694
695    switch (event) {
696        case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
697        {
698            return me->fillAudioBuffer(buffer, size);
699            break;
700        }
701
702        case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
703        {
704            ALOGV("AudioSink::CB_EVENT_STREAM_END");
705            me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
706            break;
707        }
708
709        case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
710        {
711            ALOGV("AudioSink::CB_EVENT_TEAR_DOWN");
712            me->notifyAudioTearDown();
713            break;
714        }
715    }
716
717    return 0;
718}
719
720size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
721    Mutex::Autolock autoLock(mLock);
722
723    if (!mUseAudioCallback) {
724        return 0;
725    }
726
727    bool hasEOS = false;
728
729    size_t sizeCopied = 0;
730    bool firstEntry = true;
731    QueueEntry *entry;  // will be valid after while loop if hasEOS is set.
732    while (sizeCopied < size && !mAudioQueue.empty()) {
733        entry = &*mAudioQueue.begin();
734
735        if (entry->mBuffer == NULL) { // EOS
736            hasEOS = true;
737            mAudioQueue.erase(mAudioQueue.begin());
738            break;
739        }
740
741        if (firstEntry && entry->mOffset == 0) {
742            firstEntry = false;
743            int64_t mediaTimeUs;
744            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
745            ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
746            setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
747        }
748
749        size_t copy = entry->mBuffer->size() - entry->mOffset;
750        size_t sizeRemaining = size - sizeCopied;
751        if (copy > sizeRemaining) {
752            copy = sizeRemaining;
753        }
754
755        memcpy((char *)buffer + sizeCopied,
756               entry->mBuffer->data() + entry->mOffset,
757               copy);
758
759        entry->mOffset += copy;
760        if (entry->mOffset == entry->mBuffer->size()) {
761            entry->mNotifyConsumed->post();
762            mAudioQueue.erase(mAudioQueue.begin());
763            entry = NULL;
764        }
765        sizeCopied += copy;
766
767        notifyIfMediaRenderingStarted_l();
768    }
769
770    if (mAudioFirstAnchorTimeMediaUs >= 0) {
771        int64_t nowUs = ALooper::GetNowUs();
772        int64_t nowMediaUs =
773            mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs);
774        // we don't know how much data we are queueing for offloaded tracks.
775        mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
776    }
777
778    // for non-offloaded audio, we need to compute the frames written because
779    // there is no EVENT_STREAM_END notification. The frames written gives
780    // an estimate on the pending played out duration.
781    if (!offloadingAudio()) {
782        mNumFramesWritten += sizeCopied / mAudioSink->frameSize();
783    }
784
785    if (hasEOS) {
786        (new AMessage(kWhatStopAudioSink, this))->post();
787        // As there is currently no EVENT_STREAM_END callback notification for
788        // non-offloaded audio tracks, we need to post the EOS ourselves.
789        if (!offloadingAudio()) {
790            int64_t postEOSDelayUs = 0;
791            if (mAudioSink->needsTrailingPadding()) {
792                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
793            }
794            ALOGV("fillAudioBuffer: notifyEOS "
795                    "mNumFramesWritten:%u  finalResult:%d  postEOSDelay:%lld",
796                    mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs);
797            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
798        }
799    }
800    return sizeCopied;
801}
802
803void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() {
804    List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it;
805    bool foundEOS = false;
806    while (it != mAudioQueue.end()) {
807        int32_t eos;
808        QueueEntry *entry = &*it++;
809        if (entry->mBuffer == NULL
810                || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) {
811            itEOS = it;
812            foundEOS = true;
813        }
814    }
815
816    if (foundEOS) {
817        // post all replies before EOS and drop the samples
818        for (it = mAudioQueue.begin(); it != itEOS; it++) {
819            if (it->mBuffer == NULL) {
820                // delay doesn't matter as we don't even have an AudioTrack
821                notifyEOS(true /* audio */, it->mFinalResult);
822            } else {
823                it->mNotifyConsumed->post();
824            }
825        }
826        mAudioQueue.erase(mAudioQueue.begin(), itEOS);
827    }
828}
829
830bool NuPlayer::Renderer::onDrainAudioQueue() {
831    // TODO: This call to getPosition checks if AudioTrack has been created
832    // in AudioSink before draining audio. If AudioTrack doesn't exist, then
833    // CHECKs on getPosition will fail.
834    // We still need to figure out why AudioTrack is not created when
835    // this function is called. One possible reason could be leftover
836    // audio. Another possible place is to check whether decoder
837    // has received INFO_FORMAT_CHANGED as the first buffer since
838    // AudioSink is opened there, and possible interactions with flush
839    // immediately after start. Investigate error message
840    // "vorbis_dsp_synthesis returned -135", along with RTSP.
841    uint32_t numFramesPlayed;
842    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
843        // When getPosition fails, renderer will not reschedule the draining
844        // unless new samples are queued.
845        // If we have pending EOS (or "eos" marker for discontinuities), we need
846        // to post these now as NuPlayerDecoder might be waiting for it.
847        drainAudioQueueUntilLastEOS();
848
849        ALOGW("onDrainAudioQueue(): audio sink is not ready");
850        return false;
851    }
852
853#if 0
854    ssize_t numFramesAvailableToWrite =
855        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
856
857    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
858        ALOGI("audio sink underrun");
859    } else {
860        ALOGV("audio queue has %d frames left to play",
861             mAudioSink->frameCount() - numFramesAvailableToWrite);
862    }
863#endif
864
865    uint32_t prevFramesWritten = mNumFramesWritten;
866    while (!mAudioQueue.empty()) {
867        QueueEntry *entry = &*mAudioQueue.begin();
868
869        mLastAudioBufferDrained = entry->mBufferOrdinal;
870
871        if (entry->mBuffer == NULL) {
872            // EOS
873            int64_t postEOSDelayUs = 0;
874            if (mAudioSink->needsTrailingPadding()) {
875                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
876            }
877            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
878            mLastAudioMediaTimeUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
879
880            mAudioQueue.erase(mAudioQueue.begin());
881            entry = NULL;
882            if (mAudioSink->needsTrailingPadding()) {
883                // If we're not in gapless playback (i.e. through setNextPlayer), we
884                // need to stop the track here, because that will play out the last
885                // little bit at the end of the file. Otherwise short files won't play.
886                mAudioSink->stop();
887                mNumFramesWritten = 0;
888            }
889            return false;
890        }
891
892        // ignore 0-sized buffer which could be EOS marker with no data
893        if (entry->mOffset == 0 && entry->mBuffer->size() > 0) {
894            int64_t mediaTimeUs;
895            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
896            ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs",
897                    mediaTimeUs / 1E6);
898            onNewAudioMediaTime(mediaTimeUs);
899        }
900
901        size_t copy = entry->mBuffer->size() - entry->mOffset;
902
903        ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
904                                            copy, false /* blocking */);
905        if (written < 0) {
906            // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
907            if (written == WOULD_BLOCK) {
908                ALOGV("AudioSink write would block when writing %zu bytes", copy);
909            } else {
910                ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
911                // This can only happen when AudioSink was opened with doNotReconnect flag set to
912                // true, in which case the NuPlayer will handle the reconnect.
913                notifyAudioTearDown();
914            }
915            break;
916        }
917
918        entry->mOffset += written;
919        if (entry->mOffset == entry->mBuffer->size()) {
920            entry->mNotifyConsumed->post();
921            mAudioQueue.erase(mAudioQueue.begin());
922
923            entry = NULL;
924        }
925
926        size_t copiedFrames = written / mAudioSink->frameSize();
927        mNumFramesWritten += copiedFrames;
928
929        {
930            Mutex::Autolock autoLock(mLock);
931            notifyIfMediaRenderingStarted_l();
932        }
933
934        if (written != (ssize_t)copy) {
935            // A short count was received from AudioSink::write()
936            //
937            // AudioSink write is called in non-blocking mode.
938            // It may return with a short count when:
939            //
940            // 1) Size to be copied is not a multiple of the frame size. We consider this fatal.
941            // 2) The data to be copied exceeds the available buffer in AudioSink.
942            // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
943            // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
944
945            // (Case 1)
946            // Must be a multiple of the frame size.  If it is not a multiple of a frame size, it
947            // needs to fail, as we should not carry over fractional frames between calls.
948            CHECK_EQ(copy % mAudioSink->frameSize(), 0);
949
950            // (Case 2, 3, 4)
951            // Return early to the caller.
952            // Beware of calling immediately again as this may busy-loop if you are not careful.
953            ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
954            break;
955        }
956    }
957    int64_t maxTimeMedia;
958    {
959        Mutex::Autolock autoLock(mLock);
960        maxTimeMedia =
961            mAnchorTimeMediaUs +
962                    (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
963                            * 1000LL * mAudioSink->msecsPerFrame());
964    }
965    mMediaClock->updateMaxTimeMedia(maxTimeMedia);
966
967    // calculate whether we need to reschedule another write.
968    bool reschedule = !mAudioQueue.empty()
969            && (!mPaused
970                || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers
971    //ALOGD("reschedule:%d  empty:%d  mPaused:%d  prevFramesWritten:%u  mNumFramesWritten:%u",
972    //        reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten);
973    return reschedule;
974}
975
976int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
977    int32_t sampleRate = offloadingAudio() ?
978            mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
979    if (sampleRate == 0) {
980        ALOGE("sampleRate is 0 in %s mode", offloadingAudio() ? "offload" : "non-offload");
981        return 0;
982    }
983    // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
984    return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate);
985}
986
987// Calculate duration of pending samples if played at normal rate (i.e., 1.0).
988int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
989    int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
990    return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs);
991}
992
993int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
994    int64_t realUs;
995    if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
996        // If failed to get current position, e.g. due to audio clock is
997        // not ready, then just play out video immediately without delay.
998        return nowUs;
999    }
1000    return realUs;
1001}
1002
1003void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
1004    Mutex::Autolock autoLock(mLock);
1005    // TRICKY: vorbis decoder generates multiple frames with the same
1006    // timestamp, so only update on the first frame with a given timestamp
1007    if (mediaTimeUs == mAnchorTimeMediaUs) {
1008        return;
1009    }
1010    setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
1011    int64_t nowUs = ALooper::GetNowUs();
1012    int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
1013    mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
1014    mAnchorNumFramesWritten = mNumFramesWritten;
1015    mAnchorTimeMediaUs = mediaTimeUs;
1016}
1017
1018// Called without mLock acquired.
1019void NuPlayer::Renderer::postDrainVideoQueue() {
1020    if (mDrainVideoQueuePending
1021            || getSyncQueues()
1022            || (mPaused && mVideoSampleReceived)) {
1023        return;
1024    }
1025
1026    if (mVideoQueue.empty()) {
1027        return;
1028    }
1029
1030    QueueEntry &entry = *mVideoQueue.begin();
1031
1032    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
1033    msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
1034
1035    if (entry.mBuffer == NULL) {
1036        // EOS doesn't carry a timestamp.
1037        msg->post();
1038        mDrainVideoQueuePending = true;
1039        return;
1040    }
1041
1042    int64_t delayUs;
1043    int64_t nowUs = ALooper::GetNowUs();
1044    int64_t realTimeUs;
1045    if (mFlags & FLAG_REAL_TIME) {
1046        int64_t mediaTimeUs;
1047        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1048        realTimeUs = mediaTimeUs;
1049    } else {
1050        int64_t mediaTimeUs;
1051        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1052
1053        {
1054            Mutex::Autolock autoLock(mLock);
1055            if (mAnchorTimeMediaUs < 0) {
1056                mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
1057                mAnchorTimeMediaUs = mediaTimeUs;
1058                realTimeUs = nowUs;
1059            } else {
1060                realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1061            }
1062        }
1063        if (!mHasAudio) {
1064            // smooth out videos >= 10fps
1065            mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
1066        }
1067
1068        // Heuristics to handle situation when media time changed without a
1069        // discontinuity. If we have not drained an audio buffer that was
1070        // received after this buffer, repost in 10 msec. Otherwise repost
1071        // in 500 msec.
1072        delayUs = realTimeUs - nowUs;
1073        if (delayUs > 500000) {
1074            int64_t postDelayUs = 500000;
1075            if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) {
1076                postDelayUs = 10000;
1077            }
1078            msg->setWhat(kWhatPostDrainVideoQueue);
1079            msg->post(postDelayUs);
1080            mVideoScheduler->restart();
1081            ALOGI("possible video time jump of %dms, retrying in %dms",
1082                    (int)(delayUs / 1000), (int)(postDelayUs / 1000));
1083            mDrainVideoQueuePending = true;
1084            return;
1085        }
1086    }
1087
1088    realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
1089    int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
1090
1091    delayUs = realTimeUs - nowUs;
1092
1093    ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
1094    // post 2 display refreshes before rendering is due
1095    msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
1096
1097    mDrainVideoQueuePending = true;
1098}
1099
1100void NuPlayer::Renderer::onDrainVideoQueue() {
1101    if (mVideoQueue.empty()) {
1102        return;
1103    }
1104
1105    QueueEntry *entry = &*mVideoQueue.begin();
1106
1107    if (entry->mBuffer == NULL) {
1108        // EOS
1109
1110        notifyEOS(false /* audio */, entry->mFinalResult);
1111
1112        mVideoQueue.erase(mVideoQueue.begin());
1113        entry = NULL;
1114
1115        setVideoLateByUs(0);
1116        return;
1117    }
1118
1119    int64_t nowUs = -1;
1120    int64_t realTimeUs;
1121    int64_t mediaTimeUs = -1;
1122    if (mFlags & FLAG_REAL_TIME) {
1123        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
1124    } else {
1125        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1126
1127        nowUs = ALooper::GetNowUs();
1128        realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1129    }
1130
1131    bool tooLate = false;
1132
1133    if (!mPaused) {
1134        if (nowUs == -1) {
1135            nowUs = ALooper::GetNowUs();
1136        }
1137        setVideoLateByUs(nowUs - realTimeUs);
1138        tooLate = (mVideoLateByUs > 40000);
1139
1140        if (tooLate) {
1141            ALOGV("video late by %lld us (%.2f secs)",
1142                 (long long)mVideoLateByUs, mVideoLateByUs / 1E6);
1143        } else {
1144            int64_t mediaUs = 0;
1145            mMediaClock->getMediaTime(realTimeUs, &mediaUs);
1146            ALOGV("rendering video at media time %.2f secs",
1147                    (mFlags & FLAG_REAL_TIME ? realTimeUs :
1148                    mediaUs) / 1E6);
1149
1150            if (!(mFlags & FLAG_REAL_TIME)
1151                    && mLastAudioMediaTimeUs != -1
1152                    && mediaTimeUs > mLastAudioMediaTimeUs) {
1153                // If audio ends before video, video continues to drive media clock.
1154                // Also smooth out videos >= 10fps.
1155                mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
1156            }
1157        }
1158    } else {
1159        setVideoLateByUs(0);
1160        if (!mVideoSampleReceived && !mHasAudio) {
1161            // This will ensure that the first frame after a flush won't be used as anchor
1162            // when renderer is in paused state, because resume can happen any time after seek.
1163            Mutex::Autolock autoLock(mLock);
1164            clearAnchorTime_l();
1165        }
1166    }
1167
1168    entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
1169    entry->mNotifyConsumed->setInt32("render", !tooLate);
1170    entry->mNotifyConsumed->post();
1171    mVideoQueue.erase(mVideoQueue.begin());
1172    entry = NULL;
1173
1174    mVideoSampleReceived = true;
1175
1176    if (!mPaused) {
1177        if (!mVideoRenderingStarted) {
1178            mVideoRenderingStarted = true;
1179            notifyVideoRenderingStart();
1180        }
1181        Mutex::Autolock autoLock(mLock);
1182        notifyIfMediaRenderingStarted_l();
1183    }
1184}
1185
1186void NuPlayer::Renderer::notifyVideoRenderingStart() {
1187    sp<AMessage> notify = mNotify->dup();
1188    notify->setInt32("what", kWhatVideoRenderingStart);
1189    notify->post();
1190}
1191
1192void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
1193    if (audio && delayUs > 0) {
1194        sp<AMessage> msg = new AMessage(kWhatEOS, this);
1195        msg->setInt32("audioEOSGeneration", mAudioEOSGeneration);
1196        msg->setInt32("finalResult", finalResult);
1197        msg->post(delayUs);
1198        return;
1199    }
1200    sp<AMessage> notify = mNotify->dup();
1201    notify->setInt32("what", kWhatEOS);
1202    notify->setInt32("audio", static_cast<int32_t>(audio));
1203    notify->setInt32("finalResult", finalResult);
1204    notify->post(delayUs);
1205}
1206
1207void NuPlayer::Renderer::notifyAudioTearDown() {
1208    (new AMessage(kWhatAudioTearDown, this))->post();
1209}
1210
1211void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
1212    int32_t audio;
1213    CHECK(msg->findInt32("audio", &audio));
1214
1215    if (dropBufferIfStale(audio, msg)) {
1216        return;
1217    }
1218
1219    if (audio) {
1220        mHasAudio = true;
1221    } else {
1222        mHasVideo = true;
1223    }
1224
1225    if (mHasVideo) {
1226        if (mVideoScheduler == NULL) {
1227            mVideoScheduler = new VideoFrameScheduler();
1228            mVideoScheduler->init();
1229        }
1230    }
1231
1232    sp<ABuffer> buffer;
1233    CHECK(msg->findBuffer("buffer", &buffer));
1234
1235    sp<AMessage> notifyConsumed;
1236    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
1237
1238    QueueEntry entry;
1239    entry.mBuffer = buffer;
1240    entry.mNotifyConsumed = notifyConsumed;
1241    entry.mOffset = 0;
1242    entry.mFinalResult = OK;
1243    entry.mBufferOrdinal = ++mTotalBuffersQueued;
1244
1245    if (audio) {
1246        Mutex::Autolock autoLock(mLock);
1247        mAudioQueue.push_back(entry);
1248        postDrainAudioQueue_l();
1249    } else {
1250        mVideoQueue.push_back(entry);
1251        postDrainVideoQueue();
1252    }
1253
1254    Mutex::Autolock autoLock(mLock);
1255    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
1256        return;
1257    }
1258
1259    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
1260    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
1261
1262    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
1263        // EOS signalled on either queue.
1264        syncQueuesDone_l();
1265        return;
1266    }
1267
1268    int64_t firstAudioTimeUs;
1269    int64_t firstVideoTimeUs;
1270    CHECK(firstAudioBuffer->meta()
1271            ->findInt64("timeUs", &firstAudioTimeUs));
1272    CHECK(firstVideoBuffer->meta()
1273            ->findInt64("timeUs", &firstVideoTimeUs));
1274
1275    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
1276
1277    ALOGV("queueDiff = %.2f secs", diff / 1E6);
1278
1279    if (diff > 100000ll) {
1280        // Audio data starts More than 0.1 secs before video.
1281        // Drop some audio.
1282
1283        (*mAudioQueue.begin()).mNotifyConsumed->post();
1284        mAudioQueue.erase(mAudioQueue.begin());
1285        return;
1286    }
1287
1288    syncQueuesDone_l();
1289}
1290
1291void NuPlayer::Renderer::syncQueuesDone_l() {
1292    if (!mSyncQueues) {
1293        return;
1294    }
1295
1296    mSyncQueues = false;
1297
1298    if (!mAudioQueue.empty()) {
1299        postDrainAudioQueue_l();
1300    }
1301
1302    if (!mVideoQueue.empty()) {
1303        mLock.unlock();
1304        postDrainVideoQueue();
1305        mLock.lock();
1306    }
1307}
1308
1309void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1310    int32_t audio;
1311    CHECK(msg->findInt32("audio", &audio));
1312
1313    if (dropBufferIfStale(audio, msg)) {
1314        return;
1315    }
1316
1317    int32_t finalResult;
1318    CHECK(msg->findInt32("finalResult", &finalResult));
1319
1320    QueueEntry entry;
1321    entry.mOffset = 0;
1322    entry.mFinalResult = finalResult;
1323
1324    if (audio) {
1325        Mutex::Autolock autoLock(mLock);
1326        if (mAudioQueue.empty() && mSyncQueues) {
1327            syncQueuesDone_l();
1328        }
1329        mAudioQueue.push_back(entry);
1330        postDrainAudioQueue_l();
1331    } else {
1332        if (mVideoQueue.empty() && getSyncQueues()) {
1333            Mutex::Autolock autoLock(mLock);
1334            syncQueuesDone_l();
1335        }
1336        mVideoQueue.push_back(entry);
1337        postDrainVideoQueue();
1338    }
1339}
1340
1341void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
1342    int32_t audio, notifyComplete;
1343    CHECK(msg->findInt32("audio", &audio));
1344
1345    {
1346        Mutex::Autolock autoLock(mLock);
1347        if (audio) {
1348            notifyComplete = mNotifyCompleteAudio;
1349            mNotifyCompleteAudio = false;
1350            mLastAudioMediaTimeUs = -1;
1351        } else {
1352            notifyComplete = mNotifyCompleteVideo;
1353            mNotifyCompleteVideo = false;
1354        }
1355
1356        // If we're currently syncing the queues, i.e. dropping audio while
1357        // aligning the first audio/video buffer times and only one of the
1358        // two queues has data, we may starve that queue by not requesting
1359        // more buffers from the decoder. If the other source then encounters
1360        // a discontinuity that leads to flushing, we'll never find the
1361        // corresponding discontinuity on the other queue.
1362        // Therefore we'll stop syncing the queues if at least one of them
1363        // is flushed.
1364        syncQueuesDone_l();
1365        clearAnchorTime_l();
1366    }
1367
1368    ALOGV("flushing %s", audio ? "audio" : "video");
1369    if (audio) {
1370        {
1371            Mutex::Autolock autoLock(mLock);
1372            flushQueue(&mAudioQueue);
1373
1374            ++mAudioDrainGeneration;
1375            ++mAudioEOSGeneration;
1376            prepareForMediaRenderingStart_l();
1377
1378            // the frame count will be reset after flush.
1379            clearAudioFirstAnchorTime_l();
1380        }
1381
1382        mDrainAudioQueuePending = false;
1383
1384        if (offloadingAudio()) {
1385            mAudioSink->pause();
1386            mAudioSink->flush();
1387            if (!mPaused) {
1388                mAudioSink->start();
1389            }
1390        } else {
1391            mAudioSink->pause();
1392            mAudioSink->flush();
1393            // Call stop() to signal to the AudioSink to completely fill the
1394            // internal buffer before resuming playback.
1395            // FIXME: this is ignored after flush().
1396            mAudioSink->stop();
1397            if (mPaused) {
1398                // Race condition: if renderer is paused and audio sink is stopped,
1399                // we need to make sure that the audio track buffer fully drains
1400                // before delivering data.
1401                // FIXME: remove this if we can detect if stop() is complete.
1402                const int delayUs = 2 * 50 * 1000; // (2 full mixer thread cycles at 50ms)
1403                mPauseDrainAudioAllowedUs = ALooper::GetNowUs() + delayUs;
1404            } else {
1405                mAudioSink->start();
1406            }
1407            mNumFramesWritten = 0;
1408        }
1409    } else {
1410        flushQueue(&mVideoQueue);
1411
1412        mDrainVideoQueuePending = false;
1413
1414        if (mVideoScheduler != NULL) {
1415            mVideoScheduler->restart();
1416        }
1417
1418        Mutex::Autolock autoLock(mLock);
1419        ++mVideoDrainGeneration;
1420        prepareForMediaRenderingStart_l();
1421    }
1422
1423    mVideoSampleReceived = false;
1424
1425    if (notifyComplete) {
1426        notifyFlushComplete(audio);
1427    }
1428}
1429
1430void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1431    while (!queue->empty()) {
1432        QueueEntry *entry = &*queue->begin();
1433
1434        if (entry->mBuffer != NULL) {
1435            entry->mNotifyConsumed->post();
1436        }
1437
1438        queue->erase(queue->begin());
1439        entry = NULL;
1440    }
1441}
1442
1443void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1444    sp<AMessage> notify = mNotify->dup();
1445    notify->setInt32("what", kWhatFlushComplete);
1446    notify->setInt32("audio", static_cast<int32_t>(audio));
1447    notify->post();
1448}
1449
1450bool NuPlayer::Renderer::dropBufferIfStale(
1451        bool audio, const sp<AMessage> &msg) {
1452    int32_t queueGeneration;
1453    CHECK(msg->findInt32("queueGeneration", &queueGeneration));
1454
1455    if (queueGeneration == getQueueGeneration(audio)) {
1456        return false;
1457    }
1458
1459    sp<AMessage> notifyConsumed;
1460    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1461        notifyConsumed->post();
1462    }
1463
1464    return true;
1465}
1466
1467void NuPlayer::Renderer::onAudioSinkChanged() {
1468    if (offloadingAudio()) {
1469        return;
1470    }
1471    CHECK(!mDrainAudioQueuePending);
1472    mNumFramesWritten = 0;
1473    {
1474        Mutex::Autolock autoLock(mLock);
1475        mAnchorNumFramesWritten = -1;
1476    }
1477    uint32_t written;
1478    if (mAudioSink->getFramesWritten(&written) == OK) {
1479        mNumFramesWritten = written;
1480    }
1481}
1482
1483void NuPlayer::Renderer::onDisableOffloadAudio() {
1484    Mutex::Autolock autoLock(mLock);
1485    mFlags &= ~FLAG_OFFLOAD_AUDIO;
1486    ++mAudioDrainGeneration;
1487    if (mAudioRenderingStartGeneration != -1) {
1488        prepareForMediaRenderingStart_l();
1489    }
1490}
1491
1492void NuPlayer::Renderer::onEnableOffloadAudio() {
1493    Mutex::Autolock autoLock(mLock);
1494    mFlags |= FLAG_OFFLOAD_AUDIO;
1495    ++mAudioDrainGeneration;
1496    if (mAudioRenderingStartGeneration != -1) {
1497        prepareForMediaRenderingStart_l();
1498    }
1499}
1500
1501void NuPlayer::Renderer::onPause() {
1502    if (mPaused) {
1503        return;
1504    }
1505
1506    {
1507        Mutex::Autolock autoLock(mLock);
1508        // we do not increment audio drain generation so that we fill audio buffer during pause.
1509        ++mVideoDrainGeneration;
1510        prepareForMediaRenderingStart_l();
1511        mPaused = true;
1512        mMediaClock->setPlaybackRate(0.0);
1513    }
1514
1515    mDrainAudioQueuePending = false;
1516    mDrainVideoQueuePending = false;
1517
1518    if (mHasAudio) {
1519        mAudioSink->pause();
1520        startAudioOffloadPauseTimeout();
1521    }
1522
1523    ALOGV("now paused audio queue has %zu entries, video has %zu entries",
1524          mAudioQueue.size(), mVideoQueue.size());
1525}
1526
1527void NuPlayer::Renderer::onResume() {
1528    if (!mPaused) {
1529        return;
1530    }
1531
1532    if (mHasAudio) {
1533        cancelAudioOffloadPauseTimeout();
1534        status_t err = mAudioSink->start();
1535        if (err != OK) {
1536            notifyAudioTearDown();
1537        }
1538    }
1539
1540    {
1541        Mutex::Autolock autoLock(mLock);
1542        mPaused = false;
1543        // rendering started message may have been delayed if we were paused.
1544        if (mRenderingDataDelivered) {
1545            notifyIfMediaRenderingStarted_l();
1546        }
1547        // configure audiosink as we did not do it when pausing
1548        if (mAudioSink != NULL && mAudioSink->ready()) {
1549            mAudioSink->setPlaybackRate(mPlaybackSettings);
1550        }
1551
1552        mMediaClock->setPlaybackRate(mPlaybackRate);
1553
1554        if (!mAudioQueue.empty()) {
1555            postDrainAudioQueue_l();
1556        }
1557    }
1558
1559    if (!mVideoQueue.empty()) {
1560        postDrainVideoQueue();
1561    }
1562}
1563
1564void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1565    if (mVideoScheduler == NULL) {
1566        mVideoScheduler = new VideoFrameScheduler();
1567    }
1568    mVideoScheduler->init(fps);
1569}
1570
1571int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
1572    Mutex::Autolock autoLock(mLock);
1573    return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
1574}
1575
1576int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
1577    Mutex::Autolock autoLock(mLock);
1578    return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
1579}
1580
1581bool NuPlayer::Renderer::getSyncQueues() {
1582    Mutex::Autolock autoLock(mLock);
1583    return mSyncQueues;
1584}
1585
1586// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs()
1587// as it acquires locks and may query the audio driver.
1588//
1589// Some calls could conceivably retrieve extrapolated data instead of
1590// accessing getTimestamp() or getPosition() every time a data buffer with
1591// a media time is received.
1592//
1593// Calculate duration of played samples if played at normal rate (i.e., 1.0).
1594int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) {
1595    uint32_t numFramesPlayed;
1596    int64_t numFramesPlayedAt;
1597    AudioTimestamp ts;
1598    static const int64_t kStaleTimestamp100ms = 100000;
1599
1600    status_t res = mAudioSink->getTimestamp(ts);
1601    if (res == OK) {                 // case 1: mixing audio tracks and offloaded tracks.
1602        numFramesPlayed = ts.mPosition;
1603        numFramesPlayedAt =
1604            ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
1605        const int64_t timestampAge = nowUs - numFramesPlayedAt;
1606        if (timestampAge > kStaleTimestamp100ms) {
1607            // This is an audio FIXME.
1608            // getTimestamp returns a timestamp which may come from audio mixing threads.
1609            // After pausing, the MixerThread may go idle, thus the mTime estimate may
1610            // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms,
1611            // the max latency should be about 25ms with an average around 12ms (to be verified).
1612            // For safety we use 100ms.
1613            ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)",
1614                    (long long)nowUs, (long long)numFramesPlayedAt);
1615            numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
1616        }
1617        //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1618    } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track
1619        numFramesPlayed = 0;
1620        numFramesPlayedAt = nowUs;
1621        //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
1622        //        numFramesPlayed, (long long)numFramesPlayedAt);
1623    } else {                         // case 3: transitory at new track or audio fast tracks.
1624        res = mAudioSink->getPosition(&numFramesPlayed);
1625        CHECK_EQ(res, (status_t)OK);
1626        numFramesPlayedAt = nowUs;
1627        numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */
1628        //ALOGD("getPosition: %u %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1629    }
1630
1631    //CHECK_EQ(numFramesPlayed & (1 << 31), 0);  // can't be negative until 12.4 hrs, test
1632    int64_t durationUs = getDurationUsIfPlayedAtSampleRate(numFramesPlayed)
1633            + nowUs - numFramesPlayedAt;
1634    if (durationUs < 0) {
1635        // Occurs when numFramesPlayed position is very small and the following:
1636        // (1) In case 1, the time nowUs is computed before getTimestamp() is called and
1637        //     numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed.
1638        // (2) In case 3, using getPosition and adding mAudioSink->latency() to
1639        //     numFramesPlayedAt, by a time amount greater than numFramesPlayed.
1640        //
1641        // Both of these are transitory conditions.
1642        ALOGV("getPlayedOutAudioDurationUs: negative duration %lld set to zero", (long long)durationUs);
1643        durationUs = 0;
1644    }
1645    ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)",
1646            (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt);
1647    return durationUs;
1648}
1649
1650void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) {
1651    if (mAudioTornDown) {
1652        return;
1653    }
1654    mAudioTornDown = true;
1655
1656    int64_t currentPositionUs;
1657    sp<AMessage> notify = mNotify->dup();
1658    if (getCurrentPosition(&currentPositionUs) == OK) {
1659        notify->setInt64("positionUs", currentPositionUs);
1660    }
1661
1662    mAudioSink->stop();
1663    mAudioSink->flush();
1664
1665    notify->setInt32("what", kWhatAudioTearDown);
1666    notify->setInt32("reason", reason);
1667    notify->post();
1668}
1669
1670void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1671    if (offloadingAudio()) {
1672        mWakeLock->acquire();
1673        sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
1674        msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
1675        msg->post(kOffloadPauseMaxUs);
1676    }
1677}
1678
1679void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
1680    if (offloadingAudio()) {
1681        mWakeLock->release(true);
1682        ++mAudioOffloadPauseTimeoutGeneration;
1683    }
1684}
1685
1686status_t NuPlayer::Renderer::onOpenAudioSink(
1687        const sp<AMessage> &format,
1688        bool offloadOnly,
1689        bool hasVideo,
1690        uint32_t flags) {
1691    ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1692            offloadOnly, offloadingAudio());
1693    bool audioSinkChanged = false;
1694
1695    int32_t numChannels;
1696    CHECK(format->findInt32("channel-count", &numChannels));
1697
1698    int32_t channelMask;
1699    if (!format->findInt32("channel-mask", &channelMask)) {
1700        // signal to the AudioSink to derive the mask from count.
1701        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1702    }
1703
1704    int32_t sampleRate;
1705    CHECK(format->findInt32("sample-rate", &sampleRate));
1706
1707    if (offloadingAudio()) {
1708        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1709        AString mime;
1710        CHECK(format->findString("mime", &mime));
1711        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1712
1713        if (err != OK) {
1714            ALOGE("Couldn't map mime \"%s\" to a valid "
1715                    "audio_format", mime.c_str());
1716            onDisableOffloadAudio();
1717        } else {
1718            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1719                    mime.c_str(), audioFormat);
1720
1721            int avgBitRate = -1;
1722            format->findInt32("bit-rate", &avgBitRate);
1723
1724            int32_t aacProfile = -1;
1725            if (audioFormat == AUDIO_FORMAT_AAC
1726                    && format->findInt32("aac-profile", &aacProfile)) {
1727                // Redefine AAC format as per aac profile
1728                mapAACProfileToAudioFormat(
1729                        audioFormat,
1730                        aacProfile);
1731            }
1732
1733            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1734            offloadInfo.duration_us = -1;
1735            format->findInt64(
1736                    "durationUs", &offloadInfo.duration_us);
1737            offloadInfo.sample_rate = sampleRate;
1738            offloadInfo.channel_mask = channelMask;
1739            offloadInfo.format = audioFormat;
1740            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1741            offloadInfo.bit_rate = avgBitRate;
1742            offloadInfo.has_video = hasVideo;
1743            offloadInfo.is_streaming = true;
1744
1745            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1746                ALOGV("openAudioSink: no change in offload mode");
1747                // no change from previous configuration, everything ok.
1748                return OK;
1749            }
1750            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1751
1752            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1753            uint32_t offloadFlags = flags;
1754            offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1755            offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1756            audioSinkChanged = true;
1757            mAudioSink->close();
1758
1759            err = mAudioSink->open(
1760                    sampleRate,
1761                    numChannels,
1762                    (audio_channel_mask_t)channelMask,
1763                    audioFormat,
1764                    0 /* bufferCount - unused */,
1765                    &NuPlayer::Renderer::AudioSinkCallback,
1766                    this,
1767                    (audio_output_flags_t)offloadFlags,
1768                    &offloadInfo);
1769
1770            if (err == OK) {
1771                err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1772            }
1773
1774            if (err == OK) {
1775                // If the playback is offloaded to h/w, we pass
1776                // the HAL some metadata information.
1777                // We don't want to do this for PCM because it
1778                // will be going through the AudioFlinger mixer
1779                // before reaching the hardware.
1780                // TODO
1781                mCurrentOffloadInfo = offloadInfo;
1782                if (!mPaused) { // for preview mode, don't start if paused
1783                    err = mAudioSink->start();
1784                }
1785                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1786            }
1787            if (err != OK) {
1788                // Clean up, fall back to non offload mode.
1789                mAudioSink->close();
1790                onDisableOffloadAudio();
1791                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1792                ALOGV("openAudioSink: offload failed");
1793            } else {
1794                mUseAudioCallback = true;  // offload mode transfers data through callback
1795                ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
1796            }
1797        }
1798    }
1799    if (!offloadOnly && !offloadingAudio()) {
1800        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1801        uint32_t pcmFlags = flags;
1802        pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1803
1804        const PcmInfo info = {
1805                (audio_channel_mask_t)channelMask,
1806                (audio_output_flags_t)pcmFlags,
1807                AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat
1808                numChannels,
1809                sampleRate
1810        };
1811        if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
1812            ALOGV("openAudioSink: no change in pcm mode");
1813            // no change from previous configuration, everything ok.
1814            return OK;
1815        }
1816
1817        audioSinkChanged = true;
1818        mAudioSink->close();
1819        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1820        // Note: It is possible to set up the callback, but not use it to send audio data.
1821        // This requires a fix in AudioSink to explicitly specify the transfer mode.
1822        mUseAudioCallback = getUseAudioCallbackSetting();
1823        if (mUseAudioCallback) {
1824            ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
1825        }
1826
1827        // Compute the desired buffer size.
1828        // For callback mode, the amount of time before wakeup is about half the buffer size.
1829        const uint32_t frameCount =
1830                (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000;
1831
1832        // The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct
1833        // AudioSink. We don't want this when there's video because it will cause a video seek to
1834        // the previous I frame. But we do want this when there's only audio because it will give
1835        // NuPlayer a chance to switch from non-offload mode to offload mode.
1836        // So we only set doNotReconnect when there's no video.
1837        const bool doNotReconnect = !hasVideo;
1838        status_t err = mAudioSink->open(
1839                    sampleRate,
1840                    numChannels,
1841                    (audio_channel_mask_t)channelMask,
1842                    AUDIO_FORMAT_PCM_16_BIT,
1843                    0 /* bufferCount - unused */,
1844                    mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL,
1845                    mUseAudioCallback ? this : NULL,
1846                    (audio_output_flags_t)pcmFlags,
1847                    NULL,
1848                    doNotReconnect,
1849                    frameCount);
1850        if (err == OK) {
1851            err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1852        }
1853        if (err != OK) {
1854            ALOGW("openAudioSink: non offloaded open failed status: %d", err);
1855            mAudioSink->close();
1856            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1857            return err;
1858        }
1859        mCurrentPcmInfo = info;
1860        if (!mPaused) { // for preview mode, don't start if paused
1861            mAudioSink->start();
1862        }
1863    }
1864    if (audioSinkChanged) {
1865        onAudioSinkChanged();
1866    }
1867    mAudioTornDown = false;
1868    return OK;
1869}
1870
1871void NuPlayer::Renderer::onCloseAudioSink() {
1872    mAudioSink->close();
1873    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1874    mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1875}
1876
1877}  // namespace android
1878
1879