NuPlayerRenderer.cpp revision 3b9eb1f8629c6264d924ab7043f80d824cdd39e2 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include <cutils/properties.h>
24
25#include <media/stagefright/foundation/ABuffer.h>
26#include <media/stagefright/foundation/ADebug.h>
27#include <media/stagefright/foundation/AMessage.h>
28#include <media/stagefright/foundation/AUtils.h>
29#include <media/stagefright/MediaErrors.h>
30#include <media/stagefright/MetaData.h>
31#include <media/stagefright/Utils.h>
32
33#include <VideoFrameScheduler.h>
34
35#include <inttypes.h>
36
37namespace android {
38
39// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
40// is closed to allow the audio DSP to power down.
41static const int64_t kOffloadPauseMaxUs = 60000000ll;
42
43// static
44const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
45
46static bool sFrameAccurateAVsync = false;
47
48static void readProperties() {
49    char value[PROPERTY_VALUE_MAX];
50    if (property_get("persist.sys.media.avsync", value, NULL)) {
51        sFrameAccurateAVsync =
52            !strcmp("1", value) || !strcasecmp("true", value);
53    }
54}
55
56NuPlayer::Renderer::Renderer(
57        const sp<MediaPlayerBase::AudioSink> &sink,
58        const sp<AMessage> &notify,
59        uint32_t flags)
60    : mAudioSink(sink),
61      mNotify(notify),
62      mFlags(flags),
63      mNumFramesWritten(0),
64      mDrainAudioQueuePending(false),
65      mDrainVideoQueuePending(false),
66      mAudioQueueGeneration(0),
67      mVideoQueueGeneration(0),
68      mAudioFirstAnchorTimeMediaUs(-1),
69      mVideoAnchorTimeMediaUs(-1),
70      mVideoAnchorTimeRealUs(-1),
71      mVideoLateByUs(0ll),
72      mHasAudio(false),
73      mHasVideo(false),
74      mPauseStartedTimeRealUs(-1),
75      mFlushingAudio(false),
76      mFlushingVideo(false),
77      mSyncQueues(false),
78      mPaused(false),
79      mVideoSampleReceived(false),
80      mVideoRenderingStarted(false),
81      mVideoRenderingStartGeneration(0),
82      mAudioRenderingStartGeneration(0),
83      mAudioOffloadPauseTimeoutGeneration(0),
84      mAudioOffloadTornDown(false),
85      mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER) {
86    readProperties();
87}
88
89NuPlayer::Renderer::~Renderer() {
90    if (offloadingAudio()) {
91        mAudioSink->stop();
92        mAudioSink->flush();
93        mAudioSink->close();
94    }
95}
96
97void NuPlayer::Renderer::queueBuffer(
98        bool audio,
99        const sp<ABuffer> &buffer,
100        const sp<AMessage> &notifyConsumed) {
101    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
102    msg->setInt32("audio", static_cast<int32_t>(audio));
103    msg->setBuffer("buffer", buffer);
104    msg->setMessage("notifyConsumed", notifyConsumed);
105    msg->post();
106}
107
108void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
109    CHECK_NE(finalResult, (status_t)OK);
110
111    sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
112    msg->setInt32("audio", static_cast<int32_t>(audio));
113    msg->setInt32("finalResult", finalResult);
114    msg->post();
115}
116
117void NuPlayer::Renderer::flush(bool audio) {
118    {
119        Mutex::Autolock autoLock(mFlushLock);
120        if (audio) {
121            if (mFlushingAudio) {
122                return;
123            }
124            mFlushingAudio = true;
125        } else {
126            if (mFlushingVideo) {
127                return;
128            }
129            mFlushingVideo = true;
130        }
131    }
132
133    sp<AMessage> msg = new AMessage(kWhatFlush, id());
134    msg->setInt32("audio", static_cast<int32_t>(audio));
135    msg->post();
136}
137
138void NuPlayer::Renderer::signalTimeDiscontinuity() {
139    Mutex::Autolock autoLock(mLock);
140    // CHECK(mAudioQueue.empty());
141    // CHECK(mVideoQueue.empty());
142    setAudioFirstAnchorTime(-1);
143    setVideoAnchorTime(-1, -1);
144    setVideoLateByUs(0);
145    mSyncQueues = false;
146}
147
148void NuPlayer::Renderer::signalAudioSinkChanged() {
149    (new AMessage(kWhatAudioSinkChanged, id()))->post();
150}
151
152void NuPlayer::Renderer::signalDisableOffloadAudio() {
153    (new AMessage(kWhatDisableOffloadAudio, id()))->post();
154}
155
156void NuPlayer::Renderer::pause() {
157    (new AMessage(kWhatPause, id()))->post();
158}
159
160void NuPlayer::Renderer::resume() {
161    (new AMessage(kWhatResume, id()))->post();
162}
163
164void NuPlayer::Renderer::setVideoFrameRate(float fps) {
165    sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, id());
166    msg->setFloat("frame-rate", fps);
167    msg->post();
168}
169
170status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
171    return getCurrentPosition(mediaUs, ALooper::GetNowUs());
172}
173
174status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs, int64_t nowUs) {
175    Mutex::Autolock autoLock(mTimeLock);
176    if (!mHasAudio && !mHasVideo) {
177        return NO_INIT;
178    }
179
180    int64_t positionUs = 0;
181    if (!mHasAudio) {
182        if (mVideoAnchorTimeMediaUs < 0) {
183            return NO_INIT;
184        }
185        positionUs = (nowUs - mVideoAnchorTimeRealUs) + mVideoAnchorTimeMediaUs;
186
187        if (mPauseStartedTimeRealUs != -1) {
188            positionUs -= (nowUs - mPauseStartedTimeRealUs);
189        }
190    } else {
191        if (mAudioFirstAnchorTimeMediaUs < 0) {
192            return NO_INIT;
193        }
194        positionUs = mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs);
195    }
196    *mediaUs = (positionUs <= 0) ? 0 : positionUs;
197    return OK;
198}
199
200void NuPlayer::Renderer::setHasMedia(bool audio) {
201    Mutex::Autolock autoLock(mTimeLock);
202    if (audio) {
203        mHasAudio = true;
204    } else {
205        mHasVideo = true;
206    }
207}
208
209void NuPlayer::Renderer::setAudioFirstAnchorTime(int64_t mediaUs) {
210    Mutex::Autolock autoLock(mTimeLock);
211    mAudioFirstAnchorTimeMediaUs = mediaUs;
212}
213
214void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded(int64_t mediaUs) {
215    Mutex::Autolock autoLock(mTimeLock);
216    if (mAudioFirstAnchorTimeMediaUs == -1) {
217        mAudioFirstAnchorTimeMediaUs = mediaUs;
218    }
219}
220
221void NuPlayer::Renderer::setVideoAnchorTime(int64_t mediaUs, int64_t realUs) {
222    Mutex::Autolock autoLock(mTimeLock);
223    mVideoAnchorTimeMediaUs = mediaUs;
224    mVideoAnchorTimeRealUs = realUs;
225}
226
227void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
228    Mutex::Autolock autoLock(mTimeLock);
229    mVideoLateByUs = lateUs;
230}
231
232int64_t NuPlayer::Renderer::getVideoLateByUs() {
233    Mutex::Autolock autoLock(mTimeLock);
234    return mVideoLateByUs;
235}
236
237void NuPlayer::Renderer::setPauseStartedTimeRealUs(int64_t realUs) {
238    Mutex::Autolock autoLock(mTimeLock);
239    mPauseStartedTimeRealUs = realUs;
240}
241
242bool NuPlayer::Renderer::openAudioSink(
243        const sp<AMessage> &format,
244        bool offloadOnly,
245        bool hasVideo,
246        uint32_t flags) {
247    sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, id());
248    msg->setMessage("format", format);
249    msg->setInt32("offload-only", offloadOnly);
250    msg->setInt32("has-video", hasVideo);
251    msg->setInt32("flags", flags);
252
253    sp<AMessage> response;
254    msg->postAndAwaitResponse(&response);
255
256    int32_t offload;
257    CHECK(response->findInt32("offload", &offload));
258    return (offload != 0);
259}
260
261void NuPlayer::Renderer::closeAudioSink() {
262    sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, id());
263
264    sp<AMessage> response;
265    msg->postAndAwaitResponse(&response);
266}
267
268void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
269    switch (msg->what()) {
270        case kWhatOpenAudioSink:
271        {
272            sp<AMessage> format;
273            CHECK(msg->findMessage("format", &format));
274
275            int32_t offloadOnly;
276            CHECK(msg->findInt32("offload-only", &offloadOnly));
277
278            int32_t hasVideo;
279            CHECK(msg->findInt32("has-video", &hasVideo));
280
281            uint32_t flags;
282            CHECK(msg->findInt32("flags", (int32_t *)&flags));
283
284            bool offload = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
285
286            sp<AMessage> response = new AMessage;
287            response->setInt32("offload", offload);
288
289            uint32_t replyID;
290            CHECK(msg->senderAwaitsResponse(&replyID));
291            response->postReply(replyID);
292
293            break;
294        }
295
296        case kWhatCloseAudioSink:
297        {
298            uint32_t replyID;
299            CHECK(msg->senderAwaitsResponse(&replyID));
300
301            onCloseAudioSink();
302
303            sp<AMessage> response = new AMessage;
304            response->postReply(replyID);
305            break;
306        }
307
308        case kWhatStopAudioSink:
309        {
310            mAudioSink->stop();
311            break;
312        }
313
314        case kWhatDrainAudioQueue:
315        {
316            int32_t generation;
317            CHECK(msg->findInt32("generation", &generation));
318            if (generation != mAudioQueueGeneration) {
319                break;
320            }
321
322            mDrainAudioQueuePending = false;
323
324            if (onDrainAudioQueue()) {
325                uint32_t numFramesPlayed;
326                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
327                         (status_t)OK);
328
329                uint32_t numFramesPendingPlayout =
330                    mNumFramesWritten - numFramesPlayed;
331
332                // This is how long the audio sink will have data to
333                // play back.
334                int64_t delayUs =
335                    mAudioSink->msecsPerFrame()
336                        * numFramesPendingPlayout * 1000ll;
337
338                // Let's give it more data after about half that time
339                // has elapsed.
340                // kWhatDrainAudioQueue is used for non-offloading mode,
341                // and mLock is used only for offloading mode. Therefore,
342                // no need to acquire mLock here.
343                postDrainAudioQueue_l(delayUs / 2);
344            }
345            break;
346        }
347
348        case kWhatDrainVideoQueue:
349        {
350            int32_t generation;
351            CHECK(msg->findInt32("generation", &generation));
352            if (generation != mVideoQueueGeneration) {
353                break;
354            }
355
356            mDrainVideoQueuePending = false;
357
358            onDrainVideoQueue();
359
360            postDrainVideoQueue();
361            break;
362        }
363
364        case kWhatQueueBuffer:
365        {
366            onQueueBuffer(msg);
367            break;
368        }
369
370        case kWhatQueueEOS:
371        {
372            onQueueEOS(msg);
373            break;
374        }
375
376        case kWhatFlush:
377        {
378            onFlush(msg);
379            break;
380        }
381
382        case kWhatAudioSinkChanged:
383        {
384            onAudioSinkChanged();
385            break;
386        }
387
388        case kWhatDisableOffloadAudio:
389        {
390            onDisableOffloadAudio();
391            break;
392        }
393
394        case kWhatPause:
395        {
396            onPause();
397            break;
398        }
399
400        case kWhatResume:
401        {
402            onResume();
403            break;
404        }
405
406        case kWhatSetVideoFrameRate:
407        {
408            float fps;
409            CHECK(msg->findFloat("frame-rate", &fps));
410            onSetVideoFrameRate(fps);
411            break;
412        }
413
414        case kWhatAudioOffloadTearDown:
415        {
416            onAudioOffloadTearDown(kDueToError);
417            break;
418        }
419
420        case kWhatAudioOffloadPauseTimeout:
421        {
422            int32_t generation;
423            CHECK(msg->findInt32("generation", &generation));
424            if (generation != mAudioOffloadPauseTimeoutGeneration) {
425                break;
426            }
427            ALOGV("Audio Offload tear down due to pause timeout.");
428            onAudioOffloadTearDown(kDueToTimeout);
429            break;
430        }
431
432        default:
433            TRESPASS();
434            break;
435    }
436}
437
438void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
439    if (mDrainAudioQueuePending || mSyncQueues || mPaused
440            || offloadingAudio()) {
441        return;
442    }
443
444    if (mAudioQueue.empty()) {
445        return;
446    }
447
448    mDrainAudioQueuePending = true;
449    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
450    msg->setInt32("generation", mAudioQueueGeneration);
451    msg->post(delayUs);
452}
453
454void NuPlayer::Renderer::prepareForMediaRenderingStart() {
455    mAudioRenderingStartGeneration = mAudioQueueGeneration;
456    mVideoRenderingStartGeneration = mVideoQueueGeneration;
457}
458
459void NuPlayer::Renderer::notifyIfMediaRenderingStarted() {
460    if (mVideoRenderingStartGeneration == mVideoQueueGeneration &&
461        mAudioRenderingStartGeneration == mAudioQueueGeneration) {
462        mVideoRenderingStartGeneration = -1;
463        mAudioRenderingStartGeneration = -1;
464
465        sp<AMessage> notify = mNotify->dup();
466        notify->setInt32("what", kWhatMediaRenderingStart);
467        notify->post();
468    }
469}
470
471// static
472size_t NuPlayer::Renderer::AudioSinkCallback(
473        MediaPlayerBase::AudioSink * /* audioSink */,
474        void *buffer,
475        size_t size,
476        void *cookie,
477        MediaPlayerBase::AudioSink::cb_event_t event) {
478    NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
479
480    switch (event) {
481        case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
482        {
483            return me->fillAudioBuffer(buffer, size);
484            break;
485        }
486
487        case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
488        {
489            me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
490            break;
491        }
492
493        case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
494        {
495            me->notifyAudioOffloadTearDown();
496            break;
497        }
498    }
499
500    return 0;
501}
502
503size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
504    Mutex::Autolock autoLock(mLock);
505
506    if (!offloadingAudio() || mPaused) {
507        return 0;
508    }
509
510    bool hasEOS = false;
511
512    size_t sizeCopied = 0;
513    bool firstEntry = true;
514    while (sizeCopied < size && !mAudioQueue.empty()) {
515        QueueEntry *entry = &*mAudioQueue.begin();
516
517        if (entry->mBuffer == NULL) { // EOS
518            hasEOS = true;
519            mAudioQueue.erase(mAudioQueue.begin());
520            entry = NULL;
521            break;
522        }
523
524        if (firstEntry && entry->mOffset == 0) {
525            firstEntry = false;
526            int64_t mediaTimeUs;
527            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
528            ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
529            setAudioFirstAnchorTimeIfNeeded(mediaTimeUs);
530        }
531
532        size_t copy = entry->mBuffer->size() - entry->mOffset;
533        size_t sizeRemaining = size - sizeCopied;
534        if (copy > sizeRemaining) {
535            copy = sizeRemaining;
536        }
537
538        memcpy((char *)buffer + sizeCopied,
539               entry->mBuffer->data() + entry->mOffset,
540               copy);
541
542        entry->mOffset += copy;
543        if (entry->mOffset == entry->mBuffer->size()) {
544            entry->mNotifyConsumed->post();
545            mAudioQueue.erase(mAudioQueue.begin());
546            entry = NULL;
547        }
548        sizeCopied += copy;
549        notifyIfMediaRenderingStarted();
550    }
551
552    if (hasEOS) {
553        (new AMessage(kWhatStopAudioSink, id()))->post();
554    }
555
556    return sizeCopied;
557}
558
559bool NuPlayer::Renderer::onDrainAudioQueue() {
560    uint32_t numFramesPlayed;
561    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
562        return false;
563    }
564
565    ssize_t numFramesAvailableToWrite =
566        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
567
568#if 0
569    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
570        ALOGI("audio sink underrun");
571    } else {
572        ALOGV("audio queue has %d frames left to play",
573             mAudioSink->frameCount() - numFramesAvailableToWrite);
574    }
575#endif
576
577    size_t numBytesAvailableToWrite =
578        numFramesAvailableToWrite * mAudioSink->frameSize();
579
580    while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) {
581        QueueEntry *entry = &*mAudioQueue.begin();
582
583        if (entry->mBuffer == NULL) {
584            // EOS
585            int64_t postEOSDelayUs = 0;
586            if (mAudioSink->needsTrailingPadding()) {
587                postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
588            }
589            notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
590
591            mAudioQueue.erase(mAudioQueue.begin());
592            entry = NULL;
593            return false;
594        }
595
596        if (entry->mOffset == 0) {
597            int64_t mediaTimeUs;
598            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
599            ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
600
601            setAudioFirstAnchorTimeIfNeeded(mediaTimeUs);
602        }
603
604        size_t copy = entry->mBuffer->size() - entry->mOffset;
605        if (copy > numBytesAvailableToWrite) {
606            copy = numBytesAvailableToWrite;
607        }
608
609        ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, copy);
610        if (written < 0) {
611            // An error in AudioSink write is fatal here.
612            LOG_ALWAYS_FATAL("AudioSink write error(%zd) when writing %zu bytes", written, copy);
613        }
614
615        entry->mOffset += written;
616        if (entry->mOffset == entry->mBuffer->size()) {
617            entry->mNotifyConsumed->post();
618            mAudioQueue.erase(mAudioQueue.begin());
619
620            entry = NULL;
621        }
622
623        numBytesAvailableToWrite -= written;
624        size_t copiedFrames = written / mAudioSink->frameSize();
625        mNumFramesWritten += copiedFrames;
626
627        notifyIfMediaRenderingStarted();
628
629        if (written != (ssize_t)copy) {
630            // A short count was received from AudioSink::write()
631            //
632            // AudioSink write should block until exactly the number of bytes are delivered.
633            // But it may return with a short count (without an error) when:
634            //
635            // 1) Size to be copied is not a multiple of the frame size. We consider this fatal.
636            // 2) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
637
638            // (Case 1)
639            // Must be a multiple of the frame size.  If it is not a multiple of a frame size, it
640            // needs to fail, as we should not carry over fractional frames between calls.
641            CHECK_EQ(copy % mAudioSink->frameSize(), 0);
642
643            // (Case 2)
644            // Return early to the caller.
645            // Beware of calling immediately again as this may busy-loop if you are not careful.
646            ALOGW("AudioSink write short frame count %zd < %zu", written, copy);
647            break;
648        }
649    }
650    return !mAudioQueue.empty();
651}
652
653int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
654    int64_t writtenAudioDurationUs =
655        mNumFramesWritten * 1000LL * mAudioSink->msecsPerFrame();
656    return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs);
657}
658
659int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
660    int64_t currentPositionUs;
661    if (getCurrentPosition(&currentPositionUs, nowUs) != OK) {
662        currentPositionUs = 0;
663    }
664    return (mediaTimeUs - currentPositionUs) + nowUs;
665}
666
667void NuPlayer::Renderer::postDrainVideoQueue() {
668    if (mDrainVideoQueuePending
669            || mSyncQueues
670            || (mPaused && mVideoSampleReceived)) {
671        return;
672    }
673
674    if (mVideoQueue.empty()) {
675        return;
676    }
677
678    QueueEntry &entry = *mVideoQueue.begin();
679
680    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
681    msg->setInt32("generation", mVideoQueueGeneration);
682
683    if (entry.mBuffer == NULL) {
684        // EOS doesn't carry a timestamp.
685        msg->post();
686        mDrainVideoQueuePending = true;
687        return;
688    }
689
690    int64_t delayUs;
691    int64_t nowUs = ALooper::GetNowUs();
692    int64_t realTimeUs;
693    if (mFlags & FLAG_REAL_TIME) {
694        int64_t mediaTimeUs;
695        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
696        realTimeUs = mediaTimeUs;
697    } else {
698        int64_t mediaTimeUs;
699        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
700
701        if (mVideoAnchorTimeMediaUs < 0) {
702            setVideoAnchorTime(mediaTimeUs, nowUs);
703            realTimeUs = nowUs;
704        } else {
705            realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
706        }
707    }
708
709    realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
710    int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
711
712    delayUs = realTimeUs - nowUs;
713
714    ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
715    // post 2 display refreshes before rendering is due
716    // FIXME currently this increases power consumption, so unless frame-accurate
717    // AV sync is requested, post closer to required render time (at 0.63 vsyncs)
718    if (!sFrameAccurateAVsync) {
719        twoVsyncsUs >>= 4;
720    }
721    msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
722
723    mDrainVideoQueuePending = true;
724}
725
726void NuPlayer::Renderer::onDrainVideoQueue() {
727    if (mVideoQueue.empty()) {
728        return;
729    }
730
731    QueueEntry *entry = &*mVideoQueue.begin();
732
733    if (entry->mBuffer == NULL) {
734        // EOS
735
736        notifyEOS(false /* audio */, entry->mFinalResult);
737
738        mVideoQueue.erase(mVideoQueue.begin());
739        entry = NULL;
740
741        setVideoLateByUs(0);
742        return;
743    }
744
745    int64_t nowUs = -1;
746    int64_t realTimeUs;
747    if (mFlags & FLAG_REAL_TIME) {
748        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
749    } else {
750        int64_t mediaTimeUs;
751        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
752
753        nowUs = ALooper::GetNowUs();
754        realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
755    }
756
757    bool tooLate = false;
758
759    if (!mPaused) {
760        if (nowUs == -1) {
761            nowUs = ALooper::GetNowUs();
762        }
763        setVideoLateByUs(nowUs - realTimeUs);
764        tooLate = (mVideoLateByUs > 40000);
765
766        if (tooLate) {
767            ALOGV("video late by %lld us (%.2f secs)",
768                 mVideoLateByUs, mVideoLateByUs / 1E6);
769        } else {
770            ALOGV("rendering video at media time %.2f secs",
771                    (mFlags & FLAG_REAL_TIME ? realTimeUs :
772                    (realTimeUs + mVideoAnchorTimeMediaUs - mVideoAnchorTimeRealUs)) / 1E6);
773        }
774    } else {
775        setVideoLateByUs(0);
776        if (!mVideoSampleReceived) {
777            // This will ensure that the first frame after a flush won't be used as anchor
778            // when renderer is in paused state, because resume can happen any time after seek.
779            setVideoAnchorTime(-1, -1);
780        }
781    }
782
783    entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
784    entry->mNotifyConsumed->setInt32("render", !tooLate);
785    entry->mNotifyConsumed->post();
786    mVideoQueue.erase(mVideoQueue.begin());
787    entry = NULL;
788
789    mVideoSampleReceived = true;
790
791    if (!mPaused) {
792        if (!mVideoRenderingStarted) {
793            mVideoRenderingStarted = true;
794            notifyVideoRenderingStart();
795        }
796        notifyIfMediaRenderingStarted();
797    }
798}
799
800void NuPlayer::Renderer::notifyVideoRenderingStart() {
801    sp<AMessage> notify = mNotify->dup();
802    notify->setInt32("what", kWhatVideoRenderingStart);
803    notify->post();
804}
805
806void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
807    sp<AMessage> notify = mNotify->dup();
808    notify->setInt32("what", kWhatEOS);
809    notify->setInt32("audio", static_cast<int32_t>(audio));
810    notify->setInt32("finalResult", finalResult);
811    notify->post(delayUs);
812}
813
814void NuPlayer::Renderer::notifyAudioOffloadTearDown() {
815    (new AMessage(kWhatAudioOffloadTearDown, id()))->post();
816}
817
818void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
819    int32_t audio;
820    CHECK(msg->findInt32("audio", &audio));
821
822    setHasMedia(audio);
823
824    if (mHasVideo) {
825        if (mVideoScheduler == NULL) {
826            mVideoScheduler = new VideoFrameScheduler();
827            mVideoScheduler->init();
828        }
829    }
830
831    if (dropBufferWhileFlushing(audio, msg)) {
832        return;
833    }
834
835    sp<ABuffer> buffer;
836    CHECK(msg->findBuffer("buffer", &buffer));
837
838    sp<AMessage> notifyConsumed;
839    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
840
841    QueueEntry entry;
842    entry.mBuffer = buffer;
843    entry.mNotifyConsumed = notifyConsumed;
844    entry.mOffset = 0;
845    entry.mFinalResult = OK;
846
847    if (audio) {
848        Mutex::Autolock autoLock(mLock);
849        mAudioQueue.push_back(entry);
850        postDrainAudioQueue_l();
851    } else {
852        mVideoQueue.push_back(entry);
853        postDrainVideoQueue();
854    }
855
856    Mutex::Autolock autoLock(mLock);
857    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
858        return;
859    }
860
861    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
862    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
863
864    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
865        // EOS signalled on either queue.
866        syncQueuesDone_l();
867        return;
868    }
869
870    int64_t firstAudioTimeUs;
871    int64_t firstVideoTimeUs;
872    CHECK(firstAudioBuffer->meta()
873            ->findInt64("timeUs", &firstAudioTimeUs));
874    CHECK(firstVideoBuffer->meta()
875            ->findInt64("timeUs", &firstVideoTimeUs));
876
877    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
878
879    ALOGV("queueDiff = %.2f secs", diff / 1E6);
880
881    if (diff > 100000ll) {
882        // Audio data starts More than 0.1 secs before video.
883        // Drop some audio.
884
885        (*mAudioQueue.begin()).mNotifyConsumed->post();
886        mAudioQueue.erase(mAudioQueue.begin());
887        return;
888    }
889
890    syncQueuesDone_l();
891}
892
893void NuPlayer::Renderer::syncQueuesDone_l() {
894    if (!mSyncQueues) {
895        return;
896    }
897
898    mSyncQueues = false;
899
900    if (!mAudioQueue.empty()) {
901        postDrainAudioQueue_l();
902    }
903
904    if (!mVideoQueue.empty()) {
905        postDrainVideoQueue();
906    }
907}
908
909void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
910    int32_t audio;
911    CHECK(msg->findInt32("audio", &audio));
912
913    if (dropBufferWhileFlushing(audio, msg)) {
914        return;
915    }
916
917    int32_t finalResult;
918    CHECK(msg->findInt32("finalResult", &finalResult));
919
920    QueueEntry entry;
921    entry.mOffset = 0;
922    entry.mFinalResult = finalResult;
923
924    if (audio) {
925        Mutex::Autolock autoLock(mLock);
926        if (mAudioQueue.empty() && mSyncQueues) {
927            syncQueuesDone_l();
928        }
929        mAudioQueue.push_back(entry);
930        postDrainAudioQueue_l();
931    } else {
932        if (mVideoQueue.empty() && mSyncQueues) {
933            Mutex::Autolock autoLock(mLock);
934            syncQueuesDone_l();
935        }
936        mVideoQueue.push_back(entry);
937        postDrainVideoQueue();
938    }
939}
940
941void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
942    int32_t audio;
943    CHECK(msg->findInt32("audio", &audio));
944
945    {
946        Mutex::Autolock autoLock(mFlushLock);
947        if (audio) {
948            mFlushingAudio = false;
949        } else {
950            mFlushingVideo = false;
951        }
952    }
953
954    // If we're currently syncing the queues, i.e. dropping audio while
955    // aligning the first audio/video buffer times and only one of the
956    // two queues has data, we may starve that queue by not requesting
957    // more buffers from the decoder. If the other source then encounters
958    // a discontinuity that leads to flushing, we'll never find the
959    // corresponding discontinuity on the other queue.
960    // Therefore we'll stop syncing the queues if at least one of them
961    // is flushed.
962    {
963         Mutex::Autolock autoLock(mLock);
964         syncQueuesDone_l();
965         setPauseStartedTimeRealUs(-1);
966    }
967
968    ALOGV("flushing %s", audio ? "audio" : "video");
969    if (audio) {
970        {
971            Mutex::Autolock autoLock(mLock);
972            flushQueue(&mAudioQueue);
973
974            ++mAudioQueueGeneration;
975            prepareForMediaRenderingStart();
976
977            if (offloadingAudio()) {
978                setAudioFirstAnchorTime(-1);
979            }
980        }
981
982        mDrainAudioQueuePending = false;
983
984        if (offloadingAudio()) {
985            mAudioSink->pause();
986            mAudioSink->flush();
987            mAudioSink->start();
988        }
989    } else {
990        flushQueue(&mVideoQueue);
991
992        mDrainVideoQueuePending = false;
993        ++mVideoQueueGeneration;
994
995        if (mVideoScheduler != NULL) {
996            mVideoScheduler->restart();
997        }
998
999        prepareForMediaRenderingStart();
1000    }
1001
1002    mVideoSampleReceived = false;
1003    notifyFlushComplete(audio);
1004}
1005
1006void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1007    while (!queue->empty()) {
1008        QueueEntry *entry = &*queue->begin();
1009
1010        if (entry->mBuffer != NULL) {
1011            entry->mNotifyConsumed->post();
1012        }
1013
1014        queue->erase(queue->begin());
1015        entry = NULL;
1016    }
1017}
1018
1019void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1020    sp<AMessage> notify = mNotify->dup();
1021    notify->setInt32("what", kWhatFlushComplete);
1022    notify->setInt32("audio", static_cast<int32_t>(audio));
1023    notify->post();
1024}
1025
1026bool NuPlayer::Renderer::dropBufferWhileFlushing(
1027        bool audio, const sp<AMessage> &msg) {
1028    bool flushing = false;
1029
1030    {
1031        Mutex::Autolock autoLock(mFlushLock);
1032        if (audio) {
1033            flushing = mFlushingAudio;
1034        } else {
1035            flushing = mFlushingVideo;
1036        }
1037    }
1038
1039    if (!flushing) {
1040        return false;
1041    }
1042
1043    sp<AMessage> notifyConsumed;
1044    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1045        notifyConsumed->post();
1046    }
1047
1048    return true;
1049}
1050
1051void NuPlayer::Renderer::onAudioSinkChanged() {
1052    if (offloadingAudio()) {
1053        return;
1054    }
1055    CHECK(!mDrainAudioQueuePending);
1056    mNumFramesWritten = 0;
1057    uint32_t written;
1058    if (mAudioSink->getFramesWritten(&written) == OK) {
1059        mNumFramesWritten = written;
1060    }
1061}
1062
1063void NuPlayer::Renderer::onDisableOffloadAudio() {
1064    Mutex::Autolock autoLock(mLock);
1065    mFlags &= ~FLAG_OFFLOAD_AUDIO;
1066    ++mAudioQueueGeneration;
1067}
1068
1069void NuPlayer::Renderer::onPause() {
1070    if (mPaused) {
1071        ALOGW("Renderer::onPause() called while already paused!");
1072        return;
1073    }
1074    {
1075        Mutex::Autolock autoLock(mLock);
1076        ++mAudioQueueGeneration;
1077        ++mVideoQueueGeneration;
1078        prepareForMediaRenderingStart();
1079        mPaused = true;
1080        setPauseStartedTimeRealUs(ALooper::GetNowUs());
1081    }
1082
1083    mDrainAudioQueuePending = false;
1084    mDrainVideoQueuePending = false;
1085
1086    if (mHasAudio) {
1087        mAudioSink->pause();
1088        startAudioOffloadPauseTimeout();
1089    }
1090
1091    ALOGV("now paused audio queue has %d entries, video has %d entries",
1092          mAudioQueue.size(), mVideoQueue.size());
1093}
1094
1095void NuPlayer::Renderer::onResume() {
1096    readProperties();
1097
1098    if (!mPaused) {
1099        return;
1100    }
1101
1102    if (mHasAudio) {
1103        cancelAudioOffloadPauseTimeout();
1104        mAudioSink->start();
1105    }
1106
1107    Mutex::Autolock autoLock(mLock);
1108    mPaused = false;
1109    if (mPauseStartedTimeRealUs != -1) {
1110        int64_t newAnchorRealUs =
1111            mVideoAnchorTimeRealUs + ALooper::GetNowUs() - mPauseStartedTimeRealUs;
1112        setVideoAnchorTime(mVideoAnchorTimeMediaUs, newAnchorRealUs);
1113        setPauseStartedTimeRealUs(-1);
1114    }
1115
1116    if (!mAudioQueue.empty()) {
1117        postDrainAudioQueue_l();
1118    }
1119
1120    if (!mVideoQueue.empty()) {
1121        postDrainVideoQueue();
1122    }
1123}
1124
1125void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1126    if (mVideoScheduler == NULL) {
1127        mVideoScheduler = new VideoFrameScheduler();
1128    }
1129    mVideoScheduler->init(fps);
1130}
1131
1132// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs()
1133// as it acquires locks and may query the audio driver.
1134//
1135// Some calls could conceivably retrieve extrapolated data instead of
1136// accessing getTimestamp() or getPosition() every time a data buffer with
1137// a media time is received.
1138//
1139int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) {
1140    uint32_t numFramesPlayed;
1141    int64_t numFramesPlayedAt;
1142    AudioTimestamp ts;
1143    static const int64_t kStaleTimestamp100ms = 100000;
1144
1145    status_t res = mAudioSink->getTimestamp(ts);
1146    if (res == OK) {                 // case 1: mixing audio tracks and offloaded tracks.
1147        numFramesPlayed = ts.mPosition;
1148        numFramesPlayedAt =
1149            ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
1150        const int64_t timestampAge = nowUs - numFramesPlayedAt;
1151        if (timestampAge > kStaleTimestamp100ms) {
1152            // This is an audio FIXME.
1153            // getTimestamp returns a timestamp which may come from audio mixing threads.
1154            // After pausing, the MixerThread may go idle, thus the mTime estimate may
1155            // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms,
1156            // the max latency should be about 25ms with an average around 12ms (to be verified).
1157            // For safety we use 100ms.
1158            ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)",
1159                    (long long)nowUs, (long long)numFramesPlayedAt);
1160            numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
1161        }
1162        //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1163    } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track
1164        numFramesPlayed = 0;
1165        numFramesPlayedAt = nowUs;
1166        //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
1167        //        numFramesPlayed, (long long)numFramesPlayedAt);
1168    } else {                         // case 3: transitory at new track or audio fast tracks.
1169        res = mAudioSink->getPosition(&numFramesPlayed);
1170        CHECK_EQ(res, (status_t)OK);
1171        numFramesPlayedAt = nowUs;
1172        numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */
1173        //ALOGD("getPosition: %d %lld", numFramesPlayed, numFramesPlayedAt);
1174    }
1175
1176    // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
1177    //CHECK_EQ(numFramesPlayed & (1 << 31), 0);  // can't be negative until 12.4 hrs, test
1178    int64_t durationUs = (int32_t)numFramesPlayed * 1000LL * mAudioSink->msecsPerFrame()
1179            + nowUs - numFramesPlayedAt;
1180    if (durationUs < 0) {
1181        // Occurs when numFramesPlayed position is very small and the following:
1182        // (1) In case 1, the time nowUs is computed before getTimestamp() is called and
1183        //     numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed.
1184        // (2) In case 3, using getPosition and adding mAudioSink->latency() to
1185        //     numFramesPlayedAt, by a time amount greater than numFramesPlayed.
1186        //
1187        // Both of these are transitory conditions.
1188        ALOGV("getPlayedOutAudioDurationUs: negative duration %lld set to zero", (long long)durationUs);
1189        durationUs = 0;
1190    }
1191    ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)",
1192            (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt);
1193    return durationUs;
1194}
1195
1196void NuPlayer::Renderer::onAudioOffloadTearDown(AudioOffloadTearDownReason reason) {
1197    if (mAudioOffloadTornDown) {
1198        return;
1199    }
1200    mAudioOffloadTornDown = true;
1201
1202    int64_t currentPositionUs;
1203    if (getCurrentPosition(&currentPositionUs) != OK) {
1204        currentPositionUs = 0;
1205    }
1206
1207    mAudioSink->stop();
1208    mAudioSink->flush();
1209
1210    sp<AMessage> notify = mNotify->dup();
1211    notify->setInt32("what", kWhatAudioOffloadTearDown);
1212    notify->setInt64("positionUs", currentPositionUs);
1213    notify->setInt32("reason", reason);
1214    notify->post();
1215}
1216
1217void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1218    if (offloadingAudio()) {
1219        sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, id());
1220        msg->setInt32("generation", mAudioOffloadPauseTimeoutGeneration);
1221        msg->post(kOffloadPauseMaxUs);
1222    }
1223}
1224
1225void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
1226    if (offloadingAudio()) {
1227        ++mAudioOffloadPauseTimeoutGeneration;
1228    }
1229}
1230
1231bool NuPlayer::Renderer::onOpenAudioSink(
1232        const sp<AMessage> &format,
1233        bool offloadOnly,
1234        bool hasVideo,
1235        uint32_t flags) {
1236    ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1237            offloadOnly, offloadingAudio());
1238    bool audioSinkChanged = false;
1239
1240    int32_t numChannels;
1241    CHECK(format->findInt32("channel-count", &numChannels));
1242
1243    int32_t channelMask;
1244    if (!format->findInt32("channel-mask", &channelMask)) {
1245        // signal to the AudioSink to derive the mask from count.
1246        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1247    }
1248
1249    int32_t sampleRate;
1250    CHECK(format->findInt32("sample-rate", &sampleRate));
1251
1252    if (offloadingAudio()) {
1253        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1254        AString mime;
1255        CHECK(format->findString("mime", &mime));
1256        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1257
1258        if (err != OK) {
1259            ALOGE("Couldn't map mime \"%s\" to a valid "
1260                    "audio_format", mime.c_str());
1261            onDisableOffloadAudio();
1262        } else {
1263            ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1264                    mime.c_str(), audioFormat);
1265
1266            int avgBitRate = -1;
1267            format->findInt32("bit-rate", &avgBitRate);
1268
1269            int32_t aacProfile = -1;
1270            if (audioFormat == AUDIO_FORMAT_AAC
1271                    && format->findInt32("aac-profile", &aacProfile)) {
1272                // Redefine AAC format as per aac profile
1273                mapAACProfileToAudioFormat(
1274                        audioFormat,
1275                        aacProfile);
1276            }
1277
1278            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1279            offloadInfo.duration_us = -1;
1280            format->findInt64(
1281                    "durationUs", &offloadInfo.duration_us);
1282            offloadInfo.sample_rate = sampleRate;
1283            offloadInfo.channel_mask = channelMask;
1284            offloadInfo.format = audioFormat;
1285            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1286            offloadInfo.bit_rate = avgBitRate;
1287            offloadInfo.has_video = hasVideo;
1288            offloadInfo.is_streaming = true;
1289
1290            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1291                ALOGV("openAudioSink: no change in offload mode");
1292                // no change from previous configuration, everything ok.
1293                return offloadingAudio();
1294            }
1295            ALOGV("openAudioSink: try to open AudioSink in offload mode");
1296            flags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1297            flags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1298            audioSinkChanged = true;
1299            mAudioSink->close();
1300            err = mAudioSink->open(
1301                    sampleRate,
1302                    numChannels,
1303                    (audio_channel_mask_t)channelMask,
1304                    audioFormat,
1305                    8 /* bufferCount */,
1306                    &NuPlayer::Renderer::AudioSinkCallback,
1307                    this,
1308                    (audio_output_flags_t)flags,
1309                    &offloadInfo);
1310
1311            if (err == OK) {
1312                // If the playback is offloaded to h/w, we pass
1313                // the HAL some metadata information.
1314                // We don't want to do this for PCM because it
1315                // will be going through the AudioFlinger mixer
1316                // before reaching the hardware.
1317                // TODO
1318                mCurrentOffloadInfo = offloadInfo;
1319                err = mAudioSink->start();
1320                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1321            }
1322            if (err != OK) {
1323                // Clean up, fall back to non offload mode.
1324                mAudioSink->close();
1325                onDisableOffloadAudio();
1326                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1327                ALOGV("openAudioSink: offload failed");
1328            }
1329        }
1330    }
1331    if (!offloadOnly && !offloadingAudio()) {
1332        flags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1333        ALOGV("openAudioSink: open AudioSink in NON-offload mode");
1334
1335        audioSinkChanged = true;
1336        mAudioSink->close();
1337        mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1338        CHECK_EQ(mAudioSink->open(
1339                    sampleRate,
1340                    numChannels,
1341                    (audio_channel_mask_t)channelMask,
1342                    AUDIO_FORMAT_PCM_16_BIT,
1343                    8 /* bufferCount */,
1344                    NULL,
1345                    NULL,
1346                    (audio_output_flags_t)flags),
1347                 (status_t)OK);
1348        mAudioSink->start();
1349    }
1350    if (audioSinkChanged) {
1351        onAudioSinkChanged();
1352    }
1353
1354    return offloadingAudio();
1355}
1356
1357void NuPlayer::Renderer::onCloseAudioSink() {
1358    mAudioSink->close();
1359    mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1360}
1361
1362}  // namespace android
1363
1364