NuPlayerRenderer.cpp revision d6074f08cc37843e8c599b2ab4b16d87c3bc67eb 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include "SoftwareRenderer.h"
24
25#include <media/stagefright/foundation/ABuffer.h>
26#include <media/stagefright/foundation/ADebug.h>
27#include <media/stagefright/foundation/AMessage.h>
28
29namespace android {
30
31// static
32const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
33
34NuPlayer::Renderer::Renderer(
35        const sp<MediaPlayerBase::AudioSink> &sink,
36        const sp<AMessage> &notify,
37        uint32_t flags)
38    : mAudioSink(sink),
39      mSoftRenderer(NULL),
40      mNotify(notify),
41      mFlags(flags),
42      mNumFramesWritten(0),
43      mDrainAudioQueuePending(false),
44      mDrainVideoQueuePending(false),
45      mAudioQueueGeneration(0),
46      mVideoQueueGeneration(0),
47      mAnchorTimeMediaUs(-1),
48      mAnchorTimeRealUs(-1),
49      mFlushingAudio(false),
50      mFlushingVideo(false),
51      mHasAudio(false),
52      mHasVideo(false),
53      mSyncQueues(false),
54      mPaused(false),
55      mVideoRenderingStarted(false),
56      mVideoRenderingStartGeneration(0),
57      mAudioRenderingStartGeneration(0),
58      mLastPositionUpdateUs(-1ll),
59      mVideoLateByUs(0ll) {
60}
61
62NuPlayer::Renderer::~Renderer() {
63    delete mSoftRenderer;
64}
65
66void NuPlayer::Renderer::setSoftRenderer(SoftwareRenderer *softRenderer) {
67    delete mSoftRenderer;
68    mSoftRenderer = softRenderer;
69}
70
71void NuPlayer::Renderer::queueBuffer(
72        bool audio,
73        const sp<ABuffer> &buffer,
74        const sp<AMessage> &notifyConsumed) {
75    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
76    msg->setInt32("audio", static_cast<int32_t>(audio));
77    msg->setBuffer("buffer", buffer);
78    msg->setMessage("notifyConsumed", notifyConsumed);
79    msg->post();
80}
81
82void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
83    CHECK_NE(finalResult, (status_t)OK);
84
85    sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
86    msg->setInt32("audio", static_cast<int32_t>(audio));
87    msg->setInt32("finalResult", finalResult);
88    msg->post();
89}
90
91void NuPlayer::Renderer::flush(bool audio) {
92    {
93        Mutex::Autolock autoLock(mFlushLock);
94        if (audio) {
95            CHECK(!mFlushingAudio);
96            mFlushingAudio = true;
97        } else {
98            CHECK(!mFlushingVideo);
99            mFlushingVideo = true;
100        }
101    }
102
103    sp<AMessage> msg = new AMessage(kWhatFlush, id());
104    msg->setInt32("audio", static_cast<int32_t>(audio));
105    msg->post();
106}
107
108void NuPlayer::Renderer::signalTimeDiscontinuity() {
109    // CHECK(mAudioQueue.empty());
110    // CHECK(mVideoQueue.empty());
111    mAnchorTimeMediaUs = -1;
112    mAnchorTimeRealUs = -1;
113    mSyncQueues = false;
114}
115
116void NuPlayer::Renderer::pause() {
117    (new AMessage(kWhatPause, id()))->post();
118}
119
120void NuPlayer::Renderer::resume() {
121    (new AMessage(kWhatResume, id()))->post();
122}
123
124void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
125    switch (msg->what()) {
126        case kWhatDrainAudioQueue:
127        {
128            int32_t generation;
129            CHECK(msg->findInt32("generation", &generation));
130            if (generation != mAudioQueueGeneration) {
131                break;
132            }
133
134            mDrainAudioQueuePending = false;
135
136            if (onDrainAudioQueue()) {
137                uint32_t numFramesPlayed;
138                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
139                         (status_t)OK);
140
141                uint32_t numFramesPendingPlayout =
142                    mNumFramesWritten - numFramesPlayed;
143
144                // This is how long the audio sink will have data to
145                // play back.
146                int64_t delayUs =
147                    mAudioSink->msecsPerFrame()
148                        * numFramesPendingPlayout * 1000ll;
149
150                // Let's give it more data after about half that time
151                // has elapsed.
152                postDrainAudioQueue(delayUs / 2);
153            }
154            break;
155        }
156
157        case kWhatDrainVideoQueue:
158        {
159            int32_t generation;
160            CHECK(msg->findInt32("generation", &generation));
161            if (generation != mVideoQueueGeneration) {
162                break;
163            }
164
165            mDrainVideoQueuePending = false;
166
167            onDrainVideoQueue();
168
169            postDrainVideoQueue();
170            break;
171        }
172
173        case kWhatQueueBuffer:
174        {
175            onQueueBuffer(msg);
176            break;
177        }
178
179        case kWhatQueueEOS:
180        {
181            onQueueEOS(msg);
182            break;
183        }
184
185        case kWhatFlush:
186        {
187            onFlush(msg);
188            break;
189        }
190
191        case kWhatAudioSinkChanged:
192        {
193            onAudioSinkChanged();
194            break;
195        }
196
197        case kWhatPause:
198        {
199            onPause();
200            break;
201        }
202
203        case kWhatResume:
204        {
205            onResume();
206            break;
207        }
208
209        default:
210            TRESPASS();
211            break;
212    }
213}
214
215void NuPlayer::Renderer::postDrainAudioQueue(int64_t delayUs) {
216    if (mDrainAudioQueuePending || mSyncQueues || mPaused) {
217        return;
218    }
219
220    if (mAudioQueue.empty()) {
221        return;
222    }
223
224    mDrainAudioQueuePending = true;
225    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
226    msg->setInt32("generation", mAudioQueueGeneration);
227    msg->post(delayUs);
228}
229
230void NuPlayer::Renderer::signalAudioSinkChanged() {
231    (new AMessage(kWhatAudioSinkChanged, id()))->post();
232}
233
234void NuPlayer::Renderer::prepareForMediaRenderingStart() {
235    mAudioRenderingStartGeneration = mAudioQueueGeneration;
236    mVideoRenderingStartGeneration = mVideoQueueGeneration;
237}
238
239void NuPlayer::Renderer::notifyIfMediaRenderingStarted() {
240    if (mVideoRenderingStartGeneration == mVideoQueueGeneration &&
241        mAudioRenderingStartGeneration == mAudioQueueGeneration) {
242        mVideoRenderingStartGeneration = -1;
243        mAudioRenderingStartGeneration = -1;
244
245        sp<AMessage> notify = mNotify->dup();
246        notify->setInt32("what", kWhatMediaRenderingStart);
247        notify->post();
248    }
249}
250
251bool NuPlayer::Renderer::onDrainAudioQueue() {
252    uint32_t numFramesPlayed;
253    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
254        return false;
255    }
256
257    ssize_t numFramesAvailableToWrite =
258        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
259
260#if 0
261    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
262        ALOGI("audio sink underrun");
263    } else {
264        ALOGV("audio queue has %d frames left to play",
265             mAudioSink->frameCount() - numFramesAvailableToWrite);
266    }
267#endif
268
269    size_t numBytesAvailableToWrite =
270        numFramesAvailableToWrite * mAudioSink->frameSize();
271
272    while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) {
273        QueueEntry *entry = &*mAudioQueue.begin();
274
275        if (entry->mBuffer == NULL) {
276            // EOS
277
278            notifyEOS(true /* audio */, entry->mFinalResult);
279
280            mAudioQueue.erase(mAudioQueue.begin());
281            entry = NULL;
282            return false;
283        }
284
285        if (entry->mOffset == 0) {
286            int64_t mediaTimeUs;
287            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
288
289            ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
290
291            mAnchorTimeMediaUs = mediaTimeUs;
292
293            uint32_t numFramesPlayed;
294            CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
295
296            uint32_t numFramesPendingPlayout =
297                mNumFramesWritten - numFramesPlayed;
298
299            int64_t realTimeOffsetUs =
300                (mAudioSink->latency() / 2  /* XXX */
301                    + numFramesPendingPlayout
302                        * mAudioSink->msecsPerFrame()) * 1000ll;
303
304            // ALOGI("realTimeOffsetUs = %lld us", realTimeOffsetUs);
305
306            mAnchorTimeRealUs =
307                ALooper::GetNowUs() + realTimeOffsetUs;
308        }
309
310        size_t copy = entry->mBuffer->size() - entry->mOffset;
311        if (copy > numBytesAvailableToWrite) {
312            copy = numBytesAvailableToWrite;
313        }
314
315        CHECK_EQ(mAudioSink->write(
316                    entry->mBuffer->data() + entry->mOffset, copy),
317                 (ssize_t)copy);
318
319        entry->mOffset += copy;
320        if (entry->mOffset == entry->mBuffer->size()) {
321            entry->mNotifyConsumed->post();
322            mAudioQueue.erase(mAudioQueue.begin());
323
324            entry = NULL;
325        }
326
327        numBytesAvailableToWrite -= copy;
328        size_t copiedFrames = copy / mAudioSink->frameSize();
329        mNumFramesWritten += copiedFrames;
330
331        notifyIfMediaRenderingStarted();
332    }
333
334    notifyPosition();
335
336    return !mAudioQueue.empty();
337}
338
339void NuPlayer::Renderer::postDrainVideoQueue() {
340    if (mDrainVideoQueuePending || mSyncQueues || mPaused) {
341        return;
342    }
343
344    if (mVideoQueue.empty()) {
345        return;
346    }
347
348    QueueEntry &entry = *mVideoQueue.begin();
349
350    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
351    msg->setInt32("generation", mVideoQueueGeneration);
352
353    int64_t delayUs;
354
355    if (entry.mBuffer == NULL) {
356        // EOS doesn't carry a timestamp.
357        delayUs = 0;
358    } else if (mFlags & FLAG_REAL_TIME) {
359        int64_t mediaTimeUs;
360        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
361
362        delayUs = mediaTimeUs - ALooper::GetNowUs();
363    } else {
364        int64_t mediaTimeUs;
365        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
366
367        if (mAnchorTimeMediaUs < 0) {
368            delayUs = 0;
369
370            if (!mHasAudio) {
371                mAnchorTimeMediaUs = mediaTimeUs;
372                mAnchorTimeRealUs = ALooper::GetNowUs();
373            }
374        } else {
375            int64_t realTimeUs =
376                (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
377
378            delayUs = realTimeUs - ALooper::GetNowUs();
379        }
380    }
381
382    msg->post(delayUs);
383
384    mDrainVideoQueuePending = true;
385}
386
387void NuPlayer::Renderer::onDrainVideoQueue() {
388    if (mVideoQueue.empty()) {
389        return;
390    }
391
392    QueueEntry *entry = &*mVideoQueue.begin();
393
394    if (entry->mBuffer == NULL) {
395        // EOS
396
397        notifyEOS(false /* audio */, entry->mFinalResult);
398
399        mVideoQueue.erase(mVideoQueue.begin());
400        entry = NULL;
401
402        mVideoLateByUs = 0ll;
403
404        notifyPosition();
405        return;
406    }
407
408    int64_t realTimeUs;
409    if (mFlags & FLAG_REAL_TIME) {
410        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
411    } else {
412        int64_t mediaTimeUs;
413        CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
414
415        realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs;
416    }
417
418    mVideoLateByUs = ALooper::GetNowUs() - realTimeUs;
419    bool tooLate = (mVideoLateByUs > 40000);
420
421    if (tooLate) {
422        ALOGV("video late by %lld us (%.2f secs)",
423             mVideoLateByUs, mVideoLateByUs / 1E6);
424    } else {
425        ALOGV("rendering video at media time %.2f secs",
426                (mFlags & FLAG_REAL_TIME ? realTimeUs :
427                (realTimeUs + mAnchorTimeMediaUs - mAnchorTimeRealUs)) / 1E6);
428        if (mSoftRenderer != NULL) {
429            mSoftRenderer->render(entry->mBuffer->data(), entry->mBuffer->size(), NULL);
430        }
431    }
432
433    entry->mNotifyConsumed->setInt32("render", !tooLate);
434    entry->mNotifyConsumed->post();
435    mVideoQueue.erase(mVideoQueue.begin());
436    entry = NULL;
437
438    if (!mVideoRenderingStarted) {
439        mVideoRenderingStarted = true;
440        notifyVideoRenderingStart();
441    }
442
443    notifyIfMediaRenderingStarted();
444
445    notifyPosition();
446}
447
448void NuPlayer::Renderer::notifyVideoRenderingStart() {
449    sp<AMessage> notify = mNotify->dup();
450    notify->setInt32("what", kWhatVideoRenderingStart);
451    notify->post();
452}
453
454void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult) {
455    sp<AMessage> notify = mNotify->dup();
456    notify->setInt32("what", kWhatEOS);
457    notify->setInt32("audio", static_cast<int32_t>(audio));
458    notify->setInt32("finalResult", finalResult);
459    notify->post();
460}
461
462void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
463    int32_t audio;
464    CHECK(msg->findInt32("audio", &audio));
465
466    if (audio) {
467        mHasAudio = true;
468    } else {
469        mHasVideo = true;
470    }
471
472    if (dropBufferWhileFlushing(audio, msg)) {
473        return;
474    }
475
476    sp<ABuffer> buffer;
477    CHECK(msg->findBuffer("buffer", &buffer));
478
479    sp<AMessage> notifyConsumed;
480    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
481
482    QueueEntry entry;
483    entry.mBuffer = buffer;
484    entry.mNotifyConsumed = notifyConsumed;
485    entry.mOffset = 0;
486    entry.mFinalResult = OK;
487
488    if (audio) {
489        mAudioQueue.push_back(entry);
490        postDrainAudioQueue();
491    } else {
492        mVideoQueue.push_back(entry);
493        postDrainVideoQueue();
494    }
495
496    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
497        return;
498    }
499
500    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
501    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
502
503    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
504        // EOS signalled on either queue.
505        syncQueuesDone();
506        return;
507    }
508
509    int64_t firstAudioTimeUs;
510    int64_t firstVideoTimeUs;
511    CHECK(firstAudioBuffer->meta()
512            ->findInt64("timeUs", &firstAudioTimeUs));
513    CHECK(firstVideoBuffer->meta()
514            ->findInt64("timeUs", &firstVideoTimeUs));
515
516    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
517
518    ALOGV("queueDiff = %.2f secs", diff / 1E6);
519
520    if (diff > 100000ll) {
521        // Audio data starts More than 0.1 secs before video.
522        // Drop some audio.
523
524        (*mAudioQueue.begin()).mNotifyConsumed->post();
525        mAudioQueue.erase(mAudioQueue.begin());
526        return;
527    }
528
529    syncQueuesDone();
530}
531
532void NuPlayer::Renderer::syncQueuesDone() {
533    if (!mSyncQueues) {
534        return;
535    }
536
537    mSyncQueues = false;
538
539    if (!mAudioQueue.empty()) {
540        postDrainAudioQueue();
541    }
542
543    if (!mVideoQueue.empty()) {
544        postDrainVideoQueue();
545    }
546}
547
548void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
549    int32_t audio;
550    CHECK(msg->findInt32("audio", &audio));
551
552    if (dropBufferWhileFlushing(audio, msg)) {
553        return;
554    }
555
556    int32_t finalResult;
557    CHECK(msg->findInt32("finalResult", &finalResult));
558
559    QueueEntry entry;
560    entry.mOffset = 0;
561    entry.mFinalResult = finalResult;
562
563    if (audio) {
564        if (mAudioQueue.empty() && mSyncQueues) {
565            syncQueuesDone();
566        }
567        mAudioQueue.push_back(entry);
568        postDrainAudioQueue();
569    } else {
570        if (mVideoQueue.empty() && mSyncQueues) {
571            syncQueuesDone();
572        }
573        mVideoQueue.push_back(entry);
574        postDrainVideoQueue();
575    }
576}
577
578void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
579    int32_t audio;
580    CHECK(msg->findInt32("audio", &audio));
581
582    // If we're currently syncing the queues, i.e. dropping audio while
583    // aligning the first audio/video buffer times and only one of the
584    // two queues has data, we may starve that queue by not requesting
585    // more buffers from the decoder. If the other source then encounters
586    // a discontinuity that leads to flushing, we'll never find the
587    // corresponding discontinuity on the other queue.
588    // Therefore we'll stop syncing the queues if at least one of them
589    // is flushed.
590    syncQueuesDone();
591
592    ALOGV("flushing %s", audio ? "audio" : "video");
593    if (audio) {
594        flushQueue(&mAudioQueue);
595
596        Mutex::Autolock autoLock(mFlushLock);
597        mFlushingAudio = false;
598
599        mDrainAudioQueuePending = false;
600        ++mAudioQueueGeneration;
601
602        prepareForMediaRenderingStart();
603    } else {
604        flushQueue(&mVideoQueue);
605
606        Mutex::Autolock autoLock(mFlushLock);
607        mFlushingVideo = false;
608
609        mDrainVideoQueuePending = false;
610        ++mVideoQueueGeneration;
611
612        prepareForMediaRenderingStart();
613    }
614
615    notifyFlushComplete(audio);
616}
617
618void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
619    while (!queue->empty()) {
620        QueueEntry *entry = &*queue->begin();
621
622        if (entry->mBuffer != NULL) {
623            entry->mNotifyConsumed->post();
624        }
625
626        queue->erase(queue->begin());
627        entry = NULL;
628    }
629}
630
631void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
632    sp<AMessage> notify = mNotify->dup();
633    notify->setInt32("what", kWhatFlushComplete);
634    notify->setInt32("audio", static_cast<int32_t>(audio));
635    notify->post();
636}
637
638bool NuPlayer::Renderer::dropBufferWhileFlushing(
639        bool audio, const sp<AMessage> &msg) {
640    bool flushing = false;
641
642    {
643        Mutex::Autolock autoLock(mFlushLock);
644        if (audio) {
645            flushing = mFlushingAudio;
646        } else {
647            flushing = mFlushingVideo;
648        }
649    }
650
651    if (!flushing) {
652        return false;
653    }
654
655    sp<AMessage> notifyConsumed;
656    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
657        notifyConsumed->post();
658    }
659
660    return true;
661}
662
663void NuPlayer::Renderer::onAudioSinkChanged() {
664    CHECK(!mDrainAudioQueuePending);
665    mNumFramesWritten = 0;
666    uint32_t written;
667    if (mAudioSink->getFramesWritten(&written) == OK) {
668        mNumFramesWritten = written;
669    }
670}
671
672void NuPlayer::Renderer::notifyPosition() {
673    if (mAnchorTimeRealUs < 0 || mAnchorTimeMediaUs < 0) {
674        return;
675    }
676
677    int64_t nowUs = ALooper::GetNowUs();
678
679    if (mLastPositionUpdateUs >= 0
680            && nowUs < mLastPositionUpdateUs + kMinPositionUpdateDelayUs) {
681        return;
682    }
683    mLastPositionUpdateUs = nowUs;
684
685    int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs;
686
687    sp<AMessage> notify = mNotify->dup();
688    notify->setInt32("what", kWhatPosition);
689    notify->setInt64("positionUs", positionUs);
690    notify->setInt64("videoLateByUs", mVideoLateByUs);
691    notify->post();
692}
693
694void NuPlayer::Renderer::onPause() {
695    CHECK(!mPaused);
696
697    mDrainAudioQueuePending = false;
698    ++mAudioQueueGeneration;
699
700    mDrainVideoQueuePending = false;
701    ++mVideoQueueGeneration;
702
703    prepareForMediaRenderingStart();
704
705    if (mHasAudio) {
706        mAudioSink->pause();
707    }
708
709    ALOGV("now paused audio queue has %d entries, video has %d entries",
710          mAudioQueue.size(), mVideoQueue.size());
711
712    mPaused = true;
713}
714
715void NuPlayer::Renderer::onResume() {
716    if (!mPaused) {
717        return;
718    }
719
720    if (mHasAudio) {
721        mAudioSink->start();
722    }
723
724    mPaused = false;
725
726    if (!mAudioQueue.empty()) {
727        postDrainAudioQueue();
728    }
729
730    if (!mVideoQueue.empty()) {
731        postDrainVideoQueue();
732    }
733}
734
735}  // namespace android
736
737