NuPlayerRenderer.cpp revision 4110c101c3d0dd8dbc44c8d2d0edd3e2e7d6652f 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26
27namespace android {
28
29// static
30const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
31
32NuPlayer::Renderer::Renderer(
33        const sp<MediaPlayerBase::AudioSink> &sink,
34        const sp<AMessage> &notify)
35    : mAudioSink(sink),
36      mNotify(notify),
37      mNumFramesWritten(0),
38      mDrainAudioQueuePending(false),
39      mDrainVideoQueuePending(false),
40      mAudioQueueGeneration(0),
41      mVideoQueueGeneration(0),
42      mAnchorTimeMediaUs(-1),
43      mAnchorTimeRealUs(-1),
44      mFlushingAudio(false),
45      mFlushingVideo(false),
46      mHasAudio(false),
47      mHasVideo(false),
48      mSyncQueues(false),
49      mPaused(false),
50      mLastPositionUpdateUs(-1ll),
51      mVideoLateByUs(0ll) {
52}
53
54NuPlayer::Renderer::~Renderer() {
55}
56
57void NuPlayer::Renderer::queueBuffer(
58        bool audio,
59        const sp<ABuffer> &buffer,
60        const sp<AMessage> &notifyConsumed) {
61    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
62    msg->setInt32("audio", static_cast<int32_t>(audio));
63    msg->setBuffer("buffer", buffer);
64    msg->setMessage("notifyConsumed", notifyConsumed);
65    msg->post();
66}
67
68void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
69    CHECK_NE(finalResult, (status_t)OK);
70
71    sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
72    msg->setInt32("audio", static_cast<int32_t>(audio));
73    msg->setInt32("finalResult", finalResult);
74    msg->post();
75}
76
77void NuPlayer::Renderer::flush(bool audio) {
78    {
79        Mutex::Autolock autoLock(mFlushLock);
80        if (audio) {
81            CHECK(!mFlushingAudio);
82            mFlushingAudio = true;
83        } else {
84            CHECK(!mFlushingVideo);
85            mFlushingVideo = true;
86        }
87    }
88
89    sp<AMessage> msg = new AMessage(kWhatFlush, id());
90    msg->setInt32("audio", static_cast<int32_t>(audio));
91    msg->post();
92}
93
94void NuPlayer::Renderer::signalTimeDiscontinuity() {
95    CHECK(mAudioQueue.empty());
96    CHECK(mVideoQueue.empty());
97    mAnchorTimeMediaUs = -1;
98    mAnchorTimeRealUs = -1;
99    mSyncQueues = mHasAudio && mHasVideo;
100}
101
102void NuPlayer::Renderer::pause() {
103    (new AMessage(kWhatPause, id()))->post();
104}
105
106void NuPlayer::Renderer::resume() {
107    (new AMessage(kWhatResume, id()))->post();
108}
109
110void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
111    switch (msg->what()) {
112        case kWhatDrainAudioQueue:
113        {
114            int32_t generation;
115            CHECK(msg->findInt32("generation", &generation));
116            if (generation != mAudioQueueGeneration) {
117                break;
118            }
119
120            mDrainAudioQueuePending = false;
121
122            if (onDrainAudioQueue()) {
123                uint32_t numFramesPlayed;
124                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
125                         (status_t)OK);
126
127                uint32_t numFramesPendingPlayout =
128                    mNumFramesWritten - numFramesPlayed;
129
130                // This is how long the audio sink will have data to
131                // play back.
132                int64_t delayUs =
133                    mAudioSink->msecsPerFrame()
134                        * numFramesPendingPlayout * 1000ll;
135
136                // Let's give it more data after about half that time
137                // has elapsed.
138                postDrainAudioQueue(delayUs / 2);
139            }
140            break;
141        }
142
143        case kWhatDrainVideoQueue:
144        {
145            int32_t generation;
146            CHECK(msg->findInt32("generation", &generation));
147            if (generation != mVideoQueueGeneration) {
148                break;
149            }
150
151            mDrainVideoQueuePending = false;
152
153            onDrainVideoQueue();
154
155            postDrainVideoQueue();
156            break;
157        }
158
159        case kWhatQueueBuffer:
160        {
161            onQueueBuffer(msg);
162            break;
163        }
164
165        case kWhatQueueEOS:
166        {
167            onQueueEOS(msg);
168            break;
169        }
170
171        case kWhatFlush:
172        {
173            onFlush(msg);
174            break;
175        }
176
177        case kWhatAudioSinkChanged:
178        {
179            onAudioSinkChanged();
180            break;
181        }
182
183        case kWhatPause:
184        {
185            onPause();
186            break;
187        }
188
189        case kWhatResume:
190        {
191            onResume();
192            break;
193        }
194
195        default:
196            TRESPASS();
197            break;
198    }
199}
200
201void NuPlayer::Renderer::postDrainAudioQueue(int64_t delayUs) {
202    if (mDrainAudioQueuePending || mSyncQueues || mPaused) {
203        return;
204    }
205
206    if (mAudioQueue.empty()) {
207        return;
208    }
209
210    mDrainAudioQueuePending = true;
211    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
212    msg->setInt32("generation", mAudioQueueGeneration);
213    msg->post(delayUs);
214}
215
216void NuPlayer::Renderer::signalAudioSinkChanged() {
217    (new AMessage(kWhatAudioSinkChanged, id()))->post();
218}
219
220bool NuPlayer::Renderer::onDrainAudioQueue() {
221    uint32_t numFramesPlayed;
222    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
223        return false;
224    }
225
226    ssize_t numFramesAvailableToWrite =
227        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
228
229#if 0
230    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
231        ALOGI("audio sink underrun");
232    } else {
233        ALOGV("audio queue has %d frames left to play",
234             mAudioSink->frameCount() - numFramesAvailableToWrite);
235    }
236#endif
237
238    size_t numBytesAvailableToWrite =
239        numFramesAvailableToWrite * mAudioSink->frameSize();
240
241    while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) {
242        QueueEntry *entry = &*mAudioQueue.begin();
243
244        if (entry->mBuffer == NULL) {
245            // EOS
246
247            notifyEOS(true /* audio */, entry->mFinalResult);
248
249            mAudioQueue.erase(mAudioQueue.begin());
250            entry = NULL;
251            return false;
252        }
253
254        if (entry->mOffset == 0) {
255            int64_t mediaTimeUs;
256            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
257
258            ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
259
260            mAnchorTimeMediaUs = mediaTimeUs;
261
262            uint32_t numFramesPlayed;
263            CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
264
265            uint32_t numFramesPendingPlayout =
266                mNumFramesWritten - numFramesPlayed;
267
268            int64_t realTimeOffsetUs =
269                (mAudioSink->latency() / 2  /* XXX */
270                    + numFramesPendingPlayout
271                        * mAudioSink->msecsPerFrame()) * 1000ll;
272
273            // ALOGI("realTimeOffsetUs = %lld us", realTimeOffsetUs);
274
275            mAnchorTimeRealUs =
276                ALooper::GetNowUs() + realTimeOffsetUs;
277        }
278
279        size_t copy = entry->mBuffer->size() - entry->mOffset;
280        if (copy > numBytesAvailableToWrite) {
281            copy = numBytesAvailableToWrite;
282        }
283
284        CHECK_EQ(mAudioSink->write(
285                    entry->mBuffer->data() + entry->mOffset, copy),
286                 (ssize_t)copy);
287
288        entry->mOffset += copy;
289        if (entry->mOffset == entry->mBuffer->size()) {
290            entry->mNotifyConsumed->post();
291            mAudioQueue.erase(mAudioQueue.begin());
292
293            entry = NULL;
294        }
295
296        numBytesAvailableToWrite -= copy;
297        size_t copiedFrames = copy / mAudioSink->frameSize();
298        mNumFramesWritten += copiedFrames;
299    }
300
301    notifyPosition();
302
303    return !mAudioQueue.empty();
304}
305
306void NuPlayer::Renderer::postDrainVideoQueue() {
307    if (mDrainVideoQueuePending || mSyncQueues || mPaused) {
308        return;
309    }
310
311    if (mVideoQueue.empty()) {
312        return;
313    }
314
315    QueueEntry &entry = *mVideoQueue.begin();
316
317    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
318    msg->setInt32("generation", mVideoQueueGeneration);
319
320    int64_t delayUs;
321
322    if (entry.mBuffer == NULL) {
323        // EOS doesn't carry a timestamp.
324        delayUs = 0;
325    } else {
326        int64_t mediaTimeUs;
327        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
328
329        if (mAnchorTimeMediaUs < 0) {
330            delayUs = 0;
331
332            if (!mHasAudio) {
333                mAnchorTimeMediaUs = mediaTimeUs;
334                mAnchorTimeRealUs = ALooper::GetNowUs();
335            }
336        } else {
337            int64_t realTimeUs =
338                (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
339
340            delayUs = realTimeUs - ALooper::GetNowUs();
341        }
342    }
343
344    msg->post(delayUs);
345
346    mDrainVideoQueuePending = true;
347}
348
349void NuPlayer::Renderer::onDrainVideoQueue() {
350    if (mVideoQueue.empty()) {
351        return;
352    }
353
354    QueueEntry *entry = &*mVideoQueue.begin();
355
356    if (entry->mBuffer == NULL) {
357        // EOS
358
359        notifyEOS(false /* audio */, entry->mFinalResult);
360
361        mVideoQueue.erase(mVideoQueue.begin());
362        entry = NULL;
363
364        mVideoLateByUs = 0ll;
365
366        notifyPosition();
367        return;
368    }
369
370    int64_t mediaTimeUs;
371    CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
372
373    int64_t realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs;
374    mVideoLateByUs = ALooper::GetNowUs() - realTimeUs;
375
376    bool tooLate = (mVideoLateByUs > 40000);
377
378    if (tooLate) {
379        ALOGV("video late by %lld us (%.2f secs)",
380             mVideoLateByUs, mVideoLateByUs / 1E6);
381    } else {
382        ALOGV("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
383    }
384
385    entry->mNotifyConsumed->setInt32("render", !tooLate);
386    entry->mNotifyConsumed->post();
387    mVideoQueue.erase(mVideoQueue.begin());
388    entry = NULL;
389
390    notifyPosition();
391}
392
393void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult) {
394    sp<AMessage> notify = mNotify->dup();
395    notify->setInt32("what", kWhatEOS);
396    notify->setInt32("audio", static_cast<int32_t>(audio));
397    notify->setInt32("finalResult", finalResult);
398    notify->post();
399}
400
401void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
402    int32_t audio;
403    CHECK(msg->findInt32("audio", &audio));
404
405    if (audio) {
406        mHasAudio = true;
407    } else {
408        mHasVideo = true;
409    }
410
411    if (dropBufferWhileFlushing(audio, msg)) {
412        return;
413    }
414
415    sp<ABuffer> buffer;
416    CHECK(msg->findBuffer("buffer", &buffer));
417
418    sp<AMessage> notifyConsumed;
419    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
420
421    QueueEntry entry;
422    entry.mBuffer = buffer;
423    entry.mNotifyConsumed = notifyConsumed;
424    entry.mOffset = 0;
425    entry.mFinalResult = OK;
426
427    if (audio) {
428        mAudioQueue.push_back(entry);
429        postDrainAudioQueue();
430    } else {
431        mVideoQueue.push_back(entry);
432        postDrainVideoQueue();
433    }
434
435    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
436        return;
437    }
438
439    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
440    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
441
442    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
443        // EOS signalled on either queue.
444        syncQueuesDone();
445        return;
446    }
447
448    int64_t firstAudioTimeUs;
449    int64_t firstVideoTimeUs;
450    CHECK(firstAudioBuffer->meta()
451            ->findInt64("timeUs", &firstAudioTimeUs));
452    CHECK(firstVideoBuffer->meta()
453            ->findInt64("timeUs", &firstVideoTimeUs));
454
455    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
456
457    ALOGV("queueDiff = %.2f secs", diff / 1E6);
458
459    if (diff > 100000ll) {
460        // Audio data starts More than 0.1 secs before video.
461        // Drop some audio.
462
463        (*mAudioQueue.begin()).mNotifyConsumed->post();
464        mAudioQueue.erase(mAudioQueue.begin());
465        return;
466    }
467
468    syncQueuesDone();
469}
470
471void NuPlayer::Renderer::syncQueuesDone() {
472    if (!mSyncQueues) {
473        return;
474    }
475
476    mSyncQueues = false;
477
478    if (!mAudioQueue.empty()) {
479        postDrainAudioQueue();
480    }
481
482    if (!mVideoQueue.empty()) {
483        postDrainVideoQueue();
484    }
485}
486
487void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
488    int32_t audio;
489    CHECK(msg->findInt32("audio", &audio));
490
491    if (dropBufferWhileFlushing(audio, msg)) {
492        return;
493    }
494
495    int32_t finalResult;
496    CHECK(msg->findInt32("finalResult", &finalResult));
497
498    QueueEntry entry;
499    entry.mOffset = 0;
500    entry.mFinalResult = finalResult;
501
502    if (audio) {
503        mAudioQueue.push_back(entry);
504        postDrainAudioQueue();
505    } else {
506        mVideoQueue.push_back(entry);
507        postDrainVideoQueue();
508    }
509}
510
511void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
512    int32_t audio;
513    CHECK(msg->findInt32("audio", &audio));
514
515    // If we're currently syncing the queues, i.e. dropping audio while
516    // aligning the first audio/video buffer times and only one of the
517    // two queues has data, we may starve that queue by not requesting
518    // more buffers from the decoder. If the other source then encounters
519    // a discontinuity that leads to flushing, we'll never find the
520    // corresponding discontinuity on the other queue.
521    // Therefore we'll stop syncing the queues if at least one of them
522    // is flushed.
523    syncQueuesDone();
524
525    if (audio) {
526        flushQueue(&mAudioQueue);
527
528        Mutex::Autolock autoLock(mFlushLock);
529        mFlushingAudio = false;
530
531        mDrainAudioQueuePending = false;
532        ++mAudioQueueGeneration;
533    } else {
534        flushQueue(&mVideoQueue);
535
536        Mutex::Autolock autoLock(mFlushLock);
537        mFlushingVideo = false;
538
539        mDrainVideoQueuePending = false;
540        ++mVideoQueueGeneration;
541    }
542
543    notifyFlushComplete(audio);
544}
545
546void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
547    while (!queue->empty()) {
548        QueueEntry *entry = &*queue->begin();
549
550        if (entry->mBuffer != NULL) {
551            entry->mNotifyConsumed->post();
552        }
553
554        queue->erase(queue->begin());
555        entry = NULL;
556    }
557}
558
559void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
560    sp<AMessage> notify = mNotify->dup();
561    notify->setInt32("what", kWhatFlushComplete);
562    notify->setInt32("audio", static_cast<int32_t>(audio));
563    notify->post();
564}
565
566bool NuPlayer::Renderer::dropBufferWhileFlushing(
567        bool audio, const sp<AMessage> &msg) {
568    bool flushing = false;
569
570    {
571        Mutex::Autolock autoLock(mFlushLock);
572        if (audio) {
573            flushing = mFlushingAudio;
574        } else {
575            flushing = mFlushingVideo;
576        }
577    }
578
579    if (!flushing) {
580        return false;
581    }
582
583    sp<AMessage> notifyConsumed;
584    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
585        notifyConsumed->post();
586    }
587
588    return true;
589}
590
591void NuPlayer::Renderer::onAudioSinkChanged() {
592    CHECK(!mDrainAudioQueuePending);
593    mNumFramesWritten = 0;
594    uint32_t written;
595    if (mAudioSink->getFramesWritten(&written) == OK) {
596        mNumFramesWritten = written;
597    }
598}
599
600void NuPlayer::Renderer::notifyPosition() {
601    if (mAnchorTimeRealUs < 0 || mAnchorTimeMediaUs < 0) {
602        return;
603    }
604
605    int64_t nowUs = ALooper::GetNowUs();
606
607    if (mLastPositionUpdateUs >= 0
608            && nowUs < mLastPositionUpdateUs + kMinPositionUpdateDelayUs) {
609        return;
610    }
611    mLastPositionUpdateUs = nowUs;
612
613    int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs;
614
615    sp<AMessage> notify = mNotify->dup();
616    notify->setInt32("what", kWhatPosition);
617    notify->setInt64("positionUs", positionUs);
618    notify->setInt64("videoLateByUs", mVideoLateByUs);
619    notify->post();
620}
621
622void NuPlayer::Renderer::onPause() {
623    CHECK(!mPaused);
624
625    mDrainAudioQueuePending = false;
626    ++mAudioQueueGeneration;
627
628    mDrainVideoQueuePending = false;
629    ++mVideoQueueGeneration;
630
631    if (mHasAudio) {
632        mAudioSink->pause();
633    }
634
635    ALOGV("now paused audio queue has %d entries, video has %d entries",
636          mAudioQueue.size(), mVideoQueue.size());
637
638    mPaused = true;
639}
640
641void NuPlayer::Renderer::onResume() {
642    if (!mPaused) {
643        return;
644    }
645
646    if (mHasAudio) {
647        mAudioSink->start();
648    }
649
650    mPaused = false;
651
652    if (!mAudioQueue.empty()) {
653        postDrainAudioQueue();
654    }
655
656    if (!mVideoQueue.empty()) {
657        postDrainVideoQueue();
658    }
659}
660
661}  // namespace android
662
663