NuPlayerRenderer.cpp revision b50e83eca302a12f0fced6e7bab1b8617d63deaa 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26
27namespace android {
28
29// static
30const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
31
32NuPlayer::Renderer::Renderer(
33        const sp<MediaPlayerBase::AudioSink> &sink,
34        const sp<AMessage> &notify)
35    : mAudioSink(sink),
36      mNotify(notify),
37      mNumFramesWritten(0),
38      mDrainAudioQueuePending(false),
39      mDrainVideoQueuePending(false),
40      mAudioQueueGeneration(0),
41      mVideoQueueGeneration(0),
42      mAnchorTimeMediaUs(-1),
43      mAnchorTimeRealUs(-1),
44      mFlushingAudio(false),
45      mFlushingVideo(false),
46      mHasAudio(false),
47      mHasVideo(false),
48      mSyncQueues(false),
49      mPaused(false),
50      mVideoRenderingStarted(false),
51      mLastPositionUpdateUs(-1ll),
52      mVideoLateByUs(0ll) {
53}
54
55NuPlayer::Renderer::~Renderer() {
56}
57
58void NuPlayer::Renderer::queueBuffer(
59        bool audio,
60        const sp<ABuffer> &buffer,
61        const sp<AMessage> &notifyConsumed) {
62    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
63    msg->setInt32("audio", static_cast<int32_t>(audio));
64    msg->setBuffer("buffer", buffer);
65    msg->setMessage("notifyConsumed", notifyConsumed);
66    msg->post();
67}
68
69void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
70    CHECK_NE(finalResult, (status_t)OK);
71
72    sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
73    msg->setInt32("audio", static_cast<int32_t>(audio));
74    msg->setInt32("finalResult", finalResult);
75    msg->post();
76}
77
78void NuPlayer::Renderer::flush(bool audio) {
79    {
80        Mutex::Autolock autoLock(mFlushLock);
81        if (audio) {
82            CHECK(!mFlushingAudio);
83            mFlushingAudio = true;
84        } else {
85            CHECK(!mFlushingVideo);
86            mFlushingVideo = true;
87        }
88    }
89
90    sp<AMessage> msg = new AMessage(kWhatFlush, id());
91    msg->setInt32("audio", static_cast<int32_t>(audio));
92    msg->post();
93}
94
95void NuPlayer::Renderer::signalTimeDiscontinuity() {
96    CHECK(mAudioQueue.empty());
97    CHECK(mVideoQueue.empty());
98    mAnchorTimeMediaUs = -1;
99    mAnchorTimeRealUs = -1;
100    mSyncQueues = mHasAudio && mHasVideo;
101}
102
103void NuPlayer::Renderer::pause() {
104    (new AMessage(kWhatPause, id()))->post();
105}
106
107void NuPlayer::Renderer::resume() {
108    (new AMessage(kWhatResume, id()))->post();
109}
110
111void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
112    switch (msg->what()) {
113        case kWhatDrainAudioQueue:
114        {
115            int32_t generation;
116            CHECK(msg->findInt32("generation", &generation));
117            if (generation != mAudioQueueGeneration) {
118                break;
119            }
120
121            mDrainAudioQueuePending = false;
122
123            if (onDrainAudioQueue()) {
124                uint32_t numFramesPlayed;
125                CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
126                         (status_t)OK);
127
128                uint32_t numFramesPendingPlayout =
129                    mNumFramesWritten - numFramesPlayed;
130
131                // This is how long the audio sink will have data to
132                // play back.
133                int64_t delayUs =
134                    mAudioSink->msecsPerFrame()
135                        * numFramesPendingPlayout * 1000ll;
136
137                // Let's give it more data after about half that time
138                // has elapsed.
139                postDrainAudioQueue(delayUs / 2);
140            }
141            break;
142        }
143
144        case kWhatDrainVideoQueue:
145        {
146            int32_t generation;
147            CHECK(msg->findInt32("generation", &generation));
148            if (generation != mVideoQueueGeneration) {
149                break;
150            }
151
152            mDrainVideoQueuePending = false;
153
154            onDrainVideoQueue();
155
156            postDrainVideoQueue();
157            break;
158        }
159
160        case kWhatQueueBuffer:
161        {
162            onQueueBuffer(msg);
163            break;
164        }
165
166        case kWhatQueueEOS:
167        {
168            onQueueEOS(msg);
169            break;
170        }
171
172        case kWhatFlush:
173        {
174            onFlush(msg);
175            break;
176        }
177
178        case kWhatAudioSinkChanged:
179        {
180            onAudioSinkChanged();
181            break;
182        }
183
184        case kWhatPause:
185        {
186            onPause();
187            break;
188        }
189
190        case kWhatResume:
191        {
192            onResume();
193            break;
194        }
195
196        default:
197            TRESPASS();
198            break;
199    }
200}
201
202void NuPlayer::Renderer::postDrainAudioQueue(int64_t delayUs) {
203    if (mDrainAudioQueuePending || mSyncQueues || mPaused) {
204        return;
205    }
206
207    if (mAudioQueue.empty()) {
208        return;
209    }
210
211    mDrainAudioQueuePending = true;
212    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
213    msg->setInt32("generation", mAudioQueueGeneration);
214    msg->post(delayUs);
215}
216
217void NuPlayer::Renderer::signalAudioSinkChanged() {
218    (new AMessage(kWhatAudioSinkChanged, id()))->post();
219}
220
221bool NuPlayer::Renderer::onDrainAudioQueue() {
222    uint32_t numFramesPlayed;
223    if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
224        return false;
225    }
226
227    ssize_t numFramesAvailableToWrite =
228        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
229
230#if 0
231    if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
232        ALOGI("audio sink underrun");
233    } else {
234        ALOGV("audio queue has %d frames left to play",
235             mAudioSink->frameCount() - numFramesAvailableToWrite);
236    }
237#endif
238
239    size_t numBytesAvailableToWrite =
240        numFramesAvailableToWrite * mAudioSink->frameSize();
241
242    while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) {
243        QueueEntry *entry = &*mAudioQueue.begin();
244
245        if (entry->mBuffer == NULL) {
246            // EOS
247
248            notifyEOS(true /* audio */, entry->mFinalResult);
249
250            mAudioQueue.erase(mAudioQueue.begin());
251            entry = NULL;
252            return false;
253        }
254
255        if (entry->mOffset == 0) {
256            int64_t mediaTimeUs;
257            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
258
259            ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
260
261            mAnchorTimeMediaUs = mediaTimeUs;
262
263            uint32_t numFramesPlayed;
264            CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
265
266            uint32_t numFramesPendingPlayout =
267                mNumFramesWritten - numFramesPlayed;
268
269            int64_t realTimeOffsetUs =
270                (mAudioSink->latency() / 2  /* XXX */
271                    + numFramesPendingPlayout
272                        * mAudioSink->msecsPerFrame()) * 1000ll;
273
274            // ALOGI("realTimeOffsetUs = %lld us", realTimeOffsetUs);
275
276            mAnchorTimeRealUs =
277                ALooper::GetNowUs() + realTimeOffsetUs;
278        }
279
280        size_t copy = entry->mBuffer->size() - entry->mOffset;
281        if (copy > numBytesAvailableToWrite) {
282            copy = numBytesAvailableToWrite;
283        }
284
285        CHECK_EQ(mAudioSink->write(
286                    entry->mBuffer->data() + entry->mOffset, copy),
287                 (ssize_t)copy);
288
289        entry->mOffset += copy;
290        if (entry->mOffset == entry->mBuffer->size()) {
291            entry->mNotifyConsumed->post();
292            mAudioQueue.erase(mAudioQueue.begin());
293
294            entry = NULL;
295        }
296
297        numBytesAvailableToWrite -= copy;
298        size_t copiedFrames = copy / mAudioSink->frameSize();
299        mNumFramesWritten += copiedFrames;
300    }
301
302    notifyPosition();
303
304    return !mAudioQueue.empty();
305}
306
307void NuPlayer::Renderer::postDrainVideoQueue() {
308    if (mDrainVideoQueuePending || mSyncQueues || mPaused) {
309        return;
310    }
311
312    if (mVideoQueue.empty()) {
313        return;
314    }
315
316    QueueEntry &entry = *mVideoQueue.begin();
317
318    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
319    msg->setInt32("generation", mVideoQueueGeneration);
320
321    int64_t delayUs;
322
323    if (entry.mBuffer == NULL) {
324        // EOS doesn't carry a timestamp.
325        delayUs = 0;
326    } else {
327        int64_t mediaTimeUs;
328        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
329
330        if (mAnchorTimeMediaUs < 0) {
331            delayUs = 0;
332
333            if (!mHasAudio) {
334                mAnchorTimeMediaUs = mediaTimeUs;
335                mAnchorTimeRealUs = ALooper::GetNowUs();
336            }
337        } else {
338            int64_t realTimeUs =
339                (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
340
341            delayUs = realTimeUs - ALooper::GetNowUs();
342        }
343    }
344
345    msg->post(delayUs);
346
347    mDrainVideoQueuePending = true;
348}
349
350void NuPlayer::Renderer::onDrainVideoQueue() {
351    if (mVideoQueue.empty()) {
352        return;
353    }
354
355    QueueEntry *entry = &*mVideoQueue.begin();
356
357    if (entry->mBuffer == NULL) {
358        // EOS
359
360        notifyEOS(false /* audio */, entry->mFinalResult);
361
362        mVideoQueue.erase(mVideoQueue.begin());
363        entry = NULL;
364
365        mVideoLateByUs = 0ll;
366
367        notifyPosition();
368        return;
369    }
370
371    int64_t mediaTimeUs;
372    CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
373
374    int64_t realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs;
375    mVideoLateByUs = ALooper::GetNowUs() - realTimeUs;
376
377    bool tooLate = (mVideoLateByUs > 40000);
378
379    if (tooLate) {
380        ALOGV("video late by %lld us (%.2f secs)",
381             mVideoLateByUs, mVideoLateByUs / 1E6);
382    } else {
383        ALOGV("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
384    }
385
386    entry->mNotifyConsumed->setInt32("render", !tooLate);
387    entry->mNotifyConsumed->post();
388    mVideoQueue.erase(mVideoQueue.begin());
389    entry = NULL;
390
391    if (!mVideoRenderingStarted) {
392        mVideoRenderingStarted = true;
393        notifyVideoRenderingStart();
394    }
395
396    notifyPosition();
397}
398
399void NuPlayer::Renderer::notifyVideoRenderingStart() {
400    sp<AMessage> notify = mNotify->dup();
401    notify->setInt32("what", kWhatVideoRenderingStart);
402    notify->post();
403}
404
405void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult) {
406    sp<AMessage> notify = mNotify->dup();
407    notify->setInt32("what", kWhatEOS);
408    notify->setInt32("audio", static_cast<int32_t>(audio));
409    notify->setInt32("finalResult", finalResult);
410    notify->post();
411}
412
413void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
414    int32_t audio;
415    CHECK(msg->findInt32("audio", &audio));
416
417    if (audio) {
418        mHasAudio = true;
419    } else {
420        mHasVideo = true;
421    }
422
423    if (dropBufferWhileFlushing(audio, msg)) {
424        return;
425    }
426
427    sp<ABuffer> buffer;
428    CHECK(msg->findBuffer("buffer", &buffer));
429
430    sp<AMessage> notifyConsumed;
431    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
432
433    QueueEntry entry;
434    entry.mBuffer = buffer;
435    entry.mNotifyConsumed = notifyConsumed;
436    entry.mOffset = 0;
437    entry.mFinalResult = OK;
438
439    if (audio) {
440        mAudioQueue.push_back(entry);
441        postDrainAudioQueue();
442    } else {
443        mVideoQueue.push_back(entry);
444        postDrainVideoQueue();
445    }
446
447    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
448        return;
449    }
450
451    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
452    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
453
454    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
455        // EOS signalled on either queue.
456        syncQueuesDone();
457        return;
458    }
459
460    int64_t firstAudioTimeUs;
461    int64_t firstVideoTimeUs;
462    CHECK(firstAudioBuffer->meta()
463            ->findInt64("timeUs", &firstAudioTimeUs));
464    CHECK(firstVideoBuffer->meta()
465            ->findInt64("timeUs", &firstVideoTimeUs));
466
467    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
468
469    ALOGV("queueDiff = %.2f secs", diff / 1E6);
470
471    if (diff > 100000ll) {
472        // Audio data starts More than 0.1 secs before video.
473        // Drop some audio.
474
475        (*mAudioQueue.begin()).mNotifyConsumed->post();
476        mAudioQueue.erase(mAudioQueue.begin());
477        return;
478    }
479
480    syncQueuesDone();
481}
482
483void NuPlayer::Renderer::syncQueuesDone() {
484    if (!mSyncQueues) {
485        return;
486    }
487
488    mSyncQueues = false;
489
490    if (!mAudioQueue.empty()) {
491        postDrainAudioQueue();
492    }
493
494    if (!mVideoQueue.empty()) {
495        postDrainVideoQueue();
496    }
497}
498
499void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
500    int32_t audio;
501    CHECK(msg->findInt32("audio", &audio));
502
503    if (dropBufferWhileFlushing(audio, msg)) {
504        return;
505    }
506
507    int32_t finalResult;
508    CHECK(msg->findInt32("finalResult", &finalResult));
509
510    QueueEntry entry;
511    entry.mOffset = 0;
512    entry.mFinalResult = finalResult;
513
514    if (audio) {
515        if (mAudioQueue.empty() && mSyncQueues) {
516            syncQueuesDone();
517        }
518        mAudioQueue.push_back(entry);
519        postDrainAudioQueue();
520    } else {
521        if (mVideoQueue.empty() && mSyncQueues) {
522            syncQueuesDone();
523        }
524        mVideoQueue.push_back(entry);
525        postDrainVideoQueue();
526    }
527}
528
529void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
530    int32_t audio;
531    CHECK(msg->findInt32("audio", &audio));
532
533    // If we're currently syncing the queues, i.e. dropping audio while
534    // aligning the first audio/video buffer times and only one of the
535    // two queues has data, we may starve that queue by not requesting
536    // more buffers from the decoder. If the other source then encounters
537    // a discontinuity that leads to flushing, we'll never find the
538    // corresponding discontinuity on the other queue.
539    // Therefore we'll stop syncing the queues if at least one of them
540    // is flushed.
541    syncQueuesDone();
542
543    if (audio) {
544        flushQueue(&mAudioQueue);
545
546        Mutex::Autolock autoLock(mFlushLock);
547        mFlushingAudio = false;
548
549        mDrainAudioQueuePending = false;
550        ++mAudioQueueGeneration;
551    } else {
552        flushQueue(&mVideoQueue);
553
554        Mutex::Autolock autoLock(mFlushLock);
555        mFlushingVideo = false;
556
557        mDrainVideoQueuePending = false;
558        ++mVideoQueueGeneration;
559    }
560
561    notifyFlushComplete(audio);
562}
563
564void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
565    while (!queue->empty()) {
566        QueueEntry *entry = &*queue->begin();
567
568        if (entry->mBuffer != NULL) {
569            entry->mNotifyConsumed->post();
570        }
571
572        queue->erase(queue->begin());
573        entry = NULL;
574    }
575}
576
577void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
578    sp<AMessage> notify = mNotify->dup();
579    notify->setInt32("what", kWhatFlushComplete);
580    notify->setInt32("audio", static_cast<int32_t>(audio));
581    notify->post();
582}
583
584bool NuPlayer::Renderer::dropBufferWhileFlushing(
585        bool audio, const sp<AMessage> &msg) {
586    bool flushing = false;
587
588    {
589        Mutex::Autolock autoLock(mFlushLock);
590        if (audio) {
591            flushing = mFlushingAudio;
592        } else {
593            flushing = mFlushingVideo;
594        }
595    }
596
597    if (!flushing) {
598        return false;
599    }
600
601    sp<AMessage> notifyConsumed;
602    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
603        notifyConsumed->post();
604    }
605
606    return true;
607}
608
609void NuPlayer::Renderer::onAudioSinkChanged() {
610    CHECK(!mDrainAudioQueuePending);
611    mNumFramesWritten = 0;
612    uint32_t written;
613    if (mAudioSink->getFramesWritten(&written) == OK) {
614        mNumFramesWritten = written;
615    }
616}
617
618void NuPlayer::Renderer::notifyPosition() {
619    if (mAnchorTimeRealUs < 0 || mAnchorTimeMediaUs < 0) {
620        return;
621    }
622
623    int64_t nowUs = ALooper::GetNowUs();
624
625    if (mLastPositionUpdateUs >= 0
626            && nowUs < mLastPositionUpdateUs + kMinPositionUpdateDelayUs) {
627        return;
628    }
629    mLastPositionUpdateUs = nowUs;
630
631    int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs;
632
633    sp<AMessage> notify = mNotify->dup();
634    notify->setInt32("what", kWhatPosition);
635    notify->setInt64("positionUs", positionUs);
636    notify->setInt64("videoLateByUs", mVideoLateByUs);
637    notify->post();
638}
639
640void NuPlayer::Renderer::onPause() {
641    CHECK(!mPaused);
642
643    mDrainAudioQueuePending = false;
644    ++mAudioQueueGeneration;
645
646    mDrainVideoQueuePending = false;
647    ++mVideoQueueGeneration;
648
649    if (mHasAudio) {
650        mAudioSink->pause();
651    }
652
653    ALOGV("now paused audio queue has %d entries, video has %d entries",
654          mAudioQueue.size(), mVideoQueue.size());
655
656    mPaused = true;
657}
658
659void NuPlayer::Renderer::onResume() {
660    if (!mPaused) {
661        return;
662    }
663
664    if (mHasAudio) {
665        mAudioSink->start();
666    }
667
668    mPaused = false;
669
670    if (!mAudioQueue.empty()) {
671        postDrainAudioQueue();
672    }
673
674    if (!mVideoQueue.empty()) {
675        postDrainVideoQueue();
676    }
677}
678
679}  // namespace android
680
681