NuPlayerRenderer.cpp revision cb67cd1b51ff9ab221e3124cf7e546515fef3c87 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26
27namespace android {
28
29NuPlayer::Renderer::Renderer(
30        const sp<MediaPlayerBase::AudioSink> &sink,
31        const sp<AMessage> &notify)
32    : mAudioSink(sink),
33      mNotify(notify),
34      mNumFramesWritten(0),
35      mDrainAudioQueuePending(false),
36      mDrainVideoQueuePending(false),
37      mAudioQueueGeneration(0),
38      mVideoQueueGeneration(0),
39      mAnchorTimeMediaUs(-1),
40      mAnchorTimeRealUs(-1),
41      mFlushingAudio(false),
42      mFlushingVideo(false),
43      mHasAudio(false),
44      mHasVideo(false),
45      mSyncQueues(false),
46      mPaused(false) {
47}
48
49NuPlayer::Renderer::~Renderer() {
50}
51
52void NuPlayer::Renderer::queueBuffer(
53        bool audio,
54        const sp<ABuffer> &buffer,
55        const sp<AMessage> &notifyConsumed) {
56    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
57    msg->setInt32("audio", static_cast<int32_t>(audio));
58    msg->setObject("buffer", buffer);
59    msg->setMessage("notifyConsumed", notifyConsumed);
60    msg->post();
61}
62
63void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
64    CHECK_NE(finalResult, (status_t)OK);
65
66    sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
67    msg->setInt32("audio", static_cast<int32_t>(audio));
68    msg->setInt32("finalResult", finalResult);
69    msg->post();
70}
71
72void NuPlayer::Renderer::flush(bool audio) {
73    {
74        Mutex::Autolock autoLock(mFlushLock);
75        if (audio) {
76            CHECK(!mFlushingAudio);
77            mFlushingAudio = true;
78        } else {
79            CHECK(!mFlushingVideo);
80            mFlushingVideo = true;
81        }
82    }
83
84    sp<AMessage> msg = new AMessage(kWhatFlush, id());
85    msg->setInt32("audio", static_cast<int32_t>(audio));
86    msg->post();
87}
88
89void NuPlayer::Renderer::signalTimeDiscontinuity() {
90    CHECK(mAudioQueue.empty());
91    CHECK(mVideoQueue.empty());
92    mAnchorTimeMediaUs = -1;
93    mAnchorTimeRealUs = -1;
94    mSyncQueues = mHasAudio && mHasVideo;
95}
96
97void NuPlayer::Renderer::pause() {
98    (new AMessage(kWhatPause, id()))->post();
99}
100
101void NuPlayer::Renderer::resume() {
102    (new AMessage(kWhatResume, id()))->post();
103}
104
105void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
106    switch (msg->what()) {
107        case kWhatDrainAudioQueue:
108        {
109            int32_t generation;
110            CHECK(msg->findInt32("generation", &generation));
111            if (generation != mAudioQueueGeneration) {
112                break;
113            }
114
115            mDrainAudioQueuePending = false;
116
117            onDrainAudioQueue();
118
119            postDrainAudioQueue();
120            break;
121        }
122
123        case kWhatDrainVideoQueue:
124        {
125            int32_t generation;
126            CHECK(msg->findInt32("generation", &generation));
127            if (generation != mVideoQueueGeneration) {
128                break;
129            }
130
131            mDrainVideoQueuePending = false;
132
133            onDrainVideoQueue();
134
135            postDrainVideoQueue();
136            break;
137        }
138
139        case kWhatQueueBuffer:
140        {
141            onQueueBuffer(msg);
142            break;
143        }
144
145        case kWhatQueueEOS:
146        {
147            onQueueEOS(msg);
148            break;
149        }
150
151        case kWhatFlush:
152        {
153            onFlush(msg);
154            break;
155        }
156
157        case kWhatAudioSinkChanged:
158        {
159            onAudioSinkChanged();
160            break;
161        }
162
163        case kWhatPause:
164        {
165            onPause();
166            break;
167        }
168
169        case kWhatResume:
170        {
171            onResume();
172            break;
173        }
174
175        default:
176            TRESPASS();
177            break;
178    }
179}
180
181void NuPlayer::Renderer::postDrainAudioQueue() {
182    if (mDrainAudioQueuePending || mSyncQueues || mPaused) {
183        return;
184    }
185
186    if (mAudioQueue.empty()) {
187        return;
188    }
189
190    mDrainAudioQueuePending = true;
191    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
192    msg->setInt32("generation", mAudioQueueGeneration);
193    msg->post(10000);
194}
195
196void NuPlayer::Renderer::signalAudioSinkChanged() {
197    (new AMessage(kWhatAudioSinkChanged, id()))->post();
198}
199
200void NuPlayer::Renderer::onDrainAudioQueue() {
201
202    for (;;) {
203        if (mAudioQueue.empty()) {
204            break;
205        }
206
207        QueueEntry *entry = &*mAudioQueue.begin();
208
209        if (entry->mBuffer == NULL) {
210            // EOS
211
212            notifyEOS(true /* audio */, entry->mFinalResult);
213
214            mAudioQueue.erase(mAudioQueue.begin());
215            entry = NULL;
216            return;
217        }
218
219        uint32_t numFramesPlayed;
220        CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
221
222        ssize_t numFramesAvailableToWrite =
223            mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
224
225        size_t numBytesAvailableToWrite =
226            numFramesAvailableToWrite * mAudioSink->frameSize();
227
228        if (numBytesAvailableToWrite == 0) {
229            break;
230        }
231
232        if (entry->mOffset == 0) {
233            int64_t mediaTimeUs;
234            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
235
236            LOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
237
238            mAnchorTimeMediaUs = mediaTimeUs;
239
240            uint32_t numFramesPlayed;
241            CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
242
243            uint32_t numFramesPendingPlayout =
244                mNumFramesWritten - numFramesPlayed;
245
246            int64_t realTimeOffsetUs =
247                (mAudioSink->latency() / 2  /* XXX */
248                    + numFramesPendingPlayout
249                        * mAudioSink->msecsPerFrame()) * 1000ll;
250
251            // LOGI("realTimeOffsetUs = %lld us", realTimeOffsetUs);
252
253            mAnchorTimeRealUs =
254                ALooper::GetNowUs() + realTimeOffsetUs;
255        }
256
257        size_t copy = entry->mBuffer->size() - entry->mOffset;
258        if (copy > numBytesAvailableToWrite) {
259            copy = numBytesAvailableToWrite;
260        }
261
262        CHECK_EQ(mAudioSink->write(
263                    entry->mBuffer->data() + entry->mOffset, copy),
264                 (ssize_t)copy);
265
266        entry->mOffset += copy;
267        if (entry->mOffset == entry->mBuffer->size()) {
268            entry->mNotifyConsumed->post();
269            mAudioQueue.erase(mAudioQueue.begin());
270
271            entry = NULL;
272        }
273
274        mNumFramesWritten += copy / mAudioSink->frameSize();
275    }
276
277    notifyPosition();
278}
279
280void NuPlayer::Renderer::postDrainVideoQueue() {
281    if (mDrainVideoQueuePending || mSyncQueues || mPaused) {
282        return;
283    }
284
285    if (mVideoQueue.empty()) {
286        return;
287    }
288
289    QueueEntry &entry = *mVideoQueue.begin();
290
291    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
292    msg->setInt32("generation", mVideoQueueGeneration);
293
294    int64_t delayUs;
295
296    if (entry.mBuffer == NULL) {
297        // EOS doesn't carry a timestamp.
298        delayUs = 0;
299    } else {
300        int64_t mediaTimeUs;
301        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
302
303        if (mAnchorTimeMediaUs < 0) {
304            delayUs = 0;
305
306            if (!mHasAudio) {
307                mAnchorTimeMediaUs = mediaTimeUs;
308                mAnchorTimeRealUs = ALooper::GetNowUs();
309            }
310        } else {
311            int64_t realTimeUs =
312                (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
313
314            delayUs = realTimeUs - ALooper::GetNowUs();
315        }
316    }
317
318    msg->post(delayUs);
319
320    mDrainVideoQueuePending = true;
321}
322
323void NuPlayer::Renderer::onDrainVideoQueue() {
324    if (mVideoQueue.empty()) {
325        return;
326    }
327
328    QueueEntry *entry = &*mVideoQueue.begin();
329
330    if (entry->mBuffer == NULL) {
331        // EOS
332
333        notifyEOS(false /* audio */, entry->mFinalResult);
334
335        mVideoQueue.erase(mVideoQueue.begin());
336        entry = NULL;
337        return;
338    }
339
340#if 0
341    int64_t mediaTimeUs;
342    CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
343
344    LOGI("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
345#endif
346
347    entry->mNotifyConsumed->setInt32("render", true);
348    entry->mNotifyConsumed->post();
349    mVideoQueue.erase(mVideoQueue.begin());
350    entry = NULL;
351
352    notifyPosition();
353}
354
355void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult) {
356    sp<AMessage> notify = mNotify->dup();
357    notify->setInt32("what", kWhatEOS);
358    notify->setInt32("audio", static_cast<int32_t>(audio));
359    notify->setInt32("finalResult", finalResult);
360    notify->post();
361}
362
363void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
364    int32_t audio;
365    CHECK(msg->findInt32("audio", &audio));
366
367    if (audio) {
368        mHasAudio = true;
369    } else {
370        mHasVideo = true;
371    }
372
373    if (dropBufferWhileFlushing(audio, msg)) {
374        return;
375    }
376
377    sp<RefBase> obj;
378    CHECK(msg->findObject("buffer", &obj));
379    sp<ABuffer> buffer = static_cast<ABuffer *>(obj.get());
380
381    sp<AMessage> notifyConsumed;
382    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
383
384    QueueEntry entry;
385    entry.mBuffer = buffer;
386    entry.mNotifyConsumed = notifyConsumed;
387    entry.mOffset = 0;
388    entry.mFinalResult = OK;
389
390    if (audio) {
391        mAudioQueue.push_back(entry);
392        postDrainAudioQueue();
393    } else {
394        mVideoQueue.push_back(entry);
395        postDrainVideoQueue();
396    }
397
398    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
399        return;
400    }
401
402    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
403    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
404
405    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
406        // EOS signalled on either queue.
407        syncQueuesDone();
408        return;
409    }
410
411    int64_t firstAudioTimeUs;
412    int64_t firstVideoTimeUs;
413    CHECK(firstAudioBuffer->meta()
414            ->findInt64("timeUs", &firstAudioTimeUs));
415    CHECK(firstVideoBuffer->meta()
416            ->findInt64("timeUs", &firstVideoTimeUs));
417
418    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
419
420    LOGV("queueDiff = %.2f secs", diff / 1E6);
421
422    if (diff > 100000ll) {
423        // Audio data starts More than 0.1 secs before video.
424        // Drop some audio.
425
426        (*mAudioQueue.begin()).mNotifyConsumed->post();
427        mAudioQueue.erase(mAudioQueue.begin());
428        return;
429    }
430
431    syncQueuesDone();
432}
433
434void NuPlayer::Renderer::syncQueuesDone() {
435    if (!mSyncQueues) {
436        return;
437    }
438
439    mSyncQueues = false;
440
441    if (!mAudioQueue.empty()) {
442        postDrainAudioQueue();
443    }
444
445    if (!mVideoQueue.empty()) {
446        postDrainVideoQueue();
447    }
448}
449
450void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
451    int32_t audio;
452    CHECK(msg->findInt32("audio", &audio));
453
454    if (dropBufferWhileFlushing(audio, msg)) {
455        return;
456    }
457
458    int32_t finalResult;
459    CHECK(msg->findInt32("finalResult", &finalResult));
460
461    QueueEntry entry;
462    entry.mOffset = 0;
463    entry.mFinalResult = finalResult;
464
465    if (audio) {
466        mAudioQueue.push_back(entry);
467        postDrainAudioQueue();
468    } else {
469        mVideoQueue.push_back(entry);
470        postDrainVideoQueue();
471    }
472}
473
474void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
475    int32_t audio;
476    CHECK(msg->findInt32("audio", &audio));
477
478    // If we're currently syncing the queues, i.e. dropping audio while
479    // aligning the first audio/video buffer times and only one of the
480    // two queues has data, we may starve that queue by not requesting
481    // more buffers from the decoder. If the other source then encounters
482    // a discontinuity that leads to flushing, we'll never find the
483    // corresponding discontinuity on the other queue.
484    // Therefore we'll stop syncing the queues if at least one of them
485    // is flushed.
486    syncQueuesDone();
487
488    if (audio) {
489        flushQueue(&mAudioQueue);
490
491        Mutex::Autolock autoLock(mFlushLock);
492        mFlushingAudio = false;
493
494        mDrainAudioQueuePending = false;
495        ++mAudioQueueGeneration;
496    } else {
497        flushQueue(&mVideoQueue);
498
499        Mutex::Autolock autoLock(mFlushLock);
500        mFlushingVideo = false;
501
502        mDrainVideoQueuePending = false;
503        ++mVideoQueueGeneration;
504    }
505
506    notifyFlushComplete(audio);
507}
508
509void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
510    while (!queue->empty()) {
511        QueueEntry *entry = &*queue->begin();
512
513        if (entry->mBuffer != NULL) {
514            entry->mNotifyConsumed->post();
515        }
516
517        queue->erase(queue->begin());
518        entry = NULL;
519    }
520}
521
522void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
523    sp<AMessage> notify = mNotify->dup();
524    notify->setInt32("what", kWhatFlushComplete);
525    notify->setInt32("audio", static_cast<int32_t>(audio));
526    notify->post();
527}
528
529bool NuPlayer::Renderer::dropBufferWhileFlushing(
530        bool audio, const sp<AMessage> &msg) {
531    bool flushing = false;
532
533    {
534        Mutex::Autolock autoLock(mFlushLock);
535        if (audio) {
536            flushing = mFlushingAudio;
537        } else {
538            flushing = mFlushingVideo;
539        }
540    }
541
542    if (!flushing) {
543        return false;
544    }
545
546    sp<AMessage> notifyConsumed;
547    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
548        notifyConsumed->post();
549    }
550
551    return true;
552}
553
554void NuPlayer::Renderer::onAudioSinkChanged() {
555    CHECK(!mDrainAudioQueuePending);
556    mNumFramesWritten = 0;
557}
558
559void NuPlayer::Renderer::notifyPosition() {
560    if (mAnchorTimeRealUs < 0 || mAnchorTimeMediaUs < 0) {
561        return;
562    }
563
564    int64_t nowUs = ALooper::GetNowUs();
565    int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs;
566
567    sp<AMessage> notify = mNotify->dup();
568    notify->setInt32("what", kWhatPosition);
569    notify->setInt64("positionUs", positionUs);
570    notify->post();
571}
572
573void NuPlayer::Renderer::onPause() {
574    CHECK(!mPaused);
575
576    mDrainAudioQueuePending = false;
577    ++mAudioQueueGeneration;
578
579    mDrainVideoQueuePending = false;
580    ++mVideoQueueGeneration;
581
582    if (mHasAudio) {
583        mAudioSink->pause();
584    }
585
586    mPaused = true;
587}
588
589void NuPlayer::Renderer::onResume() {
590    CHECK(mPaused);
591
592    if (mHasAudio) {
593        mAudioSink->start();
594    }
595
596    mPaused = false;
597
598    if (!mAudioQueue.empty()) {
599        postDrainAudioQueue();
600    }
601
602    if (!mVideoQueue.empty()) {
603        postDrainVideoQueue();
604    }
605}
606
607}  // namespace android
608
609