NuPlayerRenderer.cpp revision 714aa7b7c52ce07d5fb44870c0853b4d8e5a758e 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26
27namespace android {
28
29// static
30const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
31
32NuPlayer::Renderer::Renderer(
33        const sp<MediaPlayerBase::AudioSink> &sink,
34        const sp<AMessage> &notify)
35    : mAudioSink(sink),
36      mNotify(notify),
37      mNumFramesWritten(0),
38      mDrainAudioQueuePending(false),
39      mDrainVideoQueuePending(false),
40      mAudioQueueGeneration(0),
41      mVideoQueueGeneration(0),
42      mAnchorTimeMediaUs(-1),
43      mAnchorTimeRealUs(-1),
44      mFlushingAudio(false),
45      mFlushingVideo(false),
46      mHasAudio(false),
47      mHasVideo(false),
48      mSyncQueues(false),
49      mPaused(false),
50      mLastPositionUpdateUs(-1ll) {
51}
52
53NuPlayer::Renderer::~Renderer() {
54}
55
56void NuPlayer::Renderer::queueBuffer(
57        bool audio,
58        const sp<ABuffer> &buffer,
59        const sp<AMessage> &notifyConsumed) {
60    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
61    msg->setInt32("audio", static_cast<int32_t>(audio));
62    msg->setObject("buffer", buffer);
63    msg->setMessage("notifyConsumed", notifyConsumed);
64    msg->post();
65}
66
67void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
68    CHECK_NE(finalResult, (status_t)OK);
69
70    sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
71    msg->setInt32("audio", static_cast<int32_t>(audio));
72    msg->setInt32("finalResult", finalResult);
73    msg->post();
74}
75
76void NuPlayer::Renderer::flush(bool audio) {
77    {
78        Mutex::Autolock autoLock(mFlushLock);
79        if (audio) {
80            CHECK(!mFlushingAudio);
81            mFlushingAudio = true;
82        } else {
83            CHECK(!mFlushingVideo);
84            mFlushingVideo = true;
85        }
86    }
87
88    sp<AMessage> msg = new AMessage(kWhatFlush, id());
89    msg->setInt32("audio", static_cast<int32_t>(audio));
90    msg->post();
91}
92
93void NuPlayer::Renderer::signalTimeDiscontinuity() {
94    CHECK(mAudioQueue.empty());
95    CHECK(mVideoQueue.empty());
96    mAnchorTimeMediaUs = -1;
97    mAnchorTimeRealUs = -1;
98    mSyncQueues = mHasAudio && mHasVideo;
99}
100
101void NuPlayer::Renderer::pause() {
102    (new AMessage(kWhatPause, id()))->post();
103}
104
105void NuPlayer::Renderer::resume() {
106    (new AMessage(kWhatResume, id()))->post();
107}
108
109void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
110    switch (msg->what()) {
111        case kWhatDrainAudioQueue:
112        {
113            int32_t generation;
114            CHECK(msg->findInt32("generation", &generation));
115            if (generation != mAudioQueueGeneration) {
116                break;
117            }
118
119            mDrainAudioQueuePending = false;
120
121            onDrainAudioQueue();
122
123            postDrainAudioQueue();
124            break;
125        }
126
127        case kWhatDrainVideoQueue:
128        {
129            int32_t generation;
130            CHECK(msg->findInt32("generation", &generation));
131            if (generation != mVideoQueueGeneration) {
132                break;
133            }
134
135            mDrainVideoQueuePending = false;
136
137            onDrainVideoQueue();
138
139            postDrainVideoQueue();
140            break;
141        }
142
143        case kWhatQueueBuffer:
144        {
145            onQueueBuffer(msg);
146            break;
147        }
148
149        case kWhatQueueEOS:
150        {
151            onQueueEOS(msg);
152            break;
153        }
154
155        case kWhatFlush:
156        {
157            onFlush(msg);
158            break;
159        }
160
161        case kWhatAudioSinkChanged:
162        {
163            onAudioSinkChanged();
164            break;
165        }
166
167        case kWhatPause:
168        {
169            onPause();
170            break;
171        }
172
173        case kWhatResume:
174        {
175            onResume();
176            break;
177        }
178
179        default:
180            TRESPASS();
181            break;
182    }
183}
184
185void NuPlayer::Renderer::postDrainAudioQueue() {
186    if (mDrainAudioQueuePending || mSyncQueues || mPaused) {
187        return;
188    }
189
190    if (mAudioQueue.empty()) {
191        return;
192    }
193
194    mDrainAudioQueuePending = true;
195    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
196    msg->setInt32("generation", mAudioQueueGeneration);
197    msg->post();
198}
199
200void NuPlayer::Renderer::signalAudioSinkChanged() {
201    (new AMessage(kWhatAudioSinkChanged, id()))->post();
202}
203
204void NuPlayer::Renderer::onDrainAudioQueue() {
205    for (;;) {
206        if (mAudioQueue.empty()) {
207            break;
208        }
209
210        QueueEntry *entry = &*mAudioQueue.begin();
211
212        if (entry->mBuffer == NULL) {
213            // EOS
214
215            notifyEOS(true /* audio */, entry->mFinalResult);
216
217            mAudioQueue.erase(mAudioQueue.begin());
218            entry = NULL;
219            return;
220        }
221
222        uint32_t numFramesPlayed;
223        CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
224
225        ssize_t numFramesAvailableToWrite =
226            mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
227
228        size_t numBytesAvailableToWrite =
229            numFramesAvailableToWrite * mAudioSink->frameSize();
230
231        if (numBytesAvailableToWrite == 0) {
232            break;
233        }
234
235        if (entry->mOffset == 0) {
236            int64_t mediaTimeUs;
237            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
238
239            LOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
240
241            mAnchorTimeMediaUs = mediaTimeUs;
242
243            uint32_t numFramesPlayed;
244            CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
245
246            uint32_t numFramesPendingPlayout =
247                mNumFramesWritten - numFramesPlayed;
248
249            int64_t realTimeOffsetUs =
250                (mAudioSink->latency() / 2  /* XXX */
251                    + numFramesPendingPlayout
252                        * mAudioSink->msecsPerFrame()) * 1000ll;
253
254            // LOGI("realTimeOffsetUs = %lld us", realTimeOffsetUs);
255
256            mAnchorTimeRealUs =
257                ALooper::GetNowUs() + realTimeOffsetUs;
258        }
259
260        size_t copy = entry->mBuffer->size() - entry->mOffset;
261        if (copy > numBytesAvailableToWrite) {
262            copy = numBytesAvailableToWrite;
263        }
264
265        CHECK_EQ(mAudioSink->write(
266                    entry->mBuffer->data() + entry->mOffset, copy),
267                 (ssize_t)copy);
268
269        entry->mOffset += copy;
270        if (entry->mOffset == entry->mBuffer->size()) {
271            entry->mNotifyConsumed->post();
272            mAudioQueue.erase(mAudioQueue.begin());
273
274            entry = NULL;
275        }
276
277        mNumFramesWritten += copy / mAudioSink->frameSize();
278    }
279
280    notifyPosition();
281}
282
283void NuPlayer::Renderer::postDrainVideoQueue() {
284    if (mDrainVideoQueuePending || mSyncQueues || mPaused) {
285        return;
286    }
287
288    if (mVideoQueue.empty()) {
289        return;
290    }
291
292    QueueEntry &entry = *mVideoQueue.begin();
293
294    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
295    msg->setInt32("generation", mVideoQueueGeneration);
296
297    int64_t delayUs;
298
299    if (entry.mBuffer == NULL) {
300        // EOS doesn't carry a timestamp.
301        delayUs = 0;
302    } else {
303        int64_t mediaTimeUs;
304        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
305
306        if (mAnchorTimeMediaUs < 0) {
307            delayUs = 0;
308
309            if (!mHasAudio) {
310                mAnchorTimeMediaUs = mediaTimeUs;
311                mAnchorTimeRealUs = ALooper::GetNowUs();
312            }
313        } else {
314            int64_t realTimeUs =
315                (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
316
317            delayUs = realTimeUs - ALooper::GetNowUs();
318        }
319    }
320
321    msg->post(delayUs);
322
323    mDrainVideoQueuePending = true;
324}
325
326void NuPlayer::Renderer::onDrainVideoQueue() {
327    if (mVideoQueue.empty()) {
328        return;
329    }
330
331    QueueEntry *entry = &*mVideoQueue.begin();
332
333    if (entry->mBuffer == NULL) {
334        // EOS
335
336        notifyEOS(false /* audio */, entry->mFinalResult);
337
338        mVideoQueue.erase(mVideoQueue.begin());
339        entry = NULL;
340        return;
341    }
342
343#if 0
344    int64_t mediaTimeUs;
345    CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
346
347    LOGI("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
348#endif
349
350    entry->mNotifyConsumed->setInt32("render", true);
351    entry->mNotifyConsumed->post();
352    mVideoQueue.erase(mVideoQueue.begin());
353    entry = NULL;
354
355    notifyPosition();
356}
357
358void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult) {
359    sp<AMessage> notify = mNotify->dup();
360    notify->setInt32("what", kWhatEOS);
361    notify->setInt32("audio", static_cast<int32_t>(audio));
362    notify->setInt32("finalResult", finalResult);
363    notify->post();
364}
365
366void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
367    int32_t audio;
368    CHECK(msg->findInt32("audio", &audio));
369
370    if (audio) {
371        mHasAudio = true;
372    } else {
373        mHasVideo = true;
374    }
375
376    if (dropBufferWhileFlushing(audio, msg)) {
377        return;
378    }
379
380    sp<RefBase> obj;
381    CHECK(msg->findObject("buffer", &obj));
382    sp<ABuffer> buffer = static_cast<ABuffer *>(obj.get());
383
384    sp<AMessage> notifyConsumed;
385    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
386
387    QueueEntry entry;
388    entry.mBuffer = buffer;
389    entry.mNotifyConsumed = notifyConsumed;
390    entry.mOffset = 0;
391    entry.mFinalResult = OK;
392
393    if (audio) {
394        mAudioQueue.push_back(entry);
395        postDrainAudioQueue();
396    } else {
397        mVideoQueue.push_back(entry);
398        postDrainVideoQueue();
399    }
400
401    if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
402        return;
403    }
404
405    sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
406    sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
407
408    if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
409        // EOS signalled on either queue.
410        syncQueuesDone();
411        return;
412    }
413
414    int64_t firstAudioTimeUs;
415    int64_t firstVideoTimeUs;
416    CHECK(firstAudioBuffer->meta()
417            ->findInt64("timeUs", &firstAudioTimeUs));
418    CHECK(firstVideoBuffer->meta()
419            ->findInt64("timeUs", &firstVideoTimeUs));
420
421    int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
422
423    LOGV("queueDiff = %.2f secs", diff / 1E6);
424
425    if (diff > 100000ll) {
426        // Audio data starts More than 0.1 secs before video.
427        // Drop some audio.
428
429        (*mAudioQueue.begin()).mNotifyConsumed->post();
430        mAudioQueue.erase(mAudioQueue.begin());
431        return;
432    }
433
434    syncQueuesDone();
435}
436
437void NuPlayer::Renderer::syncQueuesDone() {
438    if (!mSyncQueues) {
439        return;
440    }
441
442    mSyncQueues = false;
443
444    if (!mAudioQueue.empty()) {
445        postDrainAudioQueue();
446    }
447
448    if (!mVideoQueue.empty()) {
449        postDrainVideoQueue();
450    }
451}
452
453void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
454    int32_t audio;
455    CHECK(msg->findInt32("audio", &audio));
456
457    if (dropBufferWhileFlushing(audio, msg)) {
458        return;
459    }
460
461    int32_t finalResult;
462    CHECK(msg->findInt32("finalResult", &finalResult));
463
464    QueueEntry entry;
465    entry.mOffset = 0;
466    entry.mFinalResult = finalResult;
467
468    if (audio) {
469        mAudioQueue.push_back(entry);
470        postDrainAudioQueue();
471    } else {
472        mVideoQueue.push_back(entry);
473        postDrainVideoQueue();
474    }
475}
476
477void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
478    int32_t audio;
479    CHECK(msg->findInt32("audio", &audio));
480
481    // If we're currently syncing the queues, i.e. dropping audio while
482    // aligning the first audio/video buffer times and only one of the
483    // two queues has data, we may starve that queue by not requesting
484    // more buffers from the decoder. If the other source then encounters
485    // a discontinuity that leads to flushing, we'll never find the
486    // corresponding discontinuity on the other queue.
487    // Therefore we'll stop syncing the queues if at least one of them
488    // is flushed.
489    syncQueuesDone();
490
491    if (audio) {
492        flushQueue(&mAudioQueue);
493
494        Mutex::Autolock autoLock(mFlushLock);
495        mFlushingAudio = false;
496
497        mDrainAudioQueuePending = false;
498        ++mAudioQueueGeneration;
499    } else {
500        flushQueue(&mVideoQueue);
501
502        Mutex::Autolock autoLock(mFlushLock);
503        mFlushingVideo = false;
504
505        mDrainVideoQueuePending = false;
506        ++mVideoQueueGeneration;
507    }
508
509    notifyFlushComplete(audio);
510}
511
512void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
513    while (!queue->empty()) {
514        QueueEntry *entry = &*queue->begin();
515
516        if (entry->mBuffer != NULL) {
517            entry->mNotifyConsumed->post();
518        }
519
520        queue->erase(queue->begin());
521        entry = NULL;
522    }
523}
524
525void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
526    sp<AMessage> notify = mNotify->dup();
527    notify->setInt32("what", kWhatFlushComplete);
528    notify->setInt32("audio", static_cast<int32_t>(audio));
529    notify->post();
530}
531
532bool NuPlayer::Renderer::dropBufferWhileFlushing(
533        bool audio, const sp<AMessage> &msg) {
534    bool flushing = false;
535
536    {
537        Mutex::Autolock autoLock(mFlushLock);
538        if (audio) {
539            flushing = mFlushingAudio;
540        } else {
541            flushing = mFlushingVideo;
542        }
543    }
544
545    if (!flushing) {
546        return false;
547    }
548
549    sp<AMessage> notifyConsumed;
550    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
551        notifyConsumed->post();
552    }
553
554    return true;
555}
556
557void NuPlayer::Renderer::onAudioSinkChanged() {
558    CHECK(!mDrainAudioQueuePending);
559    mNumFramesWritten = 0;
560}
561
562void NuPlayer::Renderer::notifyPosition() {
563    if (mAnchorTimeRealUs < 0 || mAnchorTimeMediaUs < 0) {
564        return;
565    }
566
567    int64_t nowUs = ALooper::GetNowUs();
568
569    if (mLastPositionUpdateUs >= 0
570            && nowUs < mLastPositionUpdateUs + kMinPositionUpdateDelayUs) {
571        return;
572    }
573    mLastPositionUpdateUs = nowUs;
574
575    int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs;
576
577    sp<AMessage> notify = mNotify->dup();
578    notify->setInt32("what", kWhatPosition);
579    notify->setInt64("positionUs", positionUs);
580    notify->post();
581}
582
583void NuPlayer::Renderer::onPause() {
584    CHECK(!mPaused);
585
586    mDrainAudioQueuePending = false;
587    ++mAudioQueueGeneration;
588
589    mDrainVideoQueuePending = false;
590    ++mVideoQueueGeneration;
591
592    if (mHasAudio) {
593        mAudioSink->pause();
594    }
595
596    mPaused = true;
597}
598
599void NuPlayer::Renderer::onResume() {
600    CHECK(mPaused);
601
602    if (mHasAudio) {
603        mAudioSink->start();
604    }
605
606    mPaused = false;
607
608    if (!mAudioQueue.empty()) {
609        postDrainAudioQueue();
610    }
611
612    if (!mVideoQueue.empty()) {
613        postDrainVideoQueue();
614    }
615}
616
617}  // namespace android
618
619