NuPlayerRenderer.cpp revision 43c3e6ce02215ca99d506458f596cb1211639f29 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26
27namespace android {
28
29NuPlayer::Renderer::Renderer(
30        const sp<MediaPlayerBase::AudioSink> &sink,
31        const sp<AMessage> &notify)
32    : mAudioSink(sink),
33      mNotify(notify),
34      mNumFramesWritten(0),
35      mDrainAudioQueuePending(false),
36      mDrainVideoQueuePending(false),
37      mAudioQueueGeneration(0),
38      mVideoQueueGeneration(0),
39      mAnchorTimeMediaUs(-1),
40      mAnchorTimeRealUs(-1),
41      mFlushingAudio(false),
42      mFlushingVideo(false),
43      mHasAudio(mAudioSink != NULL),
44      mHasVideo(true),
45      mSyncQueues(mHasAudio && mHasVideo) {
46}
47
48NuPlayer::Renderer::~Renderer() {
49}
50
51void NuPlayer::Renderer::queueBuffer(
52        bool audio,
53        const sp<ABuffer> &buffer,
54        const sp<AMessage> &notifyConsumed) {
55    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
56    msg->setInt32("audio", static_cast<int32_t>(audio));
57    msg->setObject("buffer", buffer);
58    msg->setMessage("notifyConsumed", notifyConsumed);
59    msg->post();
60}
61
62void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
63    CHECK_NE(finalResult, (status_t)OK);
64
65    sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
66    msg->setInt32("audio", static_cast<int32_t>(audio));
67    msg->setInt32("finalResult", finalResult);
68    msg->post();
69}
70
71void NuPlayer::Renderer::flush(bool audio) {
72    {
73        Mutex::Autolock autoLock(mFlushLock);
74        if (audio) {
75            CHECK(!mFlushingAudio);
76            mFlushingAudio = true;
77        } else {
78            CHECK(!mFlushingVideo);
79            mFlushingVideo = true;
80        }
81    }
82
83    sp<AMessage> msg = new AMessage(kWhatFlush, id());
84    msg->setInt32("audio", static_cast<int32_t>(audio));
85    msg->post();
86}
87
88void NuPlayer::Renderer::signalTimeDiscontinuity() {
89    CHECK(mAudioQueue.empty());
90    CHECK(mVideoQueue.empty());
91    mAnchorTimeMediaUs = -1;
92    mAnchorTimeRealUs = -1;
93    mSyncQueues = mHasAudio && mHasVideo;
94}
95
96void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
97    switch (msg->what()) {
98        case kWhatDrainAudioQueue:
99        {
100            int32_t generation;
101            CHECK(msg->findInt32("generation", &generation));
102            if (generation != mAudioQueueGeneration) {
103                break;
104            }
105
106            mDrainAudioQueuePending = false;
107
108            onDrainAudioQueue();
109
110            postDrainAudioQueue();
111            break;
112        }
113
114        case kWhatDrainVideoQueue:
115        {
116            int32_t generation;
117            CHECK(msg->findInt32("generation", &generation));
118            if (generation != mVideoQueueGeneration) {
119                break;
120            }
121
122            mDrainVideoQueuePending = false;
123
124            onDrainVideoQueue();
125
126            postDrainVideoQueue();
127            break;
128        }
129
130        case kWhatQueueBuffer:
131        {
132            onQueueBuffer(msg);
133            break;
134        }
135
136        case kWhatQueueEOS:
137        {
138            onQueueEOS(msg);
139            break;
140        }
141
142        case kWhatFlush:
143        {
144            onFlush(msg);
145            break;
146        }
147
148        case kWhatAudioSinkChanged:
149        {
150            onAudioSinkChanged();
151            break;
152        }
153
154        default:
155            TRESPASS();
156            break;
157    }
158}
159
160void NuPlayer::Renderer::postDrainAudioQueue() {
161    if (mDrainAudioQueuePending || mSyncQueues) {
162        return;
163    }
164
165    if (mAudioQueue.empty()) {
166        return;
167    }
168
169    mDrainAudioQueuePending = true;
170    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
171    msg->setInt32("generation", mAudioQueueGeneration);
172    msg->post(10000);
173}
174
175void NuPlayer::Renderer::signalAudioSinkChanged() {
176    (new AMessage(kWhatAudioSinkChanged, id()))->post();
177}
178
179void NuPlayer::Renderer::onDrainAudioQueue() {
180    uint32_t numFramesPlayed;
181    CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
182
183    ssize_t numFramesAvailableToWrite =
184        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
185
186    CHECK_GE(numFramesAvailableToWrite, 0);
187
188    size_t numBytesAvailableToWrite =
189        numFramesAvailableToWrite * mAudioSink->frameSize();
190
191    while (numBytesAvailableToWrite > 0) {
192        if (mAudioQueue.empty()) {
193            break;
194        }
195
196        QueueEntry *entry = &*mAudioQueue.begin();
197
198        if (entry->mBuffer == NULL) {
199            // EOS
200
201            notifyEOS(true /* audio */);
202
203            mAudioQueue.erase(mAudioQueue.begin());
204            entry = NULL;
205            return;
206        }
207
208        if (entry->mOffset == 0) {
209            int64_t mediaTimeUs;
210            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
211
212            LOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
213
214            mAnchorTimeMediaUs = mediaTimeUs;
215
216            uint32_t numFramesPlayed;
217            CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
218
219            uint32_t numFramesPendingPlayout =
220                mNumFramesWritten - numFramesPlayed;
221
222            int64_t realTimeOffsetUs =
223                (mAudioSink->latency() / 2  /* XXX */
224                    + numFramesPendingPlayout
225                        * mAudioSink->msecsPerFrame()) * 1000ll;
226
227            // LOGI("realTimeOffsetUs = %lld us", realTimeOffsetUs);
228
229            mAnchorTimeRealUs =
230                ALooper::GetNowUs() + realTimeOffsetUs;
231        }
232
233        size_t copy = entry->mBuffer->size() - entry->mOffset;
234        if (copy > numBytesAvailableToWrite) {
235            copy = numBytesAvailableToWrite;
236        }
237
238        CHECK_EQ(mAudioSink->write(
239                    entry->mBuffer->data() + entry->mOffset, copy),
240                 (ssize_t)copy);
241
242        entry->mOffset += copy;
243        if (entry->mOffset == entry->mBuffer->size()) {
244            entry->mNotifyConsumed->post();
245            mAudioQueue.erase(mAudioQueue.begin());
246            entry = NULL;
247        }
248
249        numBytesAvailableToWrite -= copy;
250        mNumFramesWritten += copy / mAudioSink->frameSize();
251    }
252
253    notifyPosition();
254}
255
256void NuPlayer::Renderer::postDrainVideoQueue() {
257    if (mDrainVideoQueuePending || mSyncQueues) {
258        return;
259    }
260
261    if (mVideoQueue.empty()) {
262        return;
263    }
264
265    QueueEntry &entry = *mVideoQueue.begin();
266
267    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
268    msg->setInt32("generation", mVideoQueueGeneration);
269
270    int64_t delayUs;
271
272    if (entry.mBuffer == NULL) {
273        // EOS doesn't carry a timestamp.
274        delayUs = 0;
275    } else {
276        int64_t mediaTimeUs;
277        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
278
279        if (mAnchorTimeMediaUs < 0) {
280            delayUs = 0;
281
282            if (!mHasAudio) {
283                mAnchorTimeMediaUs = mediaTimeUs;
284                mAnchorTimeRealUs = ALooper::GetNowUs();
285            }
286        } else {
287            int64_t realTimeUs =
288                (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
289
290            delayUs = realTimeUs - ALooper::GetNowUs();
291        }
292    }
293
294    msg->post(delayUs);
295
296    mDrainVideoQueuePending = true;
297}
298
299void NuPlayer::Renderer::onDrainVideoQueue() {
300    if (mVideoQueue.empty()) {
301        return;
302    }
303
304    QueueEntry *entry = &*mVideoQueue.begin();
305
306    if (entry->mBuffer == NULL) {
307        // EOS
308
309        notifyEOS(false /* audio */);
310
311        mVideoQueue.erase(mVideoQueue.begin());
312        entry = NULL;
313        return;
314    }
315
316#if 0
317    int64_t mediaTimeUs;
318    CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
319
320    LOGI("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
321#endif
322
323    entry->mNotifyConsumed->setInt32("render", true);
324    entry->mNotifyConsumed->post();
325    mVideoQueue.erase(mVideoQueue.begin());
326    entry = NULL;
327
328    notifyPosition();
329}
330
331void NuPlayer::Renderer::notifyEOS(bool audio) {
332    sp<AMessage> notify = mNotify->dup();
333    notify->setInt32("what", kWhatEOS);
334    notify->setInt32("audio", static_cast<int32_t>(audio));
335    notify->post();
336}
337
338void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
339    int32_t audio;
340    CHECK(msg->findInt32("audio", &audio));
341
342    if (dropBufferWhileFlushing(audio, msg)) {
343        return;
344    }
345
346    sp<RefBase> obj;
347    CHECK(msg->findObject("buffer", &obj));
348    sp<ABuffer> buffer = static_cast<ABuffer *>(obj.get());
349
350    sp<AMessage> notifyConsumed;
351    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
352
353    QueueEntry entry;
354    entry.mBuffer = buffer;
355    entry.mNotifyConsumed = notifyConsumed;
356    entry.mOffset = 0;
357    entry.mFinalResult = OK;
358
359    if (audio) {
360        mAudioQueue.push_back(entry);
361        postDrainAudioQueue();
362    } else {
363        mVideoQueue.push_back(entry);
364        postDrainVideoQueue();
365    }
366
367    if (mSyncQueues && !mAudioQueue.empty() && !mVideoQueue.empty()) {
368        int64_t firstAudioTimeUs;
369        int64_t firstVideoTimeUs;
370        CHECK((*mAudioQueue.begin()).mBuffer->meta()
371                ->findInt64("timeUs", &firstAudioTimeUs));
372        CHECK((*mVideoQueue.begin()).mBuffer->meta()
373                ->findInt64("timeUs", &firstVideoTimeUs));
374
375        int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
376
377        LOGV("queueDiff = %.2f secs", diff / 1E6);
378
379        if (diff > 100000ll) {
380            // Audio data starts More than 0.1 secs before video.
381            // Drop some audio.
382
383            (*mAudioQueue.begin()).mNotifyConsumed->post();
384            mAudioQueue.erase(mAudioQueue.begin());
385            return;
386        }
387
388        syncQueuesDone();
389    }
390}
391
392void NuPlayer::Renderer::syncQueuesDone() {
393    if (!mSyncQueues) {
394        return;
395    }
396
397    mSyncQueues = false;
398
399    if (!mAudioQueue.empty()) {
400        postDrainAudioQueue();
401    }
402
403    if (!mVideoQueue.empty()) {
404        postDrainVideoQueue();
405    }
406}
407
408void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
409    int32_t audio;
410    CHECK(msg->findInt32("audio", &audio));
411
412    if (dropBufferWhileFlushing(audio, msg)) {
413        return;
414    }
415
416    int32_t finalResult;
417    CHECK(msg->findInt32("finalResult", &finalResult));
418
419    QueueEntry entry;
420    entry.mOffset = 0;
421    entry.mFinalResult = finalResult;
422
423    if (audio) {
424        mAudioQueue.push_back(entry);
425        postDrainAudioQueue();
426    } else {
427        mVideoQueue.push_back(entry);
428        postDrainVideoQueue();
429    }
430}
431
432void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
433    int32_t audio;
434    CHECK(msg->findInt32("audio", &audio));
435
436    // If we're currently syncing the queues, i.e. dropping audio while
437    // aligning the first audio/video buffer times and only one of the
438    // two queues has data, we may starve that queue by not requesting
439    // more buffers from the decoder. If the other source then encounters
440    // a discontinuity that leads to flushing, we'll never find the
441    // corresponding discontinuity on the other queue.
442    // Therefore we'll stop syncing the queues if at least one of them
443    // is flushed.
444    syncQueuesDone();
445
446    if (audio) {
447        flushQueue(&mAudioQueue);
448
449        Mutex::Autolock autoLock(mFlushLock);
450        mFlushingAudio = false;
451
452        mDrainAudioQueuePending = false;
453        ++mAudioQueueGeneration;
454    } else {
455        flushQueue(&mVideoQueue);
456
457        Mutex::Autolock autoLock(mFlushLock);
458        mFlushingVideo = false;
459
460        mDrainVideoQueuePending = false;
461        ++mVideoQueueGeneration;
462    }
463
464    notifyFlushComplete(audio);
465}
466
467void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
468    while (!queue->empty()) {
469        QueueEntry *entry = &*queue->begin();
470
471        if (entry->mBuffer != NULL) {
472            entry->mNotifyConsumed->post();
473        }
474
475        queue->erase(queue->begin());
476        entry = NULL;
477    }
478}
479
480void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
481    sp<AMessage> notify = mNotify->dup();
482    notify->setInt32("what", kWhatFlushComplete);
483    notify->setInt32("audio", static_cast<int32_t>(audio));
484    notify->post();
485}
486
487bool NuPlayer::Renderer::dropBufferWhileFlushing(
488        bool audio, const sp<AMessage> &msg) {
489    bool flushing = false;
490
491    {
492        Mutex::Autolock autoLock(mFlushLock);
493        if (audio) {
494            flushing = mFlushingAudio;
495        } else {
496            flushing = mFlushingVideo;
497        }
498    }
499
500    if (!flushing) {
501        return false;
502    }
503
504    sp<AMessage> notifyConsumed;
505    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
506        notifyConsumed->post();
507    }
508
509    return true;
510}
511
512void NuPlayer::Renderer::onAudioSinkChanged() {
513    CHECK(!mDrainAudioQueuePending);
514    mNumFramesWritten = 0;
515}
516
517void NuPlayer::Renderer::notifyPosition() {
518    if (mAnchorTimeRealUs < 0 || mAnchorTimeMediaUs < 0) {
519        return;
520    }
521
522    int64_t nowUs = ALooper::GetNowUs();
523    int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs;
524
525    sp<AMessage> notify = mNotify->dup();
526    notify->setInt32("what", kWhatPosition);
527    notify->setInt64("positionUs", positionUs);
528    notify->post();
529}
530
531}  // namespace android
532
533