NuPlayerRenderer.cpp revision 5bc087c573c70c84c6a39946457590b42d392a33 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25#include <media/stagefright/foundation/AMessage.h>
26
27namespace android {
28
29NuPlayer::Renderer::Renderer(
30        const sp<MediaPlayerBase::AudioSink> &sink,
31        const sp<AMessage> &notify)
32    : mAudioSink(sink),
33      mNotify(notify),
34      mNumFramesWritten(0),
35      mDrainAudioQueuePending(false),
36      mDrainVideoQueuePending(false),
37      mAudioQueueGeneration(0),
38      mVideoQueueGeneration(0),
39      mAnchorTimeMediaUs(-1),
40      mAnchorTimeRealUs(-1),
41      mFlushingAudio(false),
42      mFlushingVideo(false),
43      mHasAudio(mAudioSink != NULL),
44      mHasVideo(true),
45      mSyncQueues(mHasAudio && mHasVideo) {
46}
47
48NuPlayer::Renderer::~Renderer() {
49}
50
51void NuPlayer::Renderer::queueBuffer(
52        bool audio,
53        const sp<ABuffer> &buffer,
54        const sp<AMessage> &notifyConsumed) {
55    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
56    msg->setInt32("audio", static_cast<int32_t>(audio));
57    msg->setObject("buffer", buffer);
58    msg->setMessage("notifyConsumed", notifyConsumed);
59    msg->post();
60}
61
62void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
63    CHECK_NE(finalResult, (status_t)OK);
64
65    sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
66    msg->setInt32("audio", static_cast<int32_t>(audio));
67    msg->setInt32("finalResult", finalResult);
68    msg->post();
69}
70
71void NuPlayer::Renderer::flush(bool audio) {
72    {
73        Mutex::Autolock autoLock(mFlushLock);
74        if (audio) {
75            CHECK(!mFlushingAudio);
76            mFlushingAudio = true;
77        } else {
78            CHECK(!mFlushingVideo);
79            mFlushingVideo = true;
80        }
81    }
82
83    sp<AMessage> msg = new AMessage(kWhatFlush, id());
84    msg->setInt32("audio", static_cast<int32_t>(audio));
85    msg->post();
86}
87
88void NuPlayer::Renderer::signalTimeDiscontinuity() {
89    CHECK(mAudioQueue.empty());
90    CHECK(mVideoQueue.empty());
91    mAnchorTimeMediaUs = -1;
92    mAnchorTimeRealUs = -1;
93    mSyncQueues = mHasAudio && mHasVideo;
94}
95
96void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
97    switch (msg->what()) {
98        case kWhatDrainAudioQueue:
99        {
100            int32_t generation;
101            CHECK(msg->findInt32("generation", &generation));
102            if (generation != mAudioQueueGeneration) {
103                break;
104            }
105
106            mDrainAudioQueuePending = false;
107
108            onDrainAudioQueue();
109
110            postDrainAudioQueue();
111            break;
112        }
113
114        case kWhatDrainVideoQueue:
115        {
116            int32_t generation;
117            CHECK(msg->findInt32("generation", &generation));
118            if (generation != mVideoQueueGeneration) {
119                break;
120            }
121
122            mDrainVideoQueuePending = false;
123
124            onDrainVideoQueue();
125
126            postDrainVideoQueue();
127            break;
128        }
129
130        case kWhatQueueBuffer:
131        {
132            onQueueBuffer(msg);
133            break;
134        }
135
136        case kWhatQueueEOS:
137        {
138            onQueueEOS(msg);
139            break;
140        }
141
142        case kWhatFlush:
143        {
144            onFlush(msg);
145            break;
146        }
147
148        case kWhatAudioSinkChanged:
149        {
150            onAudioSinkChanged();
151            break;
152        }
153
154        default:
155            TRESPASS();
156            break;
157    }
158}
159
160void NuPlayer::Renderer::postDrainAudioQueue() {
161    if (mDrainAudioQueuePending || mSyncQueues) {
162        return;
163    }
164
165    if (mAudioQueue.empty()) {
166        return;
167    }
168
169    mDrainAudioQueuePending = true;
170    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
171    msg->setInt32("generation", mAudioQueueGeneration);
172    msg->post(10000);
173}
174
175void NuPlayer::Renderer::signalAudioSinkChanged() {
176    (new AMessage(kWhatAudioSinkChanged, id()))->post();
177}
178
179void NuPlayer::Renderer::onDrainAudioQueue() {
180    uint32_t numFramesPlayed;
181    CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
182
183    ssize_t numFramesAvailableToWrite =
184        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
185
186    CHECK_GE(numFramesAvailableToWrite, 0);
187
188    size_t numBytesAvailableToWrite =
189        numFramesAvailableToWrite * mAudioSink->frameSize();
190
191    while (numBytesAvailableToWrite > 0) {
192        if (mAudioQueue.empty()) {
193            break;
194        }
195
196        QueueEntry *entry = &*mAudioQueue.begin();
197
198        if (entry->mBuffer == NULL) {
199            // EOS
200
201            notifyEOS(true /* audio */);
202
203            mAudioQueue.erase(mAudioQueue.begin());
204            entry = NULL;
205            return;
206        }
207
208        if (entry->mOffset == 0) {
209            int64_t mediaTimeUs;
210            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
211
212            LOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
213
214            mAnchorTimeMediaUs = mediaTimeUs;
215
216            uint32_t numFramesPlayed;
217            CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
218
219            uint32_t numFramesPendingPlayout =
220                mNumFramesWritten - numFramesPlayed;
221
222            int64_t realTimeOffsetUs =
223                (mAudioSink->latency() / 2  /* XXX */
224                    + numFramesPendingPlayout
225                        * mAudioSink->msecsPerFrame()) * 1000ll;
226
227            // LOGI("realTimeOffsetUs = %lld us", realTimeOffsetUs);
228
229            mAnchorTimeRealUs =
230                ALooper::GetNowUs() + realTimeOffsetUs;
231        }
232
233        size_t copy = entry->mBuffer->size() - entry->mOffset;
234        if (copy > numBytesAvailableToWrite) {
235            copy = numBytesAvailableToWrite;
236        }
237
238        CHECK_EQ(mAudioSink->write(
239                    entry->mBuffer->data() + entry->mOffset, copy),
240                 (ssize_t)copy);
241
242        entry->mOffset += copy;
243        if (entry->mOffset == entry->mBuffer->size()) {
244            entry->mNotifyConsumed->post();
245            mAudioQueue.erase(mAudioQueue.begin());
246            entry = NULL;
247        }
248
249        numBytesAvailableToWrite -= copy;
250        mNumFramesWritten += copy / mAudioSink->frameSize();
251    }
252}
253
254void NuPlayer::Renderer::postDrainVideoQueue() {
255    if (mDrainVideoQueuePending || mSyncQueues) {
256        return;
257    }
258
259    if (mVideoQueue.empty()) {
260        return;
261    }
262
263    QueueEntry &entry = *mVideoQueue.begin();
264
265    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
266    msg->setInt32("generation", mVideoQueueGeneration);
267
268    int64_t delayUs;
269
270    if (entry.mBuffer == NULL) {
271        // EOS doesn't carry a timestamp.
272        delayUs = 0;
273    } else {
274        int64_t mediaTimeUs;
275        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
276
277        if (mAnchorTimeMediaUs < 0) {
278            delayUs = 0;
279
280            if (!mHasAudio) {
281                mAnchorTimeMediaUs = mediaTimeUs;
282                mAnchorTimeRealUs = ALooper::GetNowUs();
283            }
284        } else {
285            int64_t realTimeUs =
286                (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
287
288            delayUs = realTimeUs - ALooper::GetNowUs();
289        }
290    }
291
292    msg->post(delayUs);
293
294    mDrainVideoQueuePending = true;
295}
296
297void NuPlayer::Renderer::onDrainVideoQueue() {
298    if (mVideoQueue.empty()) {
299        return;
300    }
301
302    QueueEntry *entry = &*mVideoQueue.begin();
303
304    if (entry->mBuffer == NULL) {
305        // EOS
306
307        notifyEOS(false /* audio */);
308
309        mVideoQueue.erase(mVideoQueue.begin());
310        entry = NULL;
311        return;
312    }
313
314#if 0
315    int64_t mediaTimeUs;
316    CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
317
318    LOGI("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
319#endif
320
321    entry->mNotifyConsumed->setInt32("render", true);
322    entry->mNotifyConsumed->post();
323    mVideoQueue.erase(mVideoQueue.begin());
324    entry = NULL;
325}
326
327void NuPlayer::Renderer::notifyEOS(bool audio) {
328    sp<AMessage> notify = mNotify->dup();
329    notify->setInt32("what", kWhatEOS);
330    notify->setInt32("audio", static_cast<int32_t>(audio));
331    notify->post();
332}
333
334void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
335    int32_t audio;
336    CHECK(msg->findInt32("audio", &audio));
337
338    if (dropBufferWhileFlushing(audio, msg)) {
339        return;
340    }
341
342    sp<RefBase> obj;
343    CHECK(msg->findObject("buffer", &obj));
344    sp<ABuffer> buffer = static_cast<ABuffer *>(obj.get());
345
346    sp<AMessage> notifyConsumed;
347    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
348
349    QueueEntry entry;
350    entry.mBuffer = buffer;
351    entry.mNotifyConsumed = notifyConsumed;
352    entry.mOffset = 0;
353    entry.mFinalResult = OK;
354
355    if (audio) {
356        mAudioQueue.push_back(entry);
357        postDrainAudioQueue();
358    } else {
359        mVideoQueue.push_back(entry);
360        postDrainVideoQueue();
361    }
362
363    if (mSyncQueues && !mAudioQueue.empty() && !mVideoQueue.empty()) {
364        int64_t firstAudioTimeUs;
365        int64_t firstVideoTimeUs;
366        CHECK((*mAudioQueue.begin()).mBuffer->meta()
367                ->findInt64("timeUs", &firstAudioTimeUs));
368        CHECK((*mVideoQueue.begin()).mBuffer->meta()
369                ->findInt64("timeUs", &firstVideoTimeUs));
370
371        int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
372
373        LOGV("queueDiff = %.2f secs", diff / 1E6);
374
375        if (diff > 100000ll) {
376            // Audio data starts More than 0.1 secs before video.
377            // Drop some audio.
378
379            (*mAudioQueue.begin()).mNotifyConsumed->post();
380            mAudioQueue.erase(mAudioQueue.begin());
381            return;
382        }
383
384        syncQueuesDone();
385    }
386}
387
388void NuPlayer::Renderer::syncQueuesDone() {
389    if (!mSyncQueues) {
390        return;
391    }
392
393    mSyncQueues = false;
394
395    if (!mAudioQueue.empty()) {
396        postDrainAudioQueue();
397    }
398
399    if (!mVideoQueue.empty()) {
400        postDrainVideoQueue();
401    }
402}
403
404void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
405    int32_t audio;
406    CHECK(msg->findInt32("audio", &audio));
407
408    if (dropBufferWhileFlushing(audio, msg)) {
409        return;
410    }
411
412    int32_t finalResult;
413    CHECK(msg->findInt32("finalResult", &finalResult));
414
415    QueueEntry entry;
416    entry.mOffset = 0;
417    entry.mFinalResult = finalResult;
418
419    if (audio) {
420        mAudioQueue.push_back(entry);
421        postDrainAudioQueue();
422    } else {
423        mVideoQueue.push_back(entry);
424        postDrainVideoQueue();
425    }
426}
427
428void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
429    int32_t audio;
430    CHECK(msg->findInt32("audio", &audio));
431
432    // If we're currently syncing the queues, i.e. dropping audio while
433    // aligning the first audio/video buffer times and only one of the
434    // two queues has data, we may starve that queue by not requesting
435    // more buffers from the decoder. If the other source then encounters
436    // a discontinuity that leads to flushing, we'll never find the
437    // corresponding discontinuity on the other queue.
438    // Therefore we'll stop syncing the queues if at least one of them
439    // is flushed.
440    syncQueuesDone();
441
442    if (audio) {
443        flushQueue(&mAudioQueue);
444
445        Mutex::Autolock autoLock(mFlushLock);
446        mFlushingAudio = false;
447
448        mDrainAudioQueuePending = false;
449        ++mAudioQueueGeneration;
450    } else {
451        flushQueue(&mVideoQueue);
452
453        Mutex::Autolock autoLock(mFlushLock);
454        mFlushingVideo = false;
455
456        mDrainVideoQueuePending = false;
457        ++mVideoQueueGeneration;
458    }
459
460    notifyFlushComplete(audio);
461}
462
463void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
464    while (!queue->empty()) {
465        QueueEntry *entry = &*queue->begin();
466
467        if (entry->mBuffer != NULL) {
468            entry->mNotifyConsumed->post();
469        }
470
471        queue->erase(queue->begin());
472        entry = NULL;
473    }
474}
475
476void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
477    sp<AMessage> notify = mNotify->dup();
478    notify->setInt32("what", kWhatFlushComplete);
479    notify->setInt32("audio", static_cast<int32_t>(audio));
480    notify->post();
481}
482
483bool NuPlayer::Renderer::dropBufferWhileFlushing(
484        bool audio, const sp<AMessage> &msg) {
485    bool flushing = false;
486
487    {
488        Mutex::Autolock autoLock(mFlushLock);
489        if (audio) {
490            flushing = mFlushingAudio;
491        } else {
492            flushing = mFlushingVideo;
493        }
494    }
495
496    if (!flushing) {
497        return false;
498    }
499
500    sp<AMessage> notifyConsumed;
501    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
502        notifyConsumed->post();
503    }
504
505    return true;
506}
507
508void NuPlayer::Renderer::onAudioSinkChanged() {
509    CHECK(!mDrainAudioQueuePending);
510    mNumFramesWritten = 0;
511}
512
513}  // namespace android
514
515