NuPlayerRenderer.cpp revision f933441648ef6a71dee783d733aac17b9508b452 
1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
25
26namespace android {
27
28NuPlayer::Renderer::Renderer(
29        const sp<MediaPlayerBase::AudioSink> &sink,
30        const sp<AMessage> &notify)
31    : mAudioSink(sink),
32      mNotify(notify),
33      mNumFramesWritten(0),
34      mDrainAudioQueuePending(false),
35      mDrainVideoQueuePending(false),
36      mAudioQueueGeneration(0),
37      mVideoQueueGeneration(0),
38      mAnchorTimeMediaUs(-1),
39      mAnchorTimeRealUs(-1),
40      mFlushingAudio(false),
41      mFlushingVideo(false),
42      mSyncQueues(true) {
43}
44
45NuPlayer::Renderer::~Renderer() {
46}
47
48void NuPlayer::Renderer::queueBuffer(
49        bool audio,
50        const sp<ABuffer> &buffer,
51        const sp<AMessage> &notifyConsumed) {
52    sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
53    msg->setInt32("audio", static_cast<int32_t>(audio));
54    msg->setObject("buffer", buffer);
55    msg->setMessage("notifyConsumed", notifyConsumed);
56    msg->post();
57}
58
59void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
60    CHECK_NE(finalResult, (status_t)OK);
61
62    sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
63    msg->setInt32("audio", static_cast<int32_t>(audio));
64    msg->setInt32("finalResult", finalResult);
65    msg->post();
66}
67
68void NuPlayer::Renderer::flush(bool audio) {
69    {
70        Mutex::Autolock autoLock(mFlushLock);
71        if (audio) {
72            CHECK(!mFlushingAudio);
73            mFlushingAudio = true;
74        } else {
75            CHECK(!mFlushingVideo);
76            mFlushingVideo = true;
77        }
78    }
79
80    sp<AMessage> msg = new AMessage(kWhatFlush, id());
81    msg->setInt32("audio", static_cast<int32_t>(audio));
82    msg->post();
83}
84
85void NuPlayer::Renderer::signalTimeDiscontinuity() {
86    CHECK(mAudioQueue.empty());
87    CHECK(mVideoQueue.empty());
88    mAnchorTimeMediaUs = -1;
89    mAnchorTimeRealUs = -1;
90    mSyncQueues = true;
91}
92
93void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
94    switch (msg->what()) {
95        case kWhatDrainAudioQueue:
96        {
97            int32_t generation;
98            CHECK(msg->findInt32("generation", &generation));
99            if (generation != mAudioQueueGeneration) {
100                break;
101            }
102
103            mDrainAudioQueuePending = false;
104
105            onDrainAudioQueue();
106
107            postDrainAudioQueue();
108            break;
109        }
110
111        case kWhatDrainVideoQueue:
112        {
113            int32_t generation;
114            CHECK(msg->findInt32("generation", &generation));
115            if (generation != mVideoQueueGeneration) {
116                break;
117            }
118
119            mDrainVideoQueuePending = false;
120
121            onDrainVideoQueue();
122
123            postDrainVideoQueue();
124            break;
125        }
126
127        case kWhatQueueBuffer:
128        {
129            onQueueBuffer(msg);
130            break;
131        }
132
133        case kWhatQueueEOS:
134        {
135            onQueueEOS(msg);
136            break;
137        }
138
139        case kWhatFlush:
140        {
141            onFlush(msg);
142            break;
143        }
144
145        default:
146            TRESPASS();
147            break;
148    }
149}
150
151void NuPlayer::Renderer::postDrainAudioQueue() {
152    if (mDrainAudioQueuePending || mSyncQueues) {
153        return;
154    }
155
156    if (mAudioQueue.empty()) {
157        return;
158    }
159
160    mDrainAudioQueuePending = true;
161    sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
162    msg->setInt32("generation", mAudioQueueGeneration);
163    msg->post(10000);
164}
165
166void NuPlayer::Renderer::onDrainAudioQueue() {
167    uint32_t numFramesPlayed;
168    CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
169
170    ssize_t numFramesAvailableToWrite =
171        mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
172
173    CHECK_GE(numFramesAvailableToWrite, 0);
174
175    size_t numBytesAvailableToWrite =
176        numFramesAvailableToWrite * mAudioSink->frameSize();
177
178    while (numBytesAvailableToWrite > 0) {
179        if (mAudioQueue.empty()) {
180            break;
181        }
182
183        QueueEntry *entry = &*mAudioQueue.begin();
184
185        if (entry->mBuffer == NULL) {
186            // EOS
187
188            notifyEOS(true /* audio */);
189
190            mAudioQueue.erase(mAudioQueue.begin());
191            entry = NULL;
192            return;
193        }
194
195        if (entry->mOffset == 0) {
196            int64_t mediaTimeUs;
197            CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
198
199            LOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
200
201            mAnchorTimeMediaUs = mediaTimeUs;
202
203            uint32_t numFramesPlayed;
204            CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
205
206            uint32_t numFramesPendingPlayout =
207                mNumFramesWritten - numFramesPlayed;
208
209            int64_t realTimeOffsetUs =
210                (mAudioSink->latency() / 2  /* XXX */
211                    + numFramesPendingPlayout
212                        * mAudioSink->msecsPerFrame()) * 1000ll;
213
214            // LOGI("realTimeOffsetUs = %lld us", realTimeOffsetUs);
215
216            mAnchorTimeRealUs =
217                ALooper::GetNowUs() + realTimeOffsetUs;
218        }
219
220        size_t copy = entry->mBuffer->size() - entry->mOffset;
221        if (copy > numBytesAvailableToWrite) {
222            copy = numBytesAvailableToWrite;
223        }
224
225        CHECK_EQ(mAudioSink->write(
226                    entry->mBuffer->data() + entry->mOffset, copy),
227                 (ssize_t)copy);
228
229        entry->mOffset += copy;
230        if (entry->mOffset == entry->mBuffer->size()) {
231            entry->mNotifyConsumed->post();
232            mAudioQueue.erase(mAudioQueue.begin());
233            entry = NULL;
234        }
235
236        numBytesAvailableToWrite -= copy;
237        mNumFramesWritten += copy / mAudioSink->frameSize();
238    }
239}
240
241void NuPlayer::Renderer::postDrainVideoQueue() {
242    if (mDrainVideoQueuePending || mSyncQueues) {
243        return;
244    }
245
246    if (mVideoQueue.empty()) {
247        return;
248    }
249
250    QueueEntry &entry = *mVideoQueue.begin();
251
252    sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
253    msg->setInt32("generation", mVideoQueueGeneration);
254
255    int64_t delayUs;
256
257    if (entry.mBuffer == NULL) {
258        // EOS doesn't carry a timestamp.
259        delayUs = 0;
260    } else {
261        int64_t mediaTimeUs;
262        CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
263
264        if (mAnchorTimeMediaUs < 0) {
265            delayUs = 0;
266
267            if (mAudioSink == NULL) {
268                mAnchorTimeMediaUs = mediaTimeUs;
269                mAnchorTimeRealUs = ALooper::GetNowUs();
270            }
271        } else {
272            int64_t realTimeUs =
273                (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
274
275            delayUs = realTimeUs - ALooper::GetNowUs();
276        }
277    }
278
279    msg->post(delayUs);
280
281    mDrainVideoQueuePending = true;
282}
283
284void NuPlayer::Renderer::onDrainVideoQueue() {
285    if (mVideoQueue.empty()) {
286        return;
287    }
288
289    QueueEntry *entry = &*mVideoQueue.begin();
290
291    if (entry->mBuffer == NULL) {
292        // EOS
293
294        notifyEOS(false /* audio */);
295
296        mVideoQueue.erase(mVideoQueue.begin());
297        entry = NULL;
298        return;
299    }
300
301#if 0
302    int64_t mediaTimeUs;
303    CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
304
305    LOGI("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
306#endif
307
308    entry->mNotifyConsumed->setInt32("render", true);
309    entry->mNotifyConsumed->post();
310    mVideoQueue.erase(mVideoQueue.begin());
311    entry = NULL;
312}
313
314void NuPlayer::Renderer::notifyEOS(bool audio) {
315    sp<AMessage> notify = mNotify->dup();
316    notify->setInt32("what", kWhatEOS);
317    notify->setInt32("audio", static_cast<int32_t>(audio));
318    notify->post();
319}
320
321void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
322    int32_t audio;
323    CHECK(msg->findInt32("audio", &audio));
324
325    if (dropBufferWhileFlushing(audio, msg)) {
326        return;
327    }
328
329    sp<RefBase> obj;
330    CHECK(msg->findObject("buffer", &obj));
331    sp<ABuffer> buffer = static_cast<ABuffer *>(obj.get());
332
333    sp<AMessage> notifyConsumed;
334    CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
335
336    QueueEntry entry;
337    entry.mBuffer = buffer;
338    entry.mNotifyConsumed = notifyConsumed;
339    entry.mOffset = 0;
340    entry.mFinalResult = OK;
341
342    if (audio) {
343        mAudioQueue.push_back(entry);
344        postDrainAudioQueue();
345    } else {
346        mVideoQueue.push_back(entry);
347        postDrainVideoQueue();
348    }
349
350    if (mSyncQueues && !mAudioQueue.empty() && !mVideoQueue.empty()) {
351        int64_t firstAudioTimeUs;
352        int64_t firstVideoTimeUs;
353        CHECK((*mAudioQueue.begin()).mBuffer->meta()
354                ->findInt64("timeUs", &firstAudioTimeUs));
355        CHECK((*mVideoQueue.begin()).mBuffer->meta()
356                ->findInt64("timeUs", &firstVideoTimeUs));
357
358        int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
359
360        LOGV("queueDiff = %.2f secs", diff / 1E6);
361
362        if (diff > 100000ll) {
363            // Audio data starts More than 0.1 secs before video.
364            // Drop some audio.
365
366            (*mAudioQueue.begin()).mNotifyConsumed->post();
367            mAudioQueue.erase(mAudioQueue.begin());
368            return;
369        }
370
371        syncQueuesDone();
372    }
373}
374
375void NuPlayer::Renderer::syncQueuesDone() {
376    if (!mSyncQueues) {
377        return;
378    }
379
380    mSyncQueues = false;
381
382    if (!mAudioQueue.empty()) {
383        postDrainAudioQueue();
384    }
385
386    if (!mVideoQueue.empty()) {
387        postDrainVideoQueue();
388    }
389}
390
391void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
392    int32_t audio;
393    CHECK(msg->findInt32("audio", &audio));
394
395    if (dropBufferWhileFlushing(audio, msg)) {
396        return;
397    }
398
399    int32_t finalResult;
400    CHECK(msg->findInt32("finalResult", &finalResult));
401
402    QueueEntry entry;
403    entry.mOffset = 0;
404    entry.mFinalResult = finalResult;
405
406    if (audio) {
407        mAudioQueue.push_back(entry);
408        postDrainAudioQueue();
409    } else {
410        mVideoQueue.push_back(entry);
411        postDrainVideoQueue();
412    }
413}
414
415void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
416    int32_t audio;
417    CHECK(msg->findInt32("audio", &audio));
418
419    // If we're currently syncing the queues, i.e. dropping audio while
420    // aligning the first audio/video buffer times and only one of the
421    // two queues has data, we may starve that queue by not requesting
422    // more buffers from the decoder. If the other source then encounters
423    // a discontinuity that leads to flushing, we'll never find the
424    // corresponding discontinuity on the other queue.
425    // Therefore we'll stop syncing the queues if at least one of them
426    // is flushed.
427    syncQueuesDone();
428
429    if (audio) {
430        flushQueue(&mAudioQueue);
431
432        Mutex::Autolock autoLock(mFlushLock);
433        mFlushingAudio = false;
434
435        mDrainAudioQueuePending = false;
436        ++mAudioQueueGeneration;
437    } else {
438        flushQueue(&mVideoQueue);
439
440        Mutex::Autolock autoLock(mFlushLock);
441        mFlushingVideo = false;
442
443        mDrainVideoQueuePending = false;
444        ++mVideoQueueGeneration;
445    }
446
447    notifyFlushComplete(audio);
448}
449
450void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
451    while (!queue->empty()) {
452        QueueEntry *entry = &*queue->begin();
453
454        if (entry->mBuffer != NULL) {
455            entry->mNotifyConsumed->post();
456        }
457
458        queue->erase(queue->begin());
459        entry = NULL;
460    }
461}
462
463void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
464    sp<AMessage> notify = mNotify->dup();
465    notify->setInt32("what", kWhatFlushComplete);
466    notify->setInt32("audio", static_cast<int32_t>(audio));
467    notify->post();
468}
469
470bool NuPlayer::Renderer::dropBufferWhileFlushing(
471        bool audio, const sp<AMessage> &msg) {
472    bool flushing = false;
473
474    {
475        Mutex::Autolock autoLock(mFlushLock);
476        if (audio) {
477            flushing = mFlushingAudio;
478        } else {
479            flushing = mFlushingVideo;
480        }
481    }
482
483    if (!flushing) {
484        return false;
485    }
486
487    sp<AMessage> notifyConsumed;
488    if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
489        notifyConsumed->post();
490    }
491
492    return true;
493}
494
495}  // namespace android
496
497