1/*
2 * Copyright 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "MediaSync"
19#include <inttypes.h>
20
21#include <gui/BufferQueue.h>
22#include <gui/IGraphicBufferConsumer.h>
23#include <gui/IGraphicBufferProducer.h>
24
25#include <media/AudioTrack.h>
26#include <media/stagefright/MediaClock.h>
27#include <media/stagefright/MediaSync.h>
28#include <media/stagefright/VideoFrameScheduler.h>
29#include <media/stagefright/foundation/ADebug.h>
30#include <media/stagefright/foundation/ALooper.h>
31#include <media/stagefright/foundation/AMessage.h>
32
33#include <ui/GraphicBuffer.h>
34
35// Maximum late time allowed for a video frame to be rendered. When a video
36// frame arrives later than this number, it will be discarded without rendering.
37static const int64_t kMaxAllowedVideoLateTimeUs = 40000ll;
38
39namespace android {
40
41// static
42sp<MediaSync> MediaSync::create() {
43    sp<MediaSync> sync = new MediaSync();
44    sync->mLooper->registerHandler(sync);
45    return sync;
46}
47
48MediaSync::MediaSync()
49      : mIsAbandoned(false),
50        mMutex(),
51        mReleaseCondition(),
52        mNumOutstandingBuffers(0),
53        mUsageFlagsFromOutput(0),
54        mMaxAcquiredBufferCount(1),
55        mReturnPendingInputFrame(false),
56        mNativeSampleRateInHz(0),
57        mNumFramesWritten(0),
58        mHasAudio(false),
59        mNextBufferItemMediaUs(-1),
60        mPlaybackRate(0.0) {
61    mMediaClock = new MediaClock;
62
63    // initialize settings
64    mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT;
65    mPlaybackSettings.mSpeed = mPlaybackRate;
66
67    mLooper = new ALooper;
68    mLooper->setName("MediaSync");
69    mLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
70}
71
72MediaSync::~MediaSync() {
73    if (mInput != NULL) {
74        mInput->consumerDisconnect();
75    }
76    if (mOutput != NULL) {
77        mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
78    }
79
80    if (mLooper != NULL) {
81        mLooper->unregisterHandler(id());
82        mLooper->stop();
83    }
84}
85
86status_t MediaSync::setSurface(const sp<IGraphicBufferProducer> &output) {
87    Mutex::Autolock lock(mMutex);
88
89    if (output == mOutput) {
90        return NO_ERROR;  // same output surface.
91    }
92
93    if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) {
94        ALOGE("setSurface: output surface is used as sync source and cannot be removed.");
95        return INVALID_OPERATION;
96    }
97
98    if (output != NULL) {
99        int newUsage = 0;
100        output->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &newUsage);
101
102        // Check usage flags only when current output surface has been used to create input surface.
103        if (mOutput != NULL && mInput != NULL) {
104            int ignoredFlags = (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER
105                    | GRALLOC_USAGE_EXTERNAL_DISP);
106            // New output surface is not allowed to add new usage flag except ignored ones.
107            if ((newUsage & ~(mUsageFlagsFromOutput | ignoredFlags)) != 0) {
108                ALOGE("setSurface: new output surface has new usage flag not used by current one.");
109                return BAD_VALUE;
110            }
111        }
112
113        // Try to connect to new output surface. If failed, current output surface will not
114        // be changed.
115        IGraphicBufferProducer::QueueBufferOutput queueBufferOutput;
116        sp<OutputListener> listener(new OutputListener(this, output));
117        IInterface::asBinder(output)->linkToDeath(listener);
118        status_t status =
119            output->connect(listener,
120                            NATIVE_WINDOW_API_MEDIA,
121                            true /* producerControlledByApp */,
122                            &queueBufferOutput);
123        if (status != NO_ERROR) {
124            ALOGE("setSurface: failed to connect (%d)", status);
125            return status;
126        }
127
128        if (mFrameScheduler == NULL) {
129            mFrameScheduler = new VideoFrameScheduler();
130            mFrameScheduler->init();
131        }
132    }
133
134    if (mOutput != NULL) {
135        mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
136        while (!mBuffersSentToOutput.isEmpty()) {
137            returnBufferToInput_l(mBuffersSentToOutput.valueAt(0), Fence::NO_FENCE);
138            mBuffersSentToOutput.removeItemsAt(0);
139        }
140    }
141
142    mOutput = output;
143
144    return NO_ERROR;
145}
146
147// |audioTrack| is used only for querying information.
148status_t MediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) {
149    Mutex::Autolock lock(mMutex);
150
151    // TODO: support audio track change.
152    if (mAudioTrack != NULL) {
153        ALOGE("setAudioTrack: audioTrack has already been configured.");
154        return INVALID_OPERATION;
155    }
156
157    if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) {
158        ALOGE("setAudioTrack: audioTrack is used as sync source and cannot be removed.");
159        return INVALID_OPERATION;
160    }
161
162    if (audioTrack != NULL) {
163        // check if audio track supports the playback settings
164        if (mPlaybackSettings.mSpeed != 0.f
165                && audioTrack->setPlaybackRate(mPlaybackSettings) != OK) {
166            ALOGE("playback settings are not supported by the audio track");
167            return INVALID_OPERATION;
168        }
169        uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate();
170        if (nativeSampleRateInHz <= 0) {
171            ALOGE("setAudioTrack: native sample rate should be positive.");
172            return BAD_VALUE;
173        }
174        mAudioTrack = audioTrack;
175        mNativeSampleRateInHz = nativeSampleRateInHz;
176        (void)setPlaybackSettings_l(mPlaybackSettings);
177    }
178    else {
179        mAudioTrack = NULL;
180        mNativeSampleRateInHz = 0;
181    }
182
183    // potentially resync to new source
184    resync_l();
185    return OK;
186}
187
188status_t MediaSync::createInputSurface(
189        sp<IGraphicBufferProducer> *outBufferProducer) {
190    if (outBufferProducer == NULL) {
191        return BAD_VALUE;
192    }
193
194    Mutex::Autolock lock(mMutex);
195
196    if (mOutput == NULL) {
197        return NO_INIT;
198    }
199
200    if (mInput != NULL) {
201        return INVALID_OPERATION;
202    }
203
204    sp<IGraphicBufferProducer> bufferProducer;
205    sp<IGraphicBufferConsumer> bufferConsumer;
206    BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer);
207
208    sp<InputListener> listener(new InputListener(this));
209    IInterface::asBinder(bufferConsumer)->linkToDeath(listener);
210    status_t status =
211        bufferConsumer->consumerConnect(listener, false /* controlledByApp */);
212    if (status == NO_ERROR) {
213        bufferConsumer->setConsumerName(String8("MediaSync"));
214        // propagate usage bits from output surface
215        mUsageFlagsFromOutput = 0;
216        mOutput->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &mUsageFlagsFromOutput);
217        bufferConsumer->setConsumerUsageBits(mUsageFlagsFromOutput);
218        *outBufferProducer = bufferProducer;
219        mInput = bufferConsumer;
220
221        // set undequeued buffer count
222        int minUndequeuedBuffers;
223        mOutput->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers);
224        mMaxAcquiredBufferCount = minUndequeuedBuffers;
225        bufferConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount);
226    }
227    return status;
228}
229
230void MediaSync::resync_l() {
231    AVSyncSource src = mSyncSettings.mSource;
232    if (src == AVSYNC_SOURCE_DEFAULT) {
233        if (mAudioTrack != NULL) {
234            src = AVSYNC_SOURCE_AUDIO;
235        } else {
236            src = AVSYNC_SOURCE_SYSTEM_CLOCK;
237        }
238    }
239
240    // TODO: resync ourselves to the current clock (e.g. on sync source change)
241    updatePlaybackRate_l(mPlaybackRate);
242}
243
244void MediaSync::updatePlaybackRate_l(float rate) {
245    if (rate > mPlaybackRate) {
246        mNextBufferItemMediaUs = -1;
247    }
248    mPlaybackRate = rate;
249    // TODO: update frame scheduler with this info
250    mMediaClock->setPlaybackRate(rate);
251    onDrainVideo_l();
252}
253
254sp<const MediaClock> MediaSync::getMediaClock() {
255    return mMediaClock;
256}
257
258status_t MediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) {
259    Mutex::Autolock lock(mMutex);
260    // User should check the playback rate if it doesn't want to receive a
261    // huge number for play time.
262    if (mPlaybackRate == 0.0f) {
263        *outTimeUs = INT64_MAX;
264        return OK;
265    }
266
267    uint32_t numFramesPlayed = 0;
268    if (mAudioTrack != NULL) {
269        status_t res = mAudioTrack->getPosition(&numFramesPlayed);
270        if (res != OK) {
271            return res;
272        }
273    }
274
275    int64_t numPendingFrames = mNumFramesWritten - numFramesPlayed;
276    if (numPendingFrames < 0) {
277        numPendingFrames = 0;
278        ALOGW("getPlayTimeForPendingAudioFrames: pending frame count is negative.");
279    }
280    double timeUs = numPendingFrames * 1000000.0
281            / (mNativeSampleRateInHz * (double)mPlaybackRate);
282    if (timeUs > (double)INT64_MAX) {
283        // Overflow.
284        *outTimeUs = INT64_MAX;
285        ALOGW("getPlayTimeForPendingAudioFrames: play time for pending audio frames "
286              "is too high, possibly due to super low playback rate(%f)", mPlaybackRate);
287    } else {
288        *outTimeUs = (int64_t)timeUs;
289    }
290
291    return OK;
292}
293
294status_t MediaSync::updateQueuedAudioData(
295        size_t sizeInBytes, int64_t presentationTimeUs) {
296    if (sizeInBytes == 0) {
297        return OK;
298    }
299
300    Mutex::Autolock lock(mMutex);
301
302    if (mAudioTrack == NULL) {
303        ALOGW("updateQueuedAudioData: audioTrack has NOT been configured.");
304        return INVALID_OPERATION;
305    }
306
307    int64_t numFrames = sizeInBytes / mAudioTrack->frameSize();
308    int64_t maxMediaTimeUs = presentationTimeUs
309            + getDurationIfPlayedAtNativeSampleRate_l(numFrames);
310
311    int64_t nowUs = ALooper::GetNowUs();
312    int64_t nowMediaUs = presentationTimeUs
313            - getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten)
314            + getPlayedOutAudioDurationMedia_l(nowUs);
315
316    mNumFramesWritten += numFrames;
317
318    int64_t oldRealTime = -1;
319    if (mNextBufferItemMediaUs != -1) {
320        oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
321    }
322
323    mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs);
324    mHasAudio = true;
325
326    if (oldRealTime != -1) {
327        int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
328        if (newRealTime >= oldRealTime) {
329            return OK;
330        }
331    }
332
333    mNextBufferItemMediaUs = -1;
334    onDrainVideo_l();
335    return OK;
336}
337
338void MediaSync::setName(const AString &name) {
339    Mutex::Autolock lock(mMutex);
340    mInput->setConsumerName(String8(name.c_str()));
341}
342
343void MediaSync::flush() {
344    Mutex::Autolock lock(mMutex);
345    if (mFrameScheduler != NULL) {
346        mFrameScheduler->restart();
347    }
348    while (!mBufferItems.empty()) {
349        BufferItem *bufferItem = &*mBufferItems.begin();
350        returnBufferToInput_l(bufferItem->mGraphicBuffer, bufferItem->mFence);
351        mBufferItems.erase(mBufferItems.begin());
352    }
353    mNextBufferItemMediaUs = -1;
354    mNumFramesWritten = 0;
355    mReturnPendingInputFrame = true;
356    mReleaseCondition.signal();
357    mMediaClock->clearAnchor();
358}
359
360status_t MediaSync::setVideoFrameRateHint(float rate) {
361    Mutex::Autolock lock(mMutex);
362    if (rate < 0.f) {
363        return BAD_VALUE;
364    }
365    if (mFrameScheduler != NULL) {
366        mFrameScheduler->init(rate);
367    }
368    return OK;
369}
370
371float MediaSync::getVideoFrameRate() {
372    Mutex::Autolock lock(mMutex);
373    if (mFrameScheduler != NULL) {
374        float fps = mFrameScheduler->getFrameRate();
375        if (fps > 0.f) {
376            return fps;
377        }
378    }
379
380    // we don't have or know the frame rate
381    return -1.f;
382}
383
384status_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) {
385    // validate settings
386    if (syncSettings.mSource >= AVSYNC_SOURCE_MAX
387            || syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX
388            || syncSettings.mTolerance < 0.f
389            || syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) {
390        return BAD_VALUE;
391    }
392
393    Mutex::Autolock lock(mMutex);
394
395    // verify that we have the sync source
396    switch (syncSettings.mSource) {
397        case AVSYNC_SOURCE_AUDIO:
398            if (mAudioTrack == NULL) {
399                ALOGE("setSyncSettings: audio sync source requires an audio track");
400                return BAD_VALUE;
401            }
402            break;
403        case AVSYNC_SOURCE_VSYNC:
404            if (mOutput == NULL) {
405                ALOGE("setSyncSettings: vsync sync source requires an output surface");
406                return BAD_VALUE;
407            }
408            break;
409        default:
410            break;
411    }
412
413    mSyncSettings = syncSettings;
414    resync_l();
415    return OK;
416}
417
418void MediaSync::getSyncSettings(AVSyncSettings *syncSettings) {
419    Mutex::Autolock lock(mMutex);
420    *syncSettings = mSyncSettings;
421}
422
423status_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) {
424    Mutex::Autolock lock(mMutex);
425
426    status_t err = setPlaybackSettings_l(rate);
427    if (err == OK) {
428        // TODO: adjust rate if using VSYNC as source
429        updatePlaybackRate_l(rate.mSpeed);
430    }
431    return err;
432}
433
434status_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) {
435    if (rate.mSpeed < 0.f || rate.mPitch < 0.f) {
436        // We don't validate other audio settings.
437        // They will be validated when/if audiotrack is set.
438        return BAD_VALUE;
439    }
440
441    if (mAudioTrack != NULL) {
442        if (rate.mSpeed == 0.f) {
443            mAudioTrack->pause();
444        } else {
445            status_t err = mAudioTrack->setPlaybackRate(rate);
446            if (err != OK) {
447                return BAD_VALUE;
448            }
449
450            // ignore errors
451            (void)mAudioTrack->start();
452        }
453    }
454    mPlaybackSettings = rate;
455    return OK;
456}
457
458void MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) {
459    Mutex::Autolock lock(mMutex);
460    *rate = mPlaybackSettings;
461}
462
463int64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) {
464    int64_t realUs;
465    if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
466        // If failed to get current position, e.g. due to audio clock is
467        // not ready, then just play out video immediately without delay.
468        return nowUs;
469    }
470    return realUs;
471}
472
473int64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) {
474    return (numFrames * 1000000LL / mNativeSampleRateInHz);
475}
476
477int64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) {
478    CHECK(mAudioTrack != NULL);
479
480    uint32_t numFramesPlayed;
481    int64_t numFramesPlayedAt;
482    AudioTimestamp ts;
483    static const int64_t kStaleTimestamp100ms = 100000;
484
485    status_t res = mAudioTrack->getTimestamp(ts);
486    if (res == OK) {
487        // case 1: mixing audio tracks.
488        numFramesPlayed = ts.mPosition;
489        numFramesPlayedAt =
490            ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
491        const int64_t timestampAge = nowUs - numFramesPlayedAt;
492        if (timestampAge > kStaleTimestamp100ms) {
493            // This is an audio FIXME.
494            // getTimestamp returns a timestamp which may come from audio
495            // mixing threads. After pausing, the MixerThread may go idle,
496            // thus the mTime estimate may become stale. Assuming that the
497            // MixerThread runs 20ms, with FastMixer at 5ms, the max latency
498            // should be about 25ms with an average around 12ms (to be
499            // verified). For safety we use 100ms.
500            ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) "
501                  "numFramesPlayedAt(%lld)",
502                  (long long)nowUs, (long long)numFramesPlayedAt);
503            numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
504        }
505        //ALOGD("getTimestamp: OK %d %lld",
506        //      numFramesPlayed, (long long)numFramesPlayedAt);
507    } else if (res == WOULD_BLOCK) {
508        // case 2: transitory state on start of a new track
509        numFramesPlayed = 0;
510        numFramesPlayedAt = nowUs;
511        //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
512        //      numFramesPlayed, (long long)numFramesPlayedAt);
513    } else {
514        // case 3: transitory at new track or audio fast tracks.
515        res = mAudioTrack->getPosition(&numFramesPlayed);
516        CHECK_EQ(res, (status_t)OK);
517        numFramesPlayedAt = nowUs;
518        numFramesPlayedAt += 1000LL * mAudioTrack->latency() / 2; /* XXX */
519        //ALOGD("getPosition: %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
520    }
521
522    //can't be negative until 12.4 hrs, test.
523    //CHECK_EQ(numFramesPlayed & (1 << 31), 0);
524    int64_t durationUs =
525        getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed)
526            + nowUs - numFramesPlayedAt;
527    if (durationUs < 0) {
528        // Occurs when numFramesPlayed position is very small and the following:
529        // (1) In case 1, the time nowUs is computed before getTimestamp() is
530        //     called and numFramesPlayedAt is greater than nowUs by time more
531        //     than numFramesPlayed.
532        // (2) In case 3, using getPosition and adding mAudioTrack->latency()
533        //     to numFramesPlayedAt, by a time amount greater than
534        //     numFramesPlayed.
535        //
536        // Both of these are transitory conditions.
537        ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld "
538              "set to zero", (long long)durationUs);
539        durationUs = 0;
540    }
541    ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) "
542          "framesAt(%lld)",
543          (long long)durationUs, (long long)nowUs, numFramesPlayed,
544          (long long)numFramesPlayedAt);
545    return durationUs;
546}
547
548void MediaSync::onDrainVideo_l() {
549    if (!isPlaying()) {
550        return;
551    }
552
553    while (!mBufferItems.empty()) {
554        int64_t nowUs = ALooper::GetNowUs();
555        BufferItem *bufferItem = &*mBufferItems.begin();
556        int64_t itemMediaUs = bufferItem->mTimestamp / 1000;
557        int64_t itemRealUs = getRealTime(itemMediaUs, nowUs);
558
559        // adjust video frame PTS based on vsync
560        itemRealUs = mFrameScheduler->schedule(itemRealUs * 1000) / 1000;
561        int64_t twoVsyncsUs = 2 * (mFrameScheduler->getVsyncPeriod() / 1000);
562
563        // post 2 display refreshes before rendering is due
564        if (itemRealUs <= nowUs + twoVsyncsUs) {
565            ALOGV("adjusting PTS from %lld to %lld",
566                    (long long)bufferItem->mTimestamp / 1000, (long long)itemRealUs);
567            bufferItem->mTimestamp = itemRealUs * 1000;
568            bufferItem->mIsAutoTimestamp = false;
569
570            if (mHasAudio) {
571                if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) {
572                    renderOneBufferItem_l(*bufferItem);
573                } else {
574                    // too late.
575                    returnBufferToInput_l(
576                            bufferItem->mGraphicBuffer, bufferItem->mFence);
577                    mFrameScheduler->restart();
578                }
579            } else {
580                // always render video buffer in video-only mode.
581                renderOneBufferItem_l(*bufferItem);
582
583                // smooth out videos >= 10fps
584                mMediaClock->updateAnchor(
585                        itemMediaUs, nowUs, itemMediaUs + 100000);
586            }
587
588            mBufferItems.erase(mBufferItems.begin());
589            mNextBufferItemMediaUs = -1;
590        } else {
591            if (mNextBufferItemMediaUs == -1
592                    || mNextBufferItemMediaUs > itemMediaUs) {
593                sp<AMessage> msg = new AMessage(kWhatDrainVideo, this);
594                msg->post(itemRealUs - nowUs - twoVsyncsUs);
595                mNextBufferItemMediaUs = itemMediaUs;
596            }
597            break;
598        }
599    }
600}
601
602void MediaSync::onFrameAvailableFromInput() {
603    Mutex::Autolock lock(mMutex);
604
605    const static nsecs_t kAcquireWaitTimeout = 2000000000; // 2 seconds
606
607    mReturnPendingInputFrame = false;
608
609    // If there are too many outstanding buffers, wait until a buffer is
610    // released back to the input in onBufferReleased.
611    // NOTE: BufferQueue allows dequeuing maxAcquiredBufferCount + 1 buffers
612    while (mNumOutstandingBuffers > mMaxAcquiredBufferCount
613            && !mIsAbandoned && !mReturnPendingInputFrame) {
614        if (mReleaseCondition.waitRelative(mMutex, kAcquireWaitTimeout) != OK) {
615            ALOGI_IF(mPlaybackRate != 0.f, "still waiting to release a buffer before acquire");
616        }
617
618        // If the sync is abandoned while we are waiting, the release
619        // condition variable will be broadcast, and we should just return
620        // without attempting to do anything more (since the input queue will
621        // also be abandoned).
622        if (mIsAbandoned) {
623            return;
624        }
625    }
626
627    // Acquire and detach the buffer from the input.
628    BufferItem bufferItem;
629    status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */);
630    if (status != NO_ERROR) {
631        ALOGE("acquiring buffer from input failed (%d)", status);
632        return;
633    }
634    ++mNumOutstandingBuffers;
635
636    ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId());
637
638    status = mInput->detachBuffer(bufferItem.mSlot);
639    if (status != NO_ERROR) {
640        ALOGE("detaching buffer from input failed (%d)", status);
641        if (status == NO_INIT) {
642            // If the input has been abandoned, move on.
643            onAbandoned_l(true /* isInput */);
644        }
645        return;
646    }
647
648    if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
649        // Something is wrong since this buffer should be at our hands, bail.
650        ALOGE("received buffer multiple times from input");
651        mInput->consumerDisconnect();
652        onAbandoned_l(true /* isInput */);
653        return;
654    }
655    mBuffersFromInput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
656
657    // If flush happened while waiting for a buffer to be released, simply return it
658    // TRICKY: do it here after it is detached so that we don't have to cache mGraphicBuffer.
659    if (mReturnPendingInputFrame) {
660        mReturnPendingInputFrame = false;
661        returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
662        return;
663    }
664
665    mBufferItems.push_back(bufferItem);
666
667    if (mBufferItems.size() == 1) {
668        onDrainVideo_l();
669    }
670}
671
672void MediaSync::renderOneBufferItem_l(const BufferItem &bufferItem) {
673    IGraphicBufferProducer::QueueBufferInput queueInput(
674            bufferItem.mTimestamp,
675            bufferItem.mIsAutoTimestamp,
676            bufferItem.mDataSpace,
677            bufferItem.mCrop,
678            static_cast<int32_t>(bufferItem.mScalingMode),
679            bufferItem.mTransform,
680            bufferItem.mFence);
681
682    // Attach and queue the buffer to the output.
683    int slot;
684    mOutput->setGenerationNumber(bufferItem.mGraphicBuffer->getGenerationNumber());
685    status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer);
686    ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status);
687    if (status == NO_ERROR) {
688        IGraphicBufferProducer::QueueBufferOutput queueOutput;
689        status = mOutput->queueBuffer(slot, queueInput, &queueOutput);
690        ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status);
691    }
692
693    if (status != NO_ERROR) {
694        returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
695        if (status == NO_INIT) {
696            // If the output has been abandoned, move on.
697            onAbandoned_l(false /* isInput */);
698        }
699        return;
700    }
701
702    if (mBuffersSentToOutput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
703        // Something is wrong since this buffer should be held by output now, bail.
704        mInput->consumerDisconnect();
705        onAbandoned_l(true /* isInput */);
706        return;
707    }
708    mBuffersSentToOutput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
709
710    ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId());
711}
712
713void MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) {
714    Mutex::Autolock lock(mMutex);
715
716    if (output != mOutput) {
717        return;  // This is not the current output, ignore.
718    }
719
720    sp<GraphicBuffer> buffer;
721    sp<Fence> fence;
722    status_t status = mOutput->detachNextBuffer(&buffer, &fence);
723    ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status);
724
725    if (status == NO_INIT) {
726        // If the output has been abandoned, we can't do anything else,
727        // since buffer is invalid.
728        onAbandoned_l(false /* isInput */);
729        return;
730    }
731
732    ALOGV("detached buffer %#llx from output", (long long)buffer->getId());
733
734    // If we've been abandoned, we can't return the buffer to the input, so just
735    // move on.
736    if (mIsAbandoned) {
737        return;
738    }
739
740    ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId());
741    if (ix < 0) {
742        // The buffer is unknown, maybe leftover, ignore.
743        return;
744    }
745    mBuffersSentToOutput.removeItemsAt(ix);
746
747    returnBufferToInput_l(buffer, fence);
748}
749
750void MediaSync::returnBufferToInput_l(
751        const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) {
752    ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId());
753    if (ix < 0) {
754        // The buffer is unknown, something is wrong, bail.
755        ALOGE("output returned unknown buffer");
756        mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
757        onAbandoned_l(false /* isInput */);
758        return;
759    }
760    sp<GraphicBuffer> oldBuffer = mBuffersFromInput.valueAt(ix);
761    mBuffersFromInput.removeItemsAt(ix);
762
763    // Attach and release the buffer back to the input.
764    int consumerSlot;
765    status_t status = mInput->attachBuffer(&consumerSlot, oldBuffer);
766    ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status);
767    if (status == NO_ERROR) {
768        status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */,
769                EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
770        ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status);
771    }
772
773    // Notify any waiting onFrameAvailable calls.
774    --mNumOutstandingBuffers;
775    mReleaseCondition.signal();
776
777    if (status == NO_ERROR) {
778        ALOGV("released buffer %#llx to input", (long long)oldBuffer->getId());
779    }
780}
781
782void MediaSync::onAbandoned_l(bool isInput) {
783    ALOGE("the %s has abandoned me", (isInput ? "input" : "output"));
784    if (!mIsAbandoned) {
785        if (isInput) {
786            mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
787        } else {
788            mInput->consumerDisconnect();
789        }
790        mIsAbandoned = true;
791    }
792    mReleaseCondition.broadcast();
793}
794
795void MediaSync::onMessageReceived(const sp<AMessage> &msg) {
796    switch (msg->what()) {
797        case kWhatDrainVideo:
798        {
799            Mutex::Autolock lock(mMutex);
800            if (mNextBufferItemMediaUs != -1) {
801                int64_t nowUs = ALooper::GetNowUs();
802                int64_t itemRealUs = getRealTime(mNextBufferItemMediaUs, nowUs);
803
804                // The message could arrive earlier than expected due to
805                // various reasons, e.g., media clock has been changed because
806                // of new anchor time or playback rate. In such cases, the
807                // message needs to be re-posted.
808                if (itemRealUs > nowUs) {
809                    msg->post(itemRealUs - nowUs);
810                    break;
811                }
812            }
813
814            onDrainVideo_l();
815            break;
816        }
817
818        default:
819            TRESPASS();
820            break;
821    }
822}
823
824MediaSync::InputListener::InputListener(const sp<MediaSync> &sync)
825      : mSync(sync) {}
826
827MediaSync::InputListener::~InputListener() {}
828
829void MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) {
830    mSync->onFrameAvailableFromInput();
831}
832
833// We don't care about sideband streams, since we won't relay them.
834void MediaSync::InputListener::onSidebandStreamChanged() {
835    ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly.");
836}
837
838
839void MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) {
840    Mutex::Autolock lock(mSync->mMutex);
841    mSync->onAbandoned_l(true /* isInput */);
842}
843
844MediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync,
845        const sp<IGraphicBufferProducer> &output)
846      : mSync(sync),
847        mOutput(output) {}
848
849MediaSync::OutputListener::~OutputListener() {}
850
851void MediaSync::OutputListener::onBufferReleased() {
852    mSync->onBufferReleasedByOutput(mOutput);
853}
854
855void MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) {
856    Mutex::Autolock lock(mSync->mMutex);
857    mSync->onAbandoned_l(false /* isInput */);
858}
859
860} // namespace android
861