1/*
2 * Copyright 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "MediaSync"
19#include <inttypes.h>
20
21#include <gui/BufferQueue.h>
22#include <gui/IGraphicBufferConsumer.h>
23#include <gui/IGraphicBufferProducer.h>
24
25#include <media/AudioTrack.h>
26#include <media/stagefright/MediaClock.h>
27#include <media/stagefright/MediaSync.h>
28#include <media/stagefright/VideoFrameScheduler.h>
29#include <media/stagefright/foundation/ADebug.h>
30#include <media/stagefright/foundation/ALooper.h>
31#include <media/stagefright/foundation/AMessage.h>
32
33#include <ui/GraphicBuffer.h>
34
35// Maximum late time allowed for a video frame to be rendered. When a video
36// frame arrives later than this number, it will be discarded without rendering.
37static const int64_t kMaxAllowedVideoLateTimeUs = 40000ll;
38
39namespace android {
40
41// static
42sp<MediaSync> MediaSync::create() {
43    sp<MediaSync> sync = new MediaSync();
44    sync->mLooper->registerHandler(sync);
45    return sync;
46}
47
48MediaSync::MediaSync()
49      : mIsAbandoned(false),
50        mMutex(),
51        mReleaseCondition(),
52        mNumOutstandingBuffers(0),
53        mUsageFlagsFromOutput(0),
54        mMaxAcquiredBufferCount(1),
55        mReturnPendingInputFrame(false),
56        mNativeSampleRateInHz(0),
57        mNumFramesWritten(0),
58        mHasAudio(false),
59        mNextBufferItemMediaUs(-1),
60        mPlaybackRate(0.0) {
61    mMediaClock = new MediaClock;
62
63    // initialize settings
64    mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT;
65    mPlaybackSettings.mSpeed = mPlaybackRate;
66
67    mLooper = new ALooper;
68    mLooper->setName("MediaSync");
69    mLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
70}
71
72MediaSync::~MediaSync() {
73    if (mInput != NULL) {
74        mInput->consumerDisconnect();
75    }
76    if (mOutput != NULL) {
77        mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
78    }
79
80    if (mLooper != NULL) {
81        mLooper->unregisterHandler(id());
82        mLooper->stop();
83    }
84}
85
86status_t MediaSync::setSurface(const sp<IGraphicBufferProducer> &output) {
87    Mutex::Autolock lock(mMutex);
88
89    if (output == mOutput) {
90        return NO_ERROR;  // same output surface.
91    }
92
93    if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) {
94        ALOGE("setSurface: output surface is used as sync source and cannot be removed.");
95        return INVALID_OPERATION;
96    }
97
98    if (output != NULL) {
99        int newUsage = 0;
100        output->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &newUsage);
101
102        // Check usage flags only when current output surface has been used to create input surface.
103        if (mOutput != NULL && mInput != NULL) {
104            int ignoredFlags = (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER
105                    | GRALLOC_USAGE_EXTERNAL_DISP);
106            // New output surface is not allowed to add new usage flag except ignored ones.
107            if ((newUsage & ~(mUsageFlagsFromOutput | ignoredFlags)) != 0) {
108                ALOGE("setSurface: new output surface has new usage flag not used by current one.");
109                return BAD_VALUE;
110            }
111        }
112
113        // Try to connect to new output surface. If failed, current output surface will not
114        // be changed.
115        IGraphicBufferProducer::QueueBufferOutput queueBufferOutput;
116        sp<OutputListener> listener(new OutputListener(this, output));
117        IInterface::asBinder(output)->linkToDeath(listener);
118        status_t status =
119            output->connect(listener,
120                            NATIVE_WINDOW_API_MEDIA,
121                            true /* producerControlledByApp */,
122                            &queueBufferOutput);
123        if (status != NO_ERROR) {
124            ALOGE("setSurface: failed to connect (%d)", status);
125            return status;
126        }
127
128        if (mFrameScheduler == NULL) {
129            mFrameScheduler = new VideoFrameScheduler();
130            mFrameScheduler->init();
131        }
132    }
133
134    if (mOutput != NULL) {
135        mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
136        while (!mBuffersSentToOutput.isEmpty()) {
137            returnBufferToInput_l(mBuffersSentToOutput.valueAt(0), Fence::NO_FENCE);
138            mBuffersSentToOutput.removeItemsAt(0);
139        }
140    }
141
142    mOutput = output;
143
144    return NO_ERROR;
145}
146
147// |audioTrack| is used only for querying information.
148status_t MediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) {
149    Mutex::Autolock lock(mMutex);
150
151    // TODO: support audio track change.
152    if (mAudioTrack != NULL) {
153        ALOGE("setAudioTrack: audioTrack has already been configured.");
154        return INVALID_OPERATION;
155    }
156
157    if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) {
158        ALOGE("setAudioTrack: audioTrack is used as sync source and cannot be removed.");
159        return INVALID_OPERATION;
160    }
161
162    if (audioTrack != NULL) {
163        // check if audio track supports the playback settings
164        if (mPlaybackSettings.mSpeed != 0.f
165                && audioTrack->setPlaybackRate(mPlaybackSettings) != OK) {
166            ALOGE("playback settings are not supported by the audio track");
167            return INVALID_OPERATION;
168        }
169        uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate();
170        if (nativeSampleRateInHz <= 0) {
171            ALOGE("setAudioTrack: native sample rate should be positive.");
172            return BAD_VALUE;
173        }
174        mAudioTrack = audioTrack;
175        mNativeSampleRateInHz = nativeSampleRateInHz;
176        (void)setPlaybackSettings_l(mPlaybackSettings);
177    }
178    else {
179        mAudioTrack = NULL;
180        mNativeSampleRateInHz = 0;
181    }
182
183    // potentially resync to new source
184    resync_l();
185    return OK;
186}
187
188status_t MediaSync::createInputSurface(
189        sp<IGraphicBufferProducer> *outBufferProducer) {
190    if (outBufferProducer == NULL) {
191        return BAD_VALUE;
192    }
193
194    Mutex::Autolock lock(mMutex);
195
196    if (mOutput == NULL) {
197        return NO_INIT;
198    }
199
200    if (mInput != NULL) {
201        return INVALID_OPERATION;
202    }
203
204    sp<IGraphicBufferProducer> bufferProducer;
205    sp<IGraphicBufferConsumer> bufferConsumer;
206    BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer);
207
208    sp<InputListener> listener(new InputListener(this));
209    IInterface::asBinder(bufferConsumer)->linkToDeath(listener);
210    status_t status =
211        bufferConsumer->consumerConnect(listener, false /* controlledByApp */);
212    if (status == NO_ERROR) {
213        bufferConsumer->setConsumerName(String8("MediaSync"));
214        // propagate usage bits from output surface
215        mUsageFlagsFromOutput = 0;
216        mOutput->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &mUsageFlagsFromOutput);
217        bufferConsumer->setConsumerUsageBits(mUsageFlagsFromOutput);
218        *outBufferProducer = bufferProducer;
219        mInput = bufferConsumer;
220
221        // set undequeued buffer count
222        int minUndequeuedBuffers;
223        mOutput->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers);
224        mMaxAcquiredBufferCount = minUndequeuedBuffers;
225        bufferConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount);
226    }
227    return status;
228}
229
230void MediaSync::resync_l() {
231    AVSyncSource src = mSyncSettings.mSource;
232    if (src == AVSYNC_SOURCE_DEFAULT) {
233        if (mAudioTrack != NULL) {
234            src = AVSYNC_SOURCE_AUDIO;
235        } else {
236            src = AVSYNC_SOURCE_SYSTEM_CLOCK;
237        }
238    }
239
240    // TODO: resync ourselves to the current clock (e.g. on sync source change)
241    updatePlaybackRate_l(mPlaybackRate);
242}
243
244void MediaSync::updatePlaybackRate_l(float rate) {
245    if (rate > mPlaybackRate) {
246        mNextBufferItemMediaUs = -1;
247    }
248    mPlaybackRate = rate;
249    // TODO: update frame scheduler with this info
250    mMediaClock->setPlaybackRate(rate);
251    onDrainVideo_l();
252}
253
254sp<const MediaClock> MediaSync::getMediaClock() {
255    return mMediaClock;
256}
257
258status_t MediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) {
259    Mutex::Autolock lock(mMutex);
260    // User should check the playback rate if it doesn't want to receive a
261    // huge number for play time.
262    if (mPlaybackRate == 0.0f) {
263        *outTimeUs = INT64_MAX;
264        return OK;
265    }
266
267    uint32_t numFramesPlayed = 0;
268    if (mAudioTrack != NULL) {
269        status_t res = mAudioTrack->getPosition(&numFramesPlayed);
270        if (res != OK) {
271            return res;
272        }
273    }
274
275    int64_t numPendingFrames = mNumFramesWritten - numFramesPlayed;
276    if (numPendingFrames < 0) {
277        numPendingFrames = 0;
278        ALOGW("getPlayTimeForPendingAudioFrames: pending frame count is negative.");
279    }
280    double timeUs = numPendingFrames * 1000000.0
281            / (mNativeSampleRateInHz * (double)mPlaybackRate);
282    if (timeUs > (double)INT64_MAX) {
283        // Overflow.
284        *outTimeUs = INT64_MAX;
285        ALOGW("getPlayTimeForPendingAudioFrames: play time for pending audio frames "
286              "is too high, possibly due to super low playback rate(%f)", mPlaybackRate);
287    } else {
288        *outTimeUs = (int64_t)timeUs;
289    }
290
291    return OK;
292}
293
294status_t MediaSync::updateQueuedAudioData(
295        size_t sizeInBytes, int64_t presentationTimeUs) {
296    if (sizeInBytes == 0) {
297        return OK;
298    }
299
300    Mutex::Autolock lock(mMutex);
301
302    if (mAudioTrack == NULL) {
303        ALOGW("updateQueuedAudioData: audioTrack has NOT been configured.");
304        return INVALID_OPERATION;
305    }
306
307    int64_t numFrames = sizeInBytes / mAudioTrack->frameSize();
308    int64_t maxMediaTimeUs = presentationTimeUs
309            + getDurationIfPlayedAtNativeSampleRate_l(numFrames);
310
311    int64_t nowUs = ALooper::GetNowUs();
312    int64_t nowMediaUs = presentationTimeUs
313            - getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten)
314            + getPlayedOutAudioDurationMedia_l(nowUs);
315
316    mNumFramesWritten += numFrames;
317
318    int64_t oldRealTime = -1;
319    if (mNextBufferItemMediaUs != -1) {
320        oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
321    }
322
323    mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs);
324    mHasAudio = true;
325
326    if (oldRealTime != -1) {
327        int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
328        if (newRealTime >= oldRealTime) {
329            return OK;
330        }
331    }
332
333    mNextBufferItemMediaUs = -1;
334    onDrainVideo_l();
335    return OK;
336}
337
338void MediaSync::setName(const AString &name) {
339    Mutex::Autolock lock(mMutex);
340    mInput->setConsumerName(String8(name.c_str()));
341}
342
343void MediaSync::flush() {
344    Mutex::Autolock lock(mMutex);
345    if (mFrameScheduler != NULL) {
346        mFrameScheduler->restart();
347    }
348    while (!mBufferItems.empty()) {
349        BufferItem *bufferItem = &*mBufferItems.begin();
350        returnBufferToInput_l(bufferItem->mGraphicBuffer, bufferItem->mFence);
351        mBufferItems.erase(mBufferItems.begin());
352    }
353    mNextBufferItemMediaUs = -1;
354    mNumFramesWritten = 0;
355    mReturnPendingInputFrame = true;
356    mReleaseCondition.signal();
357    mMediaClock->clearAnchor();
358}
359
360status_t MediaSync::setVideoFrameRateHint(float rate) {
361    Mutex::Autolock lock(mMutex);
362    if (rate < 0.f) {
363        return BAD_VALUE;
364    }
365    if (mFrameScheduler != NULL) {
366        mFrameScheduler->init(rate);
367    }
368    return OK;
369}
370
371float MediaSync::getVideoFrameRate() {
372    Mutex::Autolock lock(mMutex);
373    if (mFrameScheduler != NULL) {
374        float fps = mFrameScheduler->getFrameRate();
375        if (fps > 0.f) {
376            return fps;
377        }
378    }
379
380    // we don't have or know the frame rate
381    return -1.f;
382}
383
384status_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) {
385    // validate settings
386    if (syncSettings.mSource >= AVSYNC_SOURCE_MAX
387            || syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX
388            || syncSettings.mTolerance < 0.f
389            || syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) {
390        return BAD_VALUE;
391    }
392
393    Mutex::Autolock lock(mMutex);
394
395    // verify that we have the sync source
396    switch (syncSettings.mSource) {
397        case AVSYNC_SOURCE_AUDIO:
398            if (mAudioTrack == NULL) {
399                ALOGE("setSyncSettings: audio sync source requires an audio track");
400                return BAD_VALUE;
401            }
402            break;
403        case AVSYNC_SOURCE_VSYNC:
404            if (mOutput == NULL) {
405                ALOGE("setSyncSettings: vsync sync source requires an output surface");
406                return BAD_VALUE;
407            }
408            break;
409        default:
410            break;
411    }
412
413    mSyncSettings = syncSettings;
414    resync_l();
415    return OK;
416}
417
418void MediaSync::getSyncSettings(AVSyncSettings *syncSettings) {
419    Mutex::Autolock lock(mMutex);
420    *syncSettings = mSyncSettings;
421}
422
423status_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) {
424    Mutex::Autolock lock(mMutex);
425
426    status_t err = setPlaybackSettings_l(rate);
427    if (err == OK) {
428        // TODO: adjust rate if using VSYNC as source
429        updatePlaybackRate_l(rate.mSpeed);
430    }
431    return err;
432}
433
434status_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) {
435    if (rate.mSpeed < 0.f || rate.mPitch < 0.f) {
436        // We don't validate other audio settings.
437        // They will be validated when/if audiotrack is set.
438        return BAD_VALUE;
439    }
440
441    if (mAudioTrack != NULL) {
442        if (rate.mSpeed == 0.f) {
443            mAudioTrack->pause();
444        } else {
445            status_t err = mAudioTrack->setPlaybackRate(rate);
446            if (err != OK) {
447                return BAD_VALUE;
448            }
449
450            // ignore errors
451            (void)mAudioTrack->start();
452        }
453    }
454    mPlaybackSettings = rate;
455    return OK;
456}
457
458void MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) {
459    Mutex::Autolock lock(mMutex);
460    *rate = mPlaybackSettings;
461}
462
463int64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) {
464    int64_t realUs;
465    if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
466        // If failed to get current position, e.g. due to audio clock is
467        // not ready, then just play out video immediately without delay.
468        return nowUs;
469    }
470    return realUs;
471}
472
473int64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) {
474    return (numFrames * 1000000LL / mNativeSampleRateInHz);
475}
476
477int64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) {
478    CHECK(mAudioTrack != NULL);
479
480    uint32_t numFramesPlayed;
481    int64_t numFramesPlayedAtUs;
482    AudioTimestamp ts;
483
484    status_t res = mAudioTrack->getTimestamp(ts);
485    if (res == OK) {
486        // case 1: mixing audio tracks.
487        numFramesPlayed = ts.mPosition;
488        numFramesPlayedAtUs = ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
489        //ALOGD("getTimestamp: OK %d %lld",
490        //      numFramesPlayed, (long long)numFramesPlayedAtUs);
491    } else if (res == WOULD_BLOCK) {
492        // case 2: transitory state on start of a new track
493        numFramesPlayed = 0;
494        numFramesPlayedAtUs = nowUs;
495        //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
496        //      numFramesPlayed, (long long)numFramesPlayedAtUs);
497    } else {
498        // case 3: transitory at new track or audio fast tracks.
499        res = mAudioTrack->getPosition(&numFramesPlayed);
500        CHECK_EQ(res, (status_t)OK);
501        numFramesPlayedAtUs = nowUs;
502        numFramesPlayedAtUs += 1000LL * mAudioTrack->latency() / 2; /* XXX */
503        //ALOGD("getPosition: %d %lld", numFramesPlayed, (long long)numFramesPlayedAtUs);
504    }
505
506    //can't be negative until 12.4 hrs, test.
507    //CHECK_EQ(numFramesPlayed & (1 << 31), 0);
508    int64_t durationUs =
509        getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed)
510            + nowUs - numFramesPlayedAtUs;
511    if (durationUs < 0) {
512        // Occurs when numFramesPlayed position is very small and the following:
513        // (1) In case 1, the time nowUs is computed before getTimestamp() is
514        //     called and numFramesPlayedAtUs is greater than nowUs by time more
515        //     than numFramesPlayed.
516        // (2) In case 3, using getPosition and adding mAudioTrack->latency()
517        //     to numFramesPlayedAtUs, by a time amount greater than
518        //     numFramesPlayed.
519        //
520        // Both of these are transitory conditions.
521        ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld "
522              "set to zero", (long long)durationUs);
523        durationUs = 0;
524    }
525    ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) "
526          "framesAt(%lld)",
527          (long long)durationUs, (long long)nowUs, numFramesPlayed,
528          (long long)numFramesPlayedAtUs);
529    return durationUs;
530}
531
532void MediaSync::onDrainVideo_l() {
533    if (!isPlaying()) {
534        return;
535    }
536
537    while (!mBufferItems.empty()) {
538        int64_t nowUs = ALooper::GetNowUs();
539        BufferItem *bufferItem = &*mBufferItems.begin();
540        int64_t itemMediaUs = bufferItem->mTimestamp / 1000;
541        int64_t itemRealUs = getRealTime(itemMediaUs, nowUs);
542
543        // adjust video frame PTS based on vsync
544        itemRealUs = mFrameScheduler->schedule(itemRealUs * 1000) / 1000;
545        int64_t twoVsyncsUs = 2 * (mFrameScheduler->getVsyncPeriod() / 1000);
546
547        // post 2 display refreshes before rendering is due
548        if (itemRealUs <= nowUs + twoVsyncsUs) {
549            ALOGV("adjusting PTS from %lld to %lld",
550                    (long long)bufferItem->mTimestamp / 1000, (long long)itemRealUs);
551            bufferItem->mTimestamp = itemRealUs * 1000;
552            bufferItem->mIsAutoTimestamp = false;
553
554            if (mHasAudio) {
555                if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) {
556                    renderOneBufferItem_l(*bufferItem);
557                } else {
558                    // too late.
559                    returnBufferToInput_l(
560                            bufferItem->mGraphicBuffer, bufferItem->mFence);
561                    mFrameScheduler->restart();
562                }
563            } else {
564                // always render video buffer in video-only mode.
565                renderOneBufferItem_l(*bufferItem);
566
567                // smooth out videos >= 10fps
568                mMediaClock->updateAnchor(
569                        itemMediaUs, nowUs, itemMediaUs + 100000);
570            }
571
572            mBufferItems.erase(mBufferItems.begin());
573            mNextBufferItemMediaUs = -1;
574        } else {
575            if (mNextBufferItemMediaUs == -1
576                    || mNextBufferItemMediaUs > itemMediaUs) {
577                sp<AMessage> msg = new AMessage(kWhatDrainVideo, this);
578                msg->post(itemRealUs - nowUs - twoVsyncsUs);
579                mNextBufferItemMediaUs = itemMediaUs;
580            }
581            break;
582        }
583    }
584}
585
586void MediaSync::onFrameAvailableFromInput() {
587    Mutex::Autolock lock(mMutex);
588
589    const static nsecs_t kAcquireWaitTimeout = 2000000000; // 2 seconds
590
591    mReturnPendingInputFrame = false;
592
593    // If there are too many outstanding buffers, wait until a buffer is
594    // released back to the input in onBufferReleased.
595    // NOTE: BufferQueue allows dequeuing maxAcquiredBufferCount + 1 buffers
596    while (mNumOutstandingBuffers > mMaxAcquiredBufferCount
597            && !mIsAbandoned && !mReturnPendingInputFrame) {
598        if (mReleaseCondition.waitRelative(mMutex, kAcquireWaitTimeout) != OK) {
599            ALOGI_IF(mPlaybackRate != 0.f, "still waiting to release a buffer before acquire");
600        }
601
602        // If the sync is abandoned while we are waiting, the release
603        // condition variable will be broadcast, and we should just return
604        // without attempting to do anything more (since the input queue will
605        // also be abandoned).
606        if (mIsAbandoned) {
607            return;
608        }
609    }
610
611    // Acquire and detach the buffer from the input.
612    BufferItem bufferItem;
613    status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */);
614    if (status != NO_ERROR) {
615        ALOGE("acquiring buffer from input failed (%d)", status);
616        return;
617    }
618    ++mNumOutstandingBuffers;
619
620    ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId());
621
622    status = mInput->detachBuffer(bufferItem.mSlot);
623    if (status != NO_ERROR) {
624        ALOGE("detaching buffer from input failed (%d)", status);
625        if (status == NO_INIT) {
626            // If the input has been abandoned, move on.
627            onAbandoned_l(true /* isInput */);
628        }
629        return;
630    }
631
632    if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
633        // Something is wrong since this buffer should be at our hands, bail.
634        ALOGE("received buffer multiple times from input");
635        mInput->consumerDisconnect();
636        onAbandoned_l(true /* isInput */);
637        return;
638    }
639    mBuffersFromInput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
640
641    // If flush happened while waiting for a buffer to be released, simply return it
642    // TRICKY: do it here after it is detached so that we don't have to cache mGraphicBuffer.
643    if (mReturnPendingInputFrame) {
644        mReturnPendingInputFrame = false;
645        returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
646        return;
647    }
648
649    mBufferItems.push_back(bufferItem);
650
651    if (mBufferItems.size() == 1) {
652        onDrainVideo_l();
653    }
654}
655
656void MediaSync::renderOneBufferItem_l(const BufferItem &bufferItem) {
657    IGraphicBufferProducer::QueueBufferInput queueInput(
658            bufferItem.mTimestamp,
659            bufferItem.mIsAutoTimestamp,
660            bufferItem.mDataSpace,
661            bufferItem.mCrop,
662            static_cast<int32_t>(bufferItem.mScalingMode),
663            bufferItem.mTransform,
664            bufferItem.mFence);
665
666    // Attach and queue the buffer to the output.
667    int slot;
668    mOutput->setGenerationNumber(bufferItem.mGraphicBuffer->getGenerationNumber());
669    status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer);
670    ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status);
671    if (status == NO_ERROR) {
672        IGraphicBufferProducer::QueueBufferOutput queueOutput;
673        status = mOutput->queueBuffer(slot, queueInput, &queueOutput);
674        ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status);
675    }
676
677    if (status != NO_ERROR) {
678        returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
679        if (status == NO_INIT) {
680            // If the output has been abandoned, move on.
681            onAbandoned_l(false /* isInput */);
682        }
683        return;
684    }
685
686    if (mBuffersSentToOutput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
687        // Something is wrong since this buffer should be held by output now, bail.
688        mInput->consumerDisconnect();
689        onAbandoned_l(true /* isInput */);
690        return;
691    }
692    mBuffersSentToOutput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
693
694    ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId());
695}
696
697void MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) {
698    Mutex::Autolock lock(mMutex);
699
700    if (output != mOutput) {
701        return;  // This is not the current output, ignore.
702    }
703
704    sp<GraphicBuffer> buffer;
705    sp<Fence> fence;
706    status_t status = mOutput->detachNextBuffer(&buffer, &fence);
707    ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status);
708
709    if (status == NO_INIT) {
710        // If the output has been abandoned, we can't do anything else,
711        // since buffer is invalid.
712        onAbandoned_l(false /* isInput */);
713        return;
714    }
715
716    ALOGV("detached buffer %#llx from output", (long long)buffer->getId());
717
718    // If we've been abandoned, we can't return the buffer to the input, so just
719    // move on.
720    if (mIsAbandoned) {
721        return;
722    }
723
724    ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId());
725    if (ix < 0) {
726        // The buffer is unknown, maybe leftover, ignore.
727        return;
728    }
729    mBuffersSentToOutput.removeItemsAt(ix);
730
731    returnBufferToInput_l(buffer, fence);
732}
733
734void MediaSync::returnBufferToInput_l(
735        const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) {
736    ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId());
737    if (ix < 0) {
738        // The buffer is unknown, something is wrong, bail.
739        ALOGE("output returned unknown buffer");
740        mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
741        onAbandoned_l(false /* isInput */);
742        return;
743    }
744    sp<GraphicBuffer> oldBuffer = mBuffersFromInput.valueAt(ix);
745    mBuffersFromInput.removeItemsAt(ix);
746
747    // Attach and release the buffer back to the input.
748    int consumerSlot;
749    status_t status = mInput->attachBuffer(&consumerSlot, oldBuffer);
750    ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status);
751    if (status == NO_ERROR) {
752        status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */,
753                EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
754        ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status);
755    }
756
757    // Notify any waiting onFrameAvailable calls.
758    --mNumOutstandingBuffers;
759    mReleaseCondition.signal();
760
761    if (status == NO_ERROR) {
762        ALOGV("released buffer %#llx to input", (long long)oldBuffer->getId());
763    }
764}
765
766void MediaSync::onAbandoned_l(bool isInput) {
767    ALOGE("the %s has abandoned me", (isInput ? "input" : "output"));
768    if (!mIsAbandoned) {
769        if (isInput) {
770            mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
771        } else {
772            mInput->consumerDisconnect();
773        }
774        mIsAbandoned = true;
775    }
776    mReleaseCondition.broadcast();
777}
778
779void MediaSync::onMessageReceived(const sp<AMessage> &msg) {
780    switch (msg->what()) {
781        case kWhatDrainVideo:
782        {
783            Mutex::Autolock lock(mMutex);
784            if (mNextBufferItemMediaUs != -1) {
785                int64_t nowUs = ALooper::GetNowUs();
786                int64_t itemRealUs = getRealTime(mNextBufferItemMediaUs, nowUs);
787
788                // The message could arrive earlier than expected due to
789                // various reasons, e.g., media clock has been changed because
790                // of new anchor time or playback rate. In such cases, the
791                // message needs to be re-posted.
792                if (itemRealUs > nowUs) {
793                    msg->post(itemRealUs - nowUs);
794                    break;
795                }
796            }
797
798            onDrainVideo_l();
799            break;
800        }
801
802        default:
803            TRESPASS();
804            break;
805    }
806}
807
808MediaSync::InputListener::InputListener(const sp<MediaSync> &sync)
809      : mSync(sync) {}
810
811MediaSync::InputListener::~InputListener() {}
812
813void MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) {
814    mSync->onFrameAvailableFromInput();
815}
816
817// We don't care about sideband streams, since we won't relay them.
818void MediaSync::InputListener::onSidebandStreamChanged() {
819    ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly.");
820}
821
822
823void MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) {
824    Mutex::Autolock lock(mSync->mMutex);
825    mSync->onAbandoned_l(true /* isInput */);
826}
827
828MediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync,
829        const sp<IGraphicBufferProducer> &output)
830      : mSync(sync),
831        mOutput(output) {}
832
833MediaSync::OutputListener::~OutputListener() {}
834
835void MediaSync::OutputListener::onBufferReleased() {
836    mSync->onBufferReleasedByOutput(mOutput);
837}
838
839void MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) {
840    Mutex::Autolock lock(mSync->mMutex);
841    mSync->onAbandoned_l(false /* isInput */);
842}
843
844} // namespace android
845