1/*
2 * Copyright 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "MediaSync"
19#include <inttypes.h>
20
21#include <gui/BufferQueue.h>
22#include <gui/IGraphicBufferConsumer.h>
23#include <gui/IGraphicBufferProducer.h>
24
25#include <media/AudioTrack.h>
26#include <media/stagefright/MediaClock.h>
27#include <media/stagefright/MediaSync.h>
28#include <media/stagefright/VideoFrameScheduler.h>
29#include <media/stagefright/foundation/ADebug.h>
30#include <media/stagefright/foundation/ALooper.h>
31#include <media/stagefright/foundation/AMessage.h>
32
33#include <ui/GraphicBuffer.h>
34
35#include <system/window.h>
36
37// Maximum late time allowed for a video frame to be rendered. When a video
38// frame arrives later than this number, it will be discarded without rendering.
39static const int64_t kMaxAllowedVideoLateTimeUs = 40000ll;
40
41namespace android {
42
43// static
44sp<MediaSync> MediaSync::create() {
45    sp<MediaSync> sync = new MediaSync();
46    sync->mLooper->registerHandler(sync);
47    return sync;
48}
49
50MediaSync::MediaSync()
51      : mIsAbandoned(false),
52        mMutex(),
53        mReleaseCondition(),
54        mNumOutstandingBuffers(0),
55        mUsageFlagsFromOutput(0),
56        mMaxAcquiredBufferCount(1),
57        mReturnPendingInputFrame(false),
58        mNativeSampleRateInHz(0),
59        mNumFramesWritten(0),
60        mHasAudio(false),
61        mNextBufferItemMediaUs(-1),
62        mPlaybackRate(0.0) {
63    mMediaClock = new MediaClock;
64    mMediaClock->init();
65
66    // initialize settings
67    mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT;
68    mPlaybackSettings.mSpeed = mPlaybackRate;
69
70    mLooper = new ALooper;
71    mLooper->setName("MediaSync");
72    mLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
73}
74
75MediaSync::~MediaSync() {
76    if (mInput != NULL) {
77        mInput->consumerDisconnect();
78    }
79    if (mOutput != NULL) {
80        mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
81    }
82
83    if (mLooper != NULL) {
84        mLooper->unregisterHandler(id());
85        mLooper->stop();
86    }
87}
88
89status_t MediaSync::setSurface(const sp<IGraphicBufferProducer> &output) {
90    Mutex::Autolock lock(mMutex);
91
92    if (output == mOutput) {
93        return NO_ERROR;  // same output surface.
94    }
95
96    if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) {
97        ALOGE("setSurface: output surface is used as sync source and cannot be removed.");
98        return INVALID_OPERATION;
99    }
100
101    if (output != NULL) {
102        int newUsage = 0;
103        output->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &newUsage);
104
105        // Check usage flags only when current output surface has been used to create input surface.
106        if (mOutput != NULL && mInput != NULL) {
107            int ignoredFlags = (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER
108                    | GRALLOC_USAGE_EXTERNAL_DISP);
109            // New output surface is not allowed to add new usage flag except ignored ones.
110            if ((newUsage & ~(mUsageFlagsFromOutput | ignoredFlags)) != 0) {
111                ALOGE("setSurface: new output surface has new usage flag not used by current one.");
112                return BAD_VALUE;
113            }
114        }
115
116        // Try to connect to new output surface. If failed, current output surface will not
117        // be changed.
118        IGraphicBufferProducer::QueueBufferOutput queueBufferOutput;
119        sp<OutputListener> listener(new OutputListener(this, output));
120        IInterface::asBinder(output)->linkToDeath(listener);
121        status_t status =
122            output->connect(listener,
123                            NATIVE_WINDOW_API_MEDIA,
124                            true /* producerControlledByApp */,
125                            &queueBufferOutput);
126        if (status != NO_ERROR) {
127            ALOGE("setSurface: failed to connect (%d)", status);
128            return status;
129        }
130
131        if (mFrameScheduler == NULL) {
132            mFrameScheduler = new VideoFrameScheduler();
133            mFrameScheduler->init();
134        }
135    }
136
137    if (mOutput != NULL) {
138        mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
139        while (!mBuffersSentToOutput.isEmpty()) {
140            returnBufferToInput_l(mBuffersSentToOutput.valueAt(0), Fence::NO_FENCE);
141            mBuffersSentToOutput.removeItemsAt(0);
142        }
143    }
144
145    mOutput = output;
146
147    return NO_ERROR;
148}
149
150// |audioTrack| is used only for querying information.
151status_t MediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) {
152    Mutex::Autolock lock(mMutex);
153
154    // TODO: support audio track change.
155    if (mAudioTrack != NULL) {
156        ALOGE("setAudioTrack: audioTrack has already been configured.");
157        return INVALID_OPERATION;
158    }
159
160    if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) {
161        ALOGE("setAudioTrack: audioTrack is used as sync source and cannot be removed.");
162        return INVALID_OPERATION;
163    }
164
165    if (audioTrack != NULL) {
166        // check if audio track supports the playback settings
167        if (mPlaybackSettings.mSpeed != 0.f
168                && audioTrack->setPlaybackRate(mPlaybackSettings) != OK) {
169            ALOGE("playback settings are not supported by the audio track");
170            return INVALID_OPERATION;
171        }
172        uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate();
173        if (nativeSampleRateInHz <= 0) {
174            ALOGE("setAudioTrack: native sample rate should be positive.");
175            return BAD_VALUE;
176        }
177        mAudioTrack = audioTrack;
178        mNativeSampleRateInHz = nativeSampleRateInHz;
179        (void)setPlaybackSettings_l(mPlaybackSettings);
180    }
181    else {
182        mAudioTrack = NULL;
183        mNativeSampleRateInHz = 0;
184    }
185
186    // potentially resync to new source
187    resync_l();
188    return OK;
189}
190
191status_t MediaSync::createInputSurface(
192        sp<IGraphicBufferProducer> *outBufferProducer) {
193    if (outBufferProducer == NULL) {
194        return BAD_VALUE;
195    }
196
197    Mutex::Autolock lock(mMutex);
198
199    if (mOutput == NULL) {
200        return NO_INIT;
201    }
202
203    if (mInput != NULL) {
204        return INVALID_OPERATION;
205    }
206
207    sp<IGraphicBufferProducer> bufferProducer;
208    sp<IGraphicBufferConsumer> bufferConsumer;
209    BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer);
210
211    sp<InputListener> listener(new InputListener(this));
212    IInterface::asBinder(bufferConsumer)->linkToDeath(listener);
213    status_t status =
214        bufferConsumer->consumerConnect(listener, false /* controlledByApp */);
215    if (status == NO_ERROR) {
216        bufferConsumer->setConsumerName(String8("MediaSync"));
217        // propagate usage bits from output surface
218        mUsageFlagsFromOutput = 0;
219        mOutput->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &mUsageFlagsFromOutput);
220        bufferConsumer->setConsumerUsageBits(mUsageFlagsFromOutput);
221        *outBufferProducer = bufferProducer;
222        mInput = bufferConsumer;
223
224        // set undequeued buffer count
225        int minUndequeuedBuffers;
226        mOutput->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers);
227        mMaxAcquiredBufferCount = minUndequeuedBuffers;
228        bufferConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount);
229    }
230    return status;
231}
232
233void MediaSync::resync_l() {
234    AVSyncSource src = mSyncSettings.mSource;
235    if (src == AVSYNC_SOURCE_DEFAULT) {
236        if (mAudioTrack != NULL) {
237            src = AVSYNC_SOURCE_AUDIO;
238        } else {
239            src = AVSYNC_SOURCE_SYSTEM_CLOCK;
240        }
241    }
242
243    // TODO: resync ourselves to the current clock (e.g. on sync source change)
244    updatePlaybackRate_l(mPlaybackRate);
245}
246
247void MediaSync::updatePlaybackRate_l(float rate) {
248    if (rate > mPlaybackRate) {
249        mNextBufferItemMediaUs = -1;
250    }
251    mPlaybackRate = rate;
252    // TODO: update frame scheduler with this info
253    mMediaClock->setPlaybackRate(rate);
254    onDrainVideo_l();
255}
256
257sp<const MediaClock> MediaSync::getMediaClock() {
258    return mMediaClock;
259}
260
261status_t MediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) {
262    Mutex::Autolock lock(mMutex);
263    // User should check the playback rate if it doesn't want to receive a
264    // huge number for play time.
265    if (mPlaybackRate == 0.0f) {
266        *outTimeUs = INT64_MAX;
267        return OK;
268    }
269
270    uint32_t numFramesPlayed = 0;
271    if (mAudioTrack != NULL) {
272        status_t res = mAudioTrack->getPosition(&numFramesPlayed);
273        if (res != OK) {
274            return res;
275        }
276    }
277
278    int64_t numPendingFrames = mNumFramesWritten - numFramesPlayed;
279    if (numPendingFrames < 0) {
280        numPendingFrames = 0;
281        ALOGW("getPlayTimeForPendingAudioFrames: pending frame count is negative.");
282    }
283    double timeUs = numPendingFrames * 1000000.0
284            / (mNativeSampleRateInHz * (double)mPlaybackRate);
285    if (timeUs > (double)INT64_MAX) {
286        // Overflow.
287        *outTimeUs = INT64_MAX;
288        ALOGW("getPlayTimeForPendingAudioFrames: play time for pending audio frames "
289              "is too high, possibly due to super low playback rate(%f)", mPlaybackRate);
290    } else {
291        *outTimeUs = (int64_t)timeUs;
292    }
293
294    return OK;
295}
296
297status_t MediaSync::updateQueuedAudioData(
298        size_t sizeInBytes, int64_t presentationTimeUs) {
299    if (sizeInBytes == 0) {
300        return OK;
301    }
302
303    Mutex::Autolock lock(mMutex);
304
305    if (mAudioTrack == NULL) {
306        ALOGW("updateQueuedAudioData: audioTrack has NOT been configured.");
307        return INVALID_OPERATION;
308    }
309
310    int64_t numFrames = sizeInBytes / mAudioTrack->frameSize();
311    int64_t maxMediaTimeUs = presentationTimeUs
312            + getDurationIfPlayedAtNativeSampleRate_l(numFrames);
313
314    int64_t nowUs = ALooper::GetNowUs();
315    int64_t nowMediaUs = presentationTimeUs
316            - getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten)
317            + getPlayedOutAudioDurationMedia_l(nowUs);
318
319    mNumFramesWritten += numFrames;
320
321    int64_t oldRealTime = -1;
322    if (mNextBufferItemMediaUs != -1) {
323        oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
324    }
325
326    mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs);
327    mHasAudio = true;
328
329    if (oldRealTime != -1) {
330        int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
331        if (newRealTime >= oldRealTime) {
332            return OK;
333        }
334    }
335
336    mNextBufferItemMediaUs = -1;
337    onDrainVideo_l();
338    return OK;
339}
340
341void MediaSync::setName(const AString &name) {
342    Mutex::Autolock lock(mMutex);
343    mInput->setConsumerName(String8(name.c_str()));
344}
345
346void MediaSync::flush() {
347    Mutex::Autolock lock(mMutex);
348    if (mFrameScheduler != NULL) {
349        mFrameScheduler->restart();
350    }
351    while (!mBufferItems.empty()) {
352        BufferItem *bufferItem = &*mBufferItems.begin();
353        returnBufferToInput_l(bufferItem->mGraphicBuffer, bufferItem->mFence);
354        mBufferItems.erase(mBufferItems.begin());
355    }
356    mNextBufferItemMediaUs = -1;
357    mNumFramesWritten = 0;
358    mReturnPendingInputFrame = true;
359    mReleaseCondition.signal();
360    mMediaClock->clearAnchor();
361}
362
363status_t MediaSync::setVideoFrameRateHint(float rate) {
364    Mutex::Autolock lock(mMutex);
365    if (rate < 0.f) {
366        return BAD_VALUE;
367    }
368    if (mFrameScheduler != NULL) {
369        mFrameScheduler->init(rate);
370    }
371    return OK;
372}
373
374float MediaSync::getVideoFrameRate() {
375    Mutex::Autolock lock(mMutex);
376    if (mFrameScheduler != NULL) {
377        float fps = mFrameScheduler->getFrameRate();
378        if (fps > 0.f) {
379            return fps;
380        }
381    }
382
383    // we don't have or know the frame rate
384    return -1.f;
385}
386
387status_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) {
388    // validate settings
389    if (syncSettings.mSource >= AVSYNC_SOURCE_MAX
390            || syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX
391            || syncSettings.mTolerance < 0.f
392            || syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) {
393        return BAD_VALUE;
394    }
395
396    Mutex::Autolock lock(mMutex);
397
398    // verify that we have the sync source
399    switch (syncSettings.mSource) {
400        case AVSYNC_SOURCE_AUDIO:
401            if (mAudioTrack == NULL) {
402                ALOGE("setSyncSettings: audio sync source requires an audio track");
403                return BAD_VALUE;
404            }
405            break;
406        case AVSYNC_SOURCE_VSYNC:
407            if (mOutput == NULL) {
408                ALOGE("setSyncSettings: vsync sync source requires an output surface");
409                return BAD_VALUE;
410            }
411            break;
412        default:
413            break;
414    }
415
416    mSyncSettings = syncSettings;
417    resync_l();
418    return OK;
419}
420
421void MediaSync::getSyncSettings(AVSyncSettings *syncSettings) {
422    Mutex::Autolock lock(mMutex);
423    *syncSettings = mSyncSettings;
424}
425
426status_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) {
427    Mutex::Autolock lock(mMutex);
428
429    status_t err = setPlaybackSettings_l(rate);
430    if (err == OK) {
431        // TODO: adjust rate if using VSYNC as source
432        updatePlaybackRate_l(rate.mSpeed);
433    }
434    return err;
435}
436
437status_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) {
438    if (rate.mSpeed < 0.f || rate.mPitch < 0.f) {
439        // We don't validate other audio settings.
440        // They will be validated when/if audiotrack is set.
441        return BAD_VALUE;
442    }
443
444    if (mAudioTrack != NULL) {
445        if (rate.mSpeed == 0.f) {
446            mAudioTrack->pause();
447        } else {
448            status_t err = mAudioTrack->setPlaybackRate(rate);
449            if (err != OK) {
450                return BAD_VALUE;
451            }
452
453            // ignore errors
454            (void)mAudioTrack->start();
455        }
456    }
457    mPlaybackSettings = rate;
458    return OK;
459}
460
461void MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) {
462    Mutex::Autolock lock(mMutex);
463    *rate = mPlaybackSettings;
464}
465
466int64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) {
467    int64_t realUs;
468    if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
469        // If failed to get current position, e.g. due to audio clock is
470        // not ready, then just play out video immediately without delay.
471        return nowUs;
472    }
473    return realUs;
474}
475
476int64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) {
477    return (numFrames * 1000000LL / mNativeSampleRateInHz);
478}
479
480int64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) {
481    CHECK(mAudioTrack != NULL);
482
483    uint32_t numFramesPlayed;
484    int64_t numFramesPlayedAtUs;
485    AudioTimestamp ts;
486
487    status_t res = mAudioTrack->getTimestamp(ts);
488    if (res == OK) {
489        // case 1: mixing audio tracks.
490        numFramesPlayed = ts.mPosition;
491        numFramesPlayedAtUs = ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
492        //ALOGD("getTimestamp: OK %d %lld",
493        //      numFramesPlayed, (long long)numFramesPlayedAtUs);
494    } else if (res == WOULD_BLOCK) {
495        // case 2: transitory state on start of a new track
496        numFramesPlayed = 0;
497        numFramesPlayedAtUs = nowUs;
498        //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
499        //      numFramesPlayed, (long long)numFramesPlayedAtUs);
500    } else {
501        // case 3: transitory at new track or audio fast tracks.
502        res = mAudioTrack->getPosition(&numFramesPlayed);
503        CHECK_EQ(res, (status_t)OK);
504        numFramesPlayedAtUs = nowUs;
505        numFramesPlayedAtUs += 1000LL * mAudioTrack->latency() / 2; /* XXX */
506        //ALOGD("getPosition: %d %lld", numFramesPlayed, (long long)numFramesPlayedAtUs);
507    }
508
509    //can't be negative until 12.4 hrs, test.
510    //CHECK_EQ(numFramesPlayed & (1 << 31), 0);
511    int64_t durationUs =
512        getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed)
513            + nowUs - numFramesPlayedAtUs;
514    if (durationUs < 0) {
515        // Occurs when numFramesPlayed position is very small and the following:
516        // (1) In case 1, the time nowUs is computed before getTimestamp() is
517        //     called and numFramesPlayedAtUs is greater than nowUs by time more
518        //     than numFramesPlayed.
519        // (2) In case 3, using getPosition and adding mAudioTrack->latency()
520        //     to numFramesPlayedAtUs, by a time amount greater than
521        //     numFramesPlayed.
522        //
523        // Both of these are transitory conditions.
524        ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld "
525              "set to zero", (long long)durationUs);
526        durationUs = 0;
527    }
528    ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) "
529          "framesAt(%lld)",
530          (long long)durationUs, (long long)nowUs, numFramesPlayed,
531          (long long)numFramesPlayedAtUs);
532    return durationUs;
533}
534
535void MediaSync::onDrainVideo_l() {
536    if (!isPlaying()) {
537        return;
538    }
539
540    while (!mBufferItems.empty()) {
541        int64_t nowUs = ALooper::GetNowUs();
542        BufferItem *bufferItem = &*mBufferItems.begin();
543        int64_t itemMediaUs = bufferItem->mTimestamp / 1000;
544        int64_t itemRealUs = getRealTime(itemMediaUs, nowUs);
545
546        // adjust video frame PTS based on vsync
547        itemRealUs = mFrameScheduler->schedule(itemRealUs * 1000) / 1000;
548        int64_t twoVsyncsUs = 2 * (mFrameScheduler->getVsyncPeriod() / 1000);
549
550        // post 2 display refreshes before rendering is due
551        if (itemRealUs <= nowUs + twoVsyncsUs) {
552            ALOGV("adjusting PTS from %lld to %lld",
553                    (long long)bufferItem->mTimestamp / 1000, (long long)itemRealUs);
554            bufferItem->mTimestamp = itemRealUs * 1000;
555            bufferItem->mIsAutoTimestamp = false;
556
557            if (mHasAudio) {
558                if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) {
559                    renderOneBufferItem_l(*bufferItem);
560                } else {
561                    // too late.
562                    returnBufferToInput_l(
563                            bufferItem->mGraphicBuffer, bufferItem->mFence);
564                    mFrameScheduler->restart();
565                }
566            } else {
567                // always render video buffer in video-only mode.
568                renderOneBufferItem_l(*bufferItem);
569
570                // smooth out videos >= 10fps
571                mMediaClock->updateAnchor(
572                        itemMediaUs, nowUs, itemMediaUs + 100000);
573            }
574
575            mBufferItems.erase(mBufferItems.begin());
576            mNextBufferItemMediaUs = -1;
577        } else {
578            if (mNextBufferItemMediaUs == -1
579                    || mNextBufferItemMediaUs > itemMediaUs) {
580                sp<AMessage> msg = new AMessage(kWhatDrainVideo, this);
581                msg->post(itemRealUs - nowUs - twoVsyncsUs);
582                mNextBufferItemMediaUs = itemMediaUs;
583            }
584            break;
585        }
586    }
587}
588
589void MediaSync::onFrameAvailableFromInput() {
590    Mutex::Autolock lock(mMutex);
591
592    const static nsecs_t kAcquireWaitTimeout = 2000000000; // 2 seconds
593
594    mReturnPendingInputFrame = false;
595
596    // If there are too many outstanding buffers, wait until a buffer is
597    // released back to the input in onBufferReleased.
598    // NOTE: BufferQueue allows dequeuing maxAcquiredBufferCount + 1 buffers
599    while (mNumOutstandingBuffers > mMaxAcquiredBufferCount
600            && !mIsAbandoned && !mReturnPendingInputFrame) {
601        if (mReleaseCondition.waitRelative(mMutex, kAcquireWaitTimeout) != OK) {
602            ALOGI_IF(mPlaybackRate != 0.f, "still waiting to release a buffer before acquire");
603        }
604
605        // If the sync is abandoned while we are waiting, the release
606        // condition variable will be broadcast, and we should just return
607        // without attempting to do anything more (since the input queue will
608        // also be abandoned).
609        if (mIsAbandoned) {
610            return;
611        }
612    }
613
614    // Acquire and detach the buffer from the input.
615    BufferItem bufferItem;
616    status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */);
617    if (status != NO_ERROR) {
618        ALOGE("acquiring buffer from input failed (%d)", status);
619        return;
620    }
621    ++mNumOutstandingBuffers;
622
623    ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId());
624
625    status = mInput->detachBuffer(bufferItem.mSlot);
626    if (status != NO_ERROR) {
627        ALOGE("detaching buffer from input failed (%d)", status);
628        if (status == NO_INIT) {
629            // If the input has been abandoned, move on.
630            onAbandoned_l(true /* isInput */);
631        }
632        return;
633    }
634
635    if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
636        // Something is wrong since this buffer should be at our hands, bail.
637        ALOGE("received buffer multiple times from input");
638        mInput->consumerDisconnect();
639        onAbandoned_l(true /* isInput */);
640        return;
641    }
642    mBuffersFromInput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
643
644    // If flush happened while waiting for a buffer to be released, simply return it
645    // TRICKY: do it here after it is detached so that we don't have to cache mGraphicBuffer.
646    if (mReturnPendingInputFrame) {
647        mReturnPendingInputFrame = false;
648        returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
649        return;
650    }
651
652    mBufferItems.push_back(bufferItem);
653
654    if (mBufferItems.size() == 1) {
655        onDrainVideo_l();
656    }
657}
658
659void MediaSync::renderOneBufferItem_l(const BufferItem &bufferItem) {
660    IGraphicBufferProducer::QueueBufferInput queueInput(
661            bufferItem.mTimestamp,
662            bufferItem.mIsAutoTimestamp,
663            bufferItem.mDataSpace,
664            bufferItem.mCrop,
665            static_cast<int32_t>(bufferItem.mScalingMode),
666            bufferItem.mTransform,
667            bufferItem.mFence);
668
669    // Attach and queue the buffer to the output.
670    int slot;
671    mOutput->setGenerationNumber(bufferItem.mGraphicBuffer->getGenerationNumber());
672    status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer);
673    ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status);
674    if (status == NO_ERROR) {
675        IGraphicBufferProducer::QueueBufferOutput queueOutput;
676        status = mOutput->queueBuffer(slot, queueInput, &queueOutput);
677        ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status);
678    }
679
680    if (status != NO_ERROR) {
681        returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
682        if (status == NO_INIT) {
683            // If the output has been abandoned, move on.
684            onAbandoned_l(false /* isInput */);
685        }
686        return;
687    }
688
689    if (mBuffersSentToOutput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
690        // Something is wrong since this buffer should be held by output now, bail.
691        mInput->consumerDisconnect();
692        onAbandoned_l(true /* isInput */);
693        return;
694    }
695    mBuffersSentToOutput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
696
697    ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId());
698}
699
700void MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) {
701    Mutex::Autolock lock(mMutex);
702
703    if (output != mOutput) {
704        return;  // This is not the current output, ignore.
705    }
706
707    sp<GraphicBuffer> buffer;
708    sp<Fence> fence;
709    status_t status = mOutput->detachNextBuffer(&buffer, &fence);
710    ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status);
711
712    if (status == NO_INIT) {
713        // If the output has been abandoned, we can't do anything else,
714        // since buffer is invalid.
715        onAbandoned_l(false /* isInput */);
716        return;
717    }
718
719    ALOGV("detached buffer %#llx from output", (long long)buffer->getId());
720
721    // If we've been abandoned, we can't return the buffer to the input, so just
722    // move on.
723    if (mIsAbandoned) {
724        return;
725    }
726
727    ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId());
728    if (ix < 0) {
729        // The buffer is unknown, maybe leftover, ignore.
730        return;
731    }
732    mBuffersSentToOutput.removeItemsAt(ix);
733
734    returnBufferToInput_l(buffer, fence);
735}
736
737void MediaSync::returnBufferToInput_l(
738        const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) {
739    ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId());
740    if (ix < 0) {
741        // The buffer is unknown, something is wrong, bail.
742        ALOGE("output returned unknown buffer");
743        mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
744        onAbandoned_l(false /* isInput */);
745        return;
746    }
747    sp<GraphicBuffer> oldBuffer = mBuffersFromInput.valueAt(ix);
748    mBuffersFromInput.removeItemsAt(ix);
749
750    // Attach and release the buffer back to the input.
751    int consumerSlot;
752    status_t status = mInput->attachBuffer(&consumerSlot, oldBuffer);
753    ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status);
754    if (status == NO_ERROR) {
755        status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */,
756                EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
757        ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status);
758    }
759
760    // Notify any waiting onFrameAvailable calls.
761    --mNumOutstandingBuffers;
762    mReleaseCondition.signal();
763
764    if (status == NO_ERROR) {
765        ALOGV("released buffer %#llx to input", (long long)oldBuffer->getId());
766    }
767}
768
769void MediaSync::onAbandoned_l(bool isInput) {
770    ALOGE("the %s has abandoned me", (isInput ? "input" : "output"));
771    if (!mIsAbandoned) {
772        if (isInput) {
773            mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
774        } else {
775            mInput->consumerDisconnect();
776        }
777        mIsAbandoned = true;
778    }
779    mReleaseCondition.broadcast();
780}
781
782void MediaSync::onMessageReceived(const sp<AMessage> &msg) {
783    switch (msg->what()) {
784        case kWhatDrainVideo:
785        {
786            Mutex::Autolock lock(mMutex);
787            if (mNextBufferItemMediaUs != -1) {
788                int64_t nowUs = ALooper::GetNowUs();
789                int64_t itemRealUs = getRealTime(mNextBufferItemMediaUs, nowUs);
790
791                // The message could arrive earlier than expected due to
792                // various reasons, e.g., media clock has been changed because
793                // of new anchor time or playback rate. In such cases, the
794                // message needs to be re-posted.
795                if (itemRealUs > nowUs) {
796                    msg->post(itemRealUs - nowUs);
797                    break;
798                }
799            }
800
801            onDrainVideo_l();
802            break;
803        }
804
805        default:
806            TRESPASS();
807            break;
808    }
809}
810
811MediaSync::InputListener::InputListener(const sp<MediaSync> &sync)
812      : mSync(sync) {}
813
814MediaSync::InputListener::~InputListener() {}
815
816void MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) {
817    mSync->onFrameAvailableFromInput();
818}
819
820// We don't care about sideband streams, since we won't relay them.
821void MediaSync::InputListener::onSidebandStreamChanged() {
822    ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly.");
823}
824
825
826void MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) {
827    Mutex::Autolock lock(mSync->mMutex);
828    mSync->onAbandoned_l(true /* isInput */);
829}
830
831MediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync,
832        const sp<IGraphicBufferProducer> &output)
833      : mSync(sync),
834        mOutput(output) {}
835
836MediaSync::OutputListener::~OutputListener() {}
837
838void MediaSync::OutputListener::onBufferReleased() {
839    mSync->onBufferReleasedByOutput(mOutput);
840}
841
842void MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) {
843    Mutex::Autolock lock(mSync->mMutex);
844    mSync->onAbandoned_l(false /* isInput */);
845}
846
847} // namespace android
848