GraphicBufferSource.cpp revision c93a13669ce1b5a9e6527b4c86c9d8f5e92be828 
1/*
2 * Copyright (C) 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include <inttypes.h>
18
19#define LOG_TAG "GraphicBufferSource"
20//#define LOG_NDEBUG 0
21#include <utils/Log.h>
22
23#include "GraphicBufferSource.h"
24
25#include <OMX_Core.h>
26#include <OMX_IndexExt.h>
27#include <media/stagefright/foundation/ADebug.h>
28#include <media/stagefright/foundation/AMessage.h>
29
30#include <media/hardware/MetadataBufferType.h>
31#include <ui/GraphicBuffer.h>
32#include <gui/BufferItem.h>
33#include <HardwareAPI.h>
34
35#include <inttypes.h>
36#include "FrameDropper.h"
37
38namespace android {
39
40static const bool EXTRA_CHECK = true;
41
42GraphicBufferSource::PersistentProxyListener::PersistentProxyListener(
43        const wp<IGraphicBufferConsumer> &consumer,
44        const wp<ConsumerListener>& consumerListener) :
45    mConsumerListener(consumerListener),
46    mConsumer(consumer) {}
47
48GraphicBufferSource::PersistentProxyListener::~PersistentProxyListener() {}
49
50void GraphicBufferSource::PersistentProxyListener::onFrameAvailable(
51        const BufferItem& item) {
52    sp<ConsumerListener> listener(mConsumerListener.promote());
53    if (listener != NULL) {
54        listener->onFrameAvailable(item);
55    } else {
56        sp<IGraphicBufferConsumer> consumer(mConsumer.promote());
57        if (consumer == NULL) {
58            return;
59        }
60        BufferItem bi;
61        status_t err = consumer->acquireBuffer(&bi, 0);
62        if (err != OK) {
63            ALOGE("PersistentProxyListener: acquireBuffer failed (%d)", err);
64            return;
65        }
66
67        err = consumer->detachBuffer(bi.mBuf);
68        if (err != OK) {
69            ALOGE("PersistentProxyListener: detachBuffer failed (%d)", err);
70            return;
71        }
72
73        err = consumer->attachBuffer(&bi.mBuf, bi.mGraphicBuffer);
74        if (err != OK) {
75            ALOGE("PersistentProxyListener: attachBuffer failed (%d)", err);
76            return;
77        }
78
79        err = consumer->releaseBuffer(bi.mBuf, 0,
80                EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, bi.mFence);
81        if (err != OK) {
82            ALOGE("PersistentProxyListener: releaseBuffer failed (%d)", err);
83        }
84    }
85}
86
87void GraphicBufferSource::PersistentProxyListener::onFrameReplaced(
88        const BufferItem& item) {
89    sp<ConsumerListener> listener(mConsumerListener.promote());
90    if (listener != NULL) {
91        listener->onFrameReplaced(item);
92    }
93}
94
95void GraphicBufferSource::PersistentProxyListener::onBuffersReleased() {
96    sp<ConsumerListener> listener(mConsumerListener.promote());
97    if (listener != NULL) {
98        listener->onBuffersReleased();
99    }
100}
101
102void GraphicBufferSource::PersistentProxyListener::onSidebandStreamChanged() {
103    sp<ConsumerListener> listener(mConsumerListener.promote());
104    if (listener != NULL) {
105        listener->onSidebandStreamChanged();
106    }
107}
108
109GraphicBufferSource::GraphicBufferSource(
110        OMXNodeInstance* nodeInstance,
111        uint32_t bufferWidth,
112        uint32_t bufferHeight,
113        uint32_t bufferCount,
114        const sp<IGraphicBufferConsumer> &consumer) :
115    mInitCheck(UNKNOWN_ERROR),
116    mNodeInstance(nodeInstance),
117    mExecuting(false),
118    mSuspended(false),
119    mIsPersistent(false),
120    mConsumer(consumer),
121    mNumFramesAvailable(0),
122    mNumBufferAcquired(0),
123    mEndOfStream(false),
124    mEndOfStreamSent(false),
125    mMaxTimestampGapUs(-1ll),
126    mPrevOriginalTimeUs(-1ll),
127    mPrevModifiedTimeUs(-1ll),
128    mSkipFramesBeforeNs(-1ll),
129    mRepeatAfterUs(-1ll),
130    mRepeatLastFrameGeneration(0),
131    mRepeatLastFrameTimestamp(-1ll),
132    mLatestBufferId(-1),
133    mLatestBufferFrameNum(0),
134    mLatestBufferUseCount(0),
135    mLatestBufferFence(Fence::NO_FENCE),
136    mRepeatBufferDeferred(false),
137    mTimePerCaptureUs(-1ll),
138    mTimePerFrameUs(-1ll),
139    mPrevCaptureUs(-1ll),
140    mPrevFrameUs(-1ll) {
141
142    ALOGV("GraphicBufferSource w=%u h=%u c=%u",
143            bufferWidth, bufferHeight, bufferCount);
144
145    if (bufferWidth == 0 || bufferHeight == 0) {
146        ALOGE("Invalid dimensions %ux%u", bufferWidth, bufferHeight);
147        mInitCheck = BAD_VALUE;
148        return;
149    }
150
151    if (mConsumer == NULL) {
152        String8 name("GraphicBufferSource");
153
154        BufferQueue::createBufferQueue(&mProducer, &mConsumer);
155        mConsumer->setConsumerName(name);
156
157        // query consumer usage bits from encoder, but always add HW_VIDEO_ENCODER
158        // for backward compatibility.
159        uint32_t usageBits;
160        status_t err = mNodeInstance->getParameter(
161                (OMX_INDEXTYPE)OMX_IndexParamConsumerUsageBits, &usageBits, sizeof(usageBits));
162        if (err != OK) {
163            usageBits = 0;
164        }
165        usageBits |= GRALLOC_USAGE_HW_VIDEO_ENCODER;
166        mConsumer->setConsumerUsageBits(usageBits);
167
168        mInitCheck = mConsumer->setMaxAcquiredBufferCount(bufferCount);
169        if (mInitCheck != NO_ERROR) {
170            ALOGE("Unable to set BQ max acquired buffer count to %u: %d",
171                    bufferCount, mInitCheck);
172            return;
173        }
174    } else {
175        mIsPersistent = true;
176    }
177    mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight);
178    // Note that we can't create an sp<...>(this) in a ctor that will not keep a
179    // reference once the ctor ends, as that would cause the refcount of 'this'
180    // dropping to 0 at the end of the ctor.  Since all we need is a wp<...>
181    // that's what we create.
182    wp<BufferQueue::ConsumerListener> listener = static_cast<BufferQueue::ConsumerListener*>(this);
183    sp<IConsumerListener> proxy;
184    if (!mIsPersistent) {
185        proxy = new BufferQueue::ProxyConsumerListener(listener);
186    } else {
187        proxy = new PersistentProxyListener(mConsumer, listener);
188    }
189
190    mInitCheck = mConsumer->consumerConnect(proxy, false);
191    if (mInitCheck != NO_ERROR) {
192        ALOGE("Error connecting to BufferQueue: %s (%d)",
193                strerror(-mInitCheck), mInitCheck);
194        return;
195    }
196
197    CHECK(mInitCheck == NO_ERROR);
198}
199
200GraphicBufferSource::~GraphicBufferSource() {
201    if (mLatestBufferId >= 0) {
202        releaseBuffer(
203                mLatestBufferId, mLatestBufferFrameNum,
204                mBufferSlot[mLatestBufferId], mLatestBufferFence);
205    }
206    if (mNumBufferAcquired != 0) {
207        ALOGW("potential buffer leak (acquired %d)", mNumBufferAcquired);
208    }
209    if (mConsumer != NULL && !mIsPersistent) {
210        status_t err = mConsumer->consumerDisconnect();
211        if (err != NO_ERROR) {
212            ALOGW("consumerDisconnect failed: %d", err);
213        }
214    }
215}
216
217void GraphicBufferSource::omxExecuting() {
218    Mutex::Autolock autoLock(mMutex);
219    ALOGV("--> executing; avail=%zu, codec vec size=%zd",
220            mNumFramesAvailable, mCodecBuffers.size());
221    CHECK(!mExecuting);
222    mExecuting = true;
223
224    // Start by loading up as many buffers as possible.  We want to do this,
225    // rather than just submit the first buffer, to avoid a degenerate case:
226    // if all BQ buffers arrive before we start executing, and we only submit
227    // one here, the other BQ buffers will just sit until we get notified
228    // that the codec buffer has been released.  We'd then acquire and
229    // submit a single additional buffer, repeatedly, never using more than
230    // one codec buffer simultaneously.  (We could instead try to submit
231    // all BQ buffers whenever any codec buffer is freed, but if we get the
232    // initial conditions right that will never be useful.)
233    while (mNumFramesAvailable) {
234        if (!fillCodecBuffer_l()) {
235            ALOGV("stop load with frames available (codecAvail=%d)",
236                    isCodecBufferAvailable_l());
237            break;
238        }
239    }
240
241    ALOGV("done loading initial frames, avail=%zu", mNumFramesAvailable);
242
243    // If EOS has already been signaled, and there are no more frames to
244    // submit, try to send EOS now as well.
245    if (mEndOfStream && mNumFramesAvailable == 0) {
246        submitEndOfInputStream_l();
247    }
248
249    if (mRepeatAfterUs > 0ll && mLooper == NULL) {
250        mReflector = new AHandlerReflector<GraphicBufferSource>(this);
251
252        mLooper = new ALooper;
253        mLooper->registerHandler(mReflector);
254        mLooper->start();
255
256        if (mLatestBufferId >= 0) {
257            sp<AMessage> msg =
258                new AMessage(kWhatRepeatLastFrame, mReflector);
259
260            msg->setInt32("generation", ++mRepeatLastFrameGeneration);
261            msg->post(mRepeatAfterUs);
262        }
263    }
264}
265
266void GraphicBufferSource::omxIdle() {
267    ALOGV("omxIdle");
268
269    Mutex::Autolock autoLock(mMutex);
270
271    if (mExecuting) {
272        // We are only interested in the transition from executing->idle,
273        // not loaded->idle.
274        mExecuting = false;
275    }
276}
277
278void GraphicBufferSource::omxLoaded(){
279    Mutex::Autolock autoLock(mMutex);
280    if (!mExecuting) {
281        // This can happen if something failed very early.
282        ALOGW("Dropped back down to Loaded without Executing");
283    }
284
285    if (mLooper != NULL) {
286        mLooper->unregisterHandler(mReflector->id());
287        mReflector.clear();
288
289        mLooper->stop();
290        mLooper.clear();
291    }
292
293    ALOGV("--> loaded; avail=%zu eos=%d eosSent=%d",
294            mNumFramesAvailable, mEndOfStream, mEndOfStreamSent);
295
296    // Codec is no longer executing.  Discard all codec-related state.
297    mCodecBuffers.clear();
298    // TODO: scan mCodecBuffers to verify that all mGraphicBuffer entries
299    //       are null; complain if not
300
301    mExecuting = false;
302}
303
304void GraphicBufferSource::addCodecBuffer(OMX_BUFFERHEADERTYPE* header) {
305    Mutex::Autolock autoLock(mMutex);
306
307    if (mExecuting) {
308        // This should never happen -- buffers can only be allocated when
309        // transitioning from "loaded" to "idle".
310        ALOGE("addCodecBuffer: buffer added while executing");
311        return;
312    }
313
314    ALOGV("addCodecBuffer h=%p size=%" PRIu32 " p=%p",
315            header, header->nAllocLen, header->pBuffer);
316    CodecBuffer codecBuffer;
317    codecBuffer.mHeader = header;
318    mCodecBuffers.add(codecBuffer);
319}
320
321void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header, int fenceFd) {
322    Mutex::Autolock autoLock(mMutex);
323    if (!mExecuting) {
324        return;
325    }
326
327    int cbi = findMatchingCodecBuffer_l(header);
328    if (cbi < 0) {
329        // This should never happen.
330        ALOGE("codecBufferEmptied: buffer not recognized (h=%p)", header);
331        if (fenceFd >= 0) {
332            ::close(fenceFd);
333        }
334        return;
335    }
336
337    ALOGV("codecBufferEmptied h=%p size=%" PRIu32 " filled=%" PRIu32 " p=%p",
338            header, header->nAllocLen, header->nFilledLen,
339            header->pBuffer);
340    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
341
342    // header->nFilledLen may not be the original value, so we can't compare
343    // that to zero to see of this was the EOS buffer.  Instead we just
344    // see if the GraphicBuffer reference was null, which should only ever
345    // happen for EOS.
346    if (codecBuffer.mGraphicBuffer == NULL) {
347        if (!(mEndOfStream && mEndOfStreamSent)) {
348            // This can happen when broken code sends us the same buffer
349            // twice in a row.
350            ALOGE("ERROR: codecBufferEmptied on non-EOS null buffer "
351                    "(buffer emptied twice?)");
352        }
353        // No GraphicBuffer to deal with, no additional input or output is
354        // expected, so just return.
355        if (fenceFd >= 0) {
356            ::close(fenceFd);
357        }
358        return;
359    }
360
361    if (EXTRA_CHECK && header->nAllocLen >= sizeof(MetadataBufferType)) {
362        // Pull the graphic buffer handle back out of the buffer, and confirm
363        // that it matches expectations.
364        OMX_U8* data = header->pBuffer;
365        MetadataBufferType type = *(MetadataBufferType *)data;
366        if (type == kMetadataBufferTypeGrallocSource
367                && header->nAllocLen >= sizeof(VideoGrallocMetadata)) {
368            VideoGrallocMetadata &grallocMeta = *(VideoGrallocMetadata *)data;
369            if (grallocMeta.pHandle != codecBuffer.mGraphicBuffer->handle) {
370                // should never happen
371                ALOGE("codecBufferEmptied: buffer's handle is %p, expected %p",
372                        grallocMeta.pHandle, codecBuffer.mGraphicBuffer->handle);
373                CHECK(!"codecBufferEmptied: mismatched buffer");
374            }
375        } else if (type == kMetadataBufferTypeANWBuffer
376                && header->nAllocLen >= sizeof(VideoNativeMetadata)) {
377            VideoNativeMetadata &nativeMeta = *(VideoNativeMetadata *)data;
378            if (nativeMeta.pBuffer != codecBuffer.mGraphicBuffer->getNativeBuffer()) {
379                // should never happen
380                ALOGE("codecBufferEmptied: buffer is %p, expected %p",
381                        nativeMeta.pBuffer, codecBuffer.mGraphicBuffer->getNativeBuffer());
382                CHECK(!"codecBufferEmptied: mismatched buffer");
383            }
384        }
385    }
386
387    // Find matching entry in our cached copy of the BufferQueue slots.
388    // If we find a match, release that slot.  If we don't, the BufferQueue
389    // has dropped that GraphicBuffer, and there's nothing for us to release.
390    int id = codecBuffer.mBuf;
391    sp<Fence> fence = new Fence(fenceFd);
392    if (mBufferSlot[id] != NULL &&
393        mBufferSlot[id]->handle == codecBuffer.mGraphicBuffer->handle) {
394        ALOGV("cbi %d matches bq slot %d, handle=%p",
395                cbi, id, mBufferSlot[id]->handle);
396
397        if (id == mLatestBufferId) {
398            CHECK_GT(mLatestBufferUseCount--, 0);
399        } else {
400            releaseBuffer(id, codecBuffer.mFrameNumber, mBufferSlot[id], fence);
401        }
402    } else {
403        ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d",
404                cbi);
405        // we will not reuse codec buffer, so there is no need to wait for fence
406    }
407
408    // Mark the codec buffer as available by clearing the GraphicBuffer ref.
409    codecBuffer.mGraphicBuffer = NULL;
410
411    if (mNumFramesAvailable) {
412        // Fill this codec buffer.
413        CHECK(!mEndOfStreamSent);
414        ALOGV("buffer freed, %zu frames avail (eos=%d)",
415                mNumFramesAvailable, mEndOfStream);
416        fillCodecBuffer_l();
417    } else if (mEndOfStream) {
418        // No frames available, but EOS is pending, so use this buffer to
419        // send that.
420        ALOGV("buffer freed, EOS pending");
421        submitEndOfInputStream_l();
422    } else if (mRepeatBufferDeferred) {
423        bool success = repeatLatestBuffer_l();
424        if (success) {
425            ALOGV("deferred repeatLatestBuffer_l SUCCESS");
426        } else {
427            ALOGV("deferred repeatLatestBuffer_l FAILURE");
428        }
429        mRepeatBufferDeferred = false;
430    }
431
432    return;
433}
434
435void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) {
436    Mutex::Autolock autoLock(mMutex);
437
438    if (mMaxTimestampGapUs > 0ll
439            && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) {
440        ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp);
441        if (index >= 0) {
442            ALOGV("OUT timestamp: %lld -> %lld",
443                    static_cast<long long>(header->nTimeStamp),
444                    static_cast<long long>(mOriginalTimeUs[index]));
445            header->nTimeStamp = mOriginalTimeUs[index];
446            mOriginalTimeUs.removeItemsAt(index);
447        } else {
448            // giving up the effort as encoder doesn't appear to preserve pts
449            ALOGW("giving up limiting timestamp gap (pts = %lld)",
450                    header->nTimeStamp);
451            mMaxTimestampGapUs = -1ll;
452        }
453        if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) {
454            // something terribly wrong must have happened, giving up...
455            ALOGE("mOriginalTimeUs has too many entries (%zu)",
456                    mOriginalTimeUs.size());
457            mMaxTimestampGapUs = -1ll;
458        }
459    }
460}
461
462void GraphicBufferSource::suspend(bool suspend) {
463    Mutex::Autolock autoLock(mMutex);
464
465    if (suspend) {
466        mSuspended = true;
467
468        while (mNumFramesAvailable > 0) {
469            BufferItem item;
470            status_t err = mConsumer->acquireBuffer(&item, 0);
471
472            if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
473                // shouldn't happen.
474                ALOGW("suspend: frame was not available");
475                break;
476            } else if (err != OK) {
477                ALOGW("suspend: acquireBuffer returned err=%d", err);
478                break;
479            }
480
481            ++mNumBufferAcquired;
482            --mNumFramesAvailable;
483
484            releaseBuffer(item.mBuf, item.mFrameNumber,
485                    item.mGraphicBuffer, item.mFence);
486        }
487        return;
488    }
489
490    mSuspended = false;
491
492    if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) {
493        if (repeatLatestBuffer_l()) {
494            ALOGV("suspend/deferred repeatLatestBuffer_l SUCCESS");
495
496            mRepeatBufferDeferred = false;
497        } else {
498            ALOGV("suspend/deferred repeatLatestBuffer_l FAILURE");
499        }
500    }
501}
502
503bool GraphicBufferSource::fillCodecBuffer_l() {
504    CHECK(mExecuting && mNumFramesAvailable > 0);
505
506    if (mSuspended) {
507        return false;
508    }
509
510    int cbi = findAvailableCodecBuffer_l();
511    if (cbi < 0) {
512        // No buffers available, bail.
513        ALOGV("fillCodecBuffer_l: no codec buffers, avail now %zu",
514                mNumFramesAvailable);
515        return false;
516    }
517
518    ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%zu",
519            mNumFramesAvailable);
520    BufferItem item;
521    status_t err = mConsumer->acquireBuffer(&item, 0);
522    if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
523        // shouldn't happen
524        ALOGW("fillCodecBuffer_l: frame was not available");
525        return false;
526    } else if (err != OK) {
527        // now what? fake end-of-stream?
528        ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err);
529        return false;
530    }
531
532    mNumBufferAcquired++;
533    mNumFramesAvailable--;
534
535    // If this is the first time we're seeing this buffer, add it to our
536    // slot table.
537    if (item.mGraphicBuffer != NULL) {
538        ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf);
539        mBufferSlot[item.mBuf] = item.mGraphicBuffer;
540    }
541
542    err = UNKNOWN_ERROR;
543
544    // only submit sample if start time is unspecified, or sample
545    // is queued after the specified start time
546    bool dropped = false;
547    if (mSkipFramesBeforeNs < 0ll || item.mTimestamp >= mSkipFramesBeforeNs) {
548        // if start time is set, offset time stamp by start time
549        if (mSkipFramesBeforeNs > 0) {
550            item.mTimestamp -= mSkipFramesBeforeNs;
551        }
552
553        int64_t timeUs = item.mTimestamp / 1000;
554        if (mFrameDropper != NULL && mFrameDropper->shouldDrop(timeUs)) {
555            ALOGV("skipping frame (%lld) to meet max framerate", static_cast<long long>(timeUs));
556            // set err to OK so that the skipped frame can still be saved as the lastest frame
557            err = OK;
558            dropped = true;
559        } else {
560            err = submitBuffer_l(item, cbi);
561        }
562    }
563
564    if (err != OK) {
565        ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf);
566        releaseBuffer(item.mBuf, item.mFrameNumber, item.mGraphicBuffer, item.mFence);
567    } else {
568        ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi);
569        setLatestBuffer_l(item, dropped);
570    }
571
572    return true;
573}
574
575bool GraphicBufferSource::repeatLatestBuffer_l() {
576    CHECK(mExecuting && mNumFramesAvailable == 0);
577
578    if (mLatestBufferId < 0 || mSuspended) {
579        return false;
580    }
581    if (mBufferSlot[mLatestBufferId] == NULL) {
582        // This can happen if the remote side disconnects, causing
583        // onBuffersReleased() to NULL out our copy of the slots.  The
584        // buffer is gone, so we have nothing to show.
585        //
586        // To be on the safe side we try to release the buffer.
587        ALOGD("repeatLatestBuffer_l: slot was NULL");
588        mConsumer->releaseBuffer(
589                mLatestBufferId,
590                mLatestBufferFrameNum,
591                EGL_NO_DISPLAY,
592                EGL_NO_SYNC_KHR,
593                mLatestBufferFence);
594        mLatestBufferId = -1;
595        mLatestBufferFrameNum = 0;
596        mLatestBufferFence = Fence::NO_FENCE;
597        return false;
598    }
599
600    int cbi = findAvailableCodecBuffer_l();
601    if (cbi < 0) {
602        // No buffers available, bail.
603        ALOGV("repeatLatestBuffer_l: no codec buffers.");
604        return false;
605    }
606
607    BufferItem item;
608    item.mBuf = mLatestBufferId;
609    item.mFrameNumber = mLatestBufferFrameNum;
610    item.mTimestamp = mRepeatLastFrameTimestamp;
611    item.mFence = mLatestBufferFence;
612
613    status_t err = submitBuffer_l(item, cbi);
614
615    if (err != OK) {
616        return false;
617    }
618
619    ++mLatestBufferUseCount;
620
621    /* repeat last frame up to kRepeatLastFrameCount times.
622     * in case of static scene, a single repeat might not get rid of encoder
623     * ghosting completely, refresh a couple more times to get better quality
624     */
625    if (--mRepeatLastFrameCount > 0) {
626        mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
627
628        if (mReflector != NULL) {
629            sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector);
630            msg->setInt32("generation", ++mRepeatLastFrameGeneration);
631            msg->post(mRepeatAfterUs);
632        }
633    }
634
635    return true;
636}
637
638void GraphicBufferSource::setLatestBuffer_l(
639        const BufferItem &item, bool dropped) {
640    ALOGV("setLatestBuffer_l");
641
642    if (mLatestBufferId >= 0) {
643        if (mLatestBufferUseCount == 0) {
644            releaseBuffer(mLatestBufferId, mLatestBufferFrameNum,
645                    mBufferSlot[mLatestBufferId], mLatestBufferFence);
646            // mLatestBufferFence will be set to new fence just below
647        }
648    }
649
650    mLatestBufferId = item.mBuf;
651    mLatestBufferFrameNum = item.mFrameNumber;
652    mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
653
654    mLatestBufferUseCount = dropped ? 0 : 1;
655    mRepeatBufferDeferred = false;
656    mRepeatLastFrameCount = kRepeatLastFrameCount;
657    mLatestBufferFence = item.mFence;
658
659    if (mReflector != NULL) {
660        sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector);
661        msg->setInt32("generation", ++mRepeatLastFrameGeneration);
662        msg->post(mRepeatAfterUs);
663    }
664}
665
666status_t GraphicBufferSource::signalEndOfInputStream() {
667    Mutex::Autolock autoLock(mMutex);
668    ALOGV("signalEndOfInputStream: exec=%d avail=%zu eos=%d",
669            mExecuting, mNumFramesAvailable, mEndOfStream);
670
671    if (mEndOfStream) {
672        ALOGE("EOS was already signaled");
673        return INVALID_OPERATION;
674    }
675
676    // Set the end-of-stream flag.  If no frames are pending from the
677    // BufferQueue, and a codec buffer is available, and we're executing,
678    // we initiate the EOS from here.  Otherwise, we'll let
679    // codecBufferEmptied() (or omxExecuting) do it.
680    //
681    // Note: if there are no pending frames and all codec buffers are
682    // available, we *must* submit the EOS from here or we'll just
683    // stall since no future events are expected.
684    mEndOfStream = true;
685
686    if (mExecuting && mNumFramesAvailable == 0) {
687        submitEndOfInputStream_l();
688    }
689
690    return OK;
691}
692
693int64_t GraphicBufferSource::getTimestamp(const BufferItem &item) {
694    int64_t timeUs = item.mTimestamp / 1000;
695
696    if (mTimePerCaptureUs > 0ll) {
697        // Time lapse or slow motion mode
698        if (mPrevCaptureUs < 0ll) {
699            // first capture
700            mPrevCaptureUs = timeUs;
701            mPrevFrameUs = timeUs;
702        } else {
703            // snap to nearest capture point
704            int64_t nFrames = (timeUs + mTimePerCaptureUs / 2 - mPrevCaptureUs)
705                    / mTimePerCaptureUs;
706            if (nFrames <= 0) {
707                // skip this frame as it's too close to previous capture
708                ALOGV("skipping frame, timeUs %lld", static_cast<long long>(timeUs));
709                return -1;
710            }
711            mPrevCaptureUs = mPrevCaptureUs + nFrames * mTimePerCaptureUs;
712            mPrevFrameUs += mTimePerFrameUs * nFrames;
713        }
714
715        ALOGV("timeUs %lld, captureUs %lld, frameUs %lld",
716                static_cast<long long>(timeUs),
717                static_cast<long long>(mPrevCaptureUs),
718                static_cast<long long>(mPrevFrameUs));
719
720        return mPrevFrameUs;
721    } else if (mMaxTimestampGapUs > 0ll) {
722        /* Cap timestamp gap between adjacent frames to specified max
723         *
724         * In the scenario of cast mirroring, encoding could be suspended for
725         * prolonged periods. Limiting the pts gap to workaround the problem
726         * where encoder's rate control logic produces huge frames after a
727         * long period of suspension.
728         */
729
730        int64_t originalTimeUs = timeUs;
731        if (mPrevOriginalTimeUs >= 0ll) {
732            if (originalTimeUs < mPrevOriginalTimeUs) {
733                // Drop the frame if it's going backward in time. Bad timestamp
734                // could disrupt encoder's rate control completely.
735                ALOGW("Dropping frame that's going backward in time");
736                return -1;
737            }
738            int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs;
739            timeUs = (timestampGapUs < mMaxTimestampGapUs ?
740                    timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs;
741        }
742        mPrevOriginalTimeUs = originalTimeUs;
743        mPrevModifiedTimeUs = timeUs;
744        mOriginalTimeUs.add(timeUs, originalTimeUs);
745        ALOGV("IN  timestamp: %lld -> %lld",
746            static_cast<long long>(originalTimeUs),
747            static_cast<long long>(timeUs));
748    }
749
750    return timeUs;
751}
752
753status_t GraphicBufferSource::submitBuffer_l(const BufferItem &item, int cbi) {
754    ALOGV("submitBuffer_l cbi=%d", cbi);
755
756    int64_t timeUs = getTimestamp(item);
757    if (timeUs < 0ll) {
758        return UNKNOWN_ERROR;
759    }
760
761    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
762    codecBuffer.mGraphicBuffer = mBufferSlot[item.mBuf];
763    codecBuffer.mBuf = item.mBuf;
764    codecBuffer.mFrameNumber = item.mFrameNumber;
765
766    OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
767    sp<GraphicBuffer> buffer = codecBuffer.mGraphicBuffer;
768    status_t err = mNodeInstance->emptyGraphicBuffer(
769            header, buffer, OMX_BUFFERFLAG_ENDOFFRAME, timeUs,
770            item.mFence->isValid() ? item.mFence->dup() : -1);
771    if (err != OK) {
772        ALOGW("WARNING: emptyNativeWindowBuffer failed: 0x%x", err);
773        codecBuffer.mGraphicBuffer = NULL;
774        return err;
775    }
776
777    ALOGV("emptyNativeWindowBuffer succeeded, h=%p p=%p buf=%p bufhandle=%p",
778            header, header->pBuffer, buffer->getNativeBuffer(), buffer->handle);
779    return OK;
780}
781
782void GraphicBufferSource::submitEndOfInputStream_l() {
783    CHECK(mEndOfStream);
784    if (mEndOfStreamSent) {
785        ALOGV("EOS already sent");
786        return;
787    }
788
789    int cbi = findAvailableCodecBuffer_l();
790    if (cbi < 0) {
791        ALOGV("submitEndOfInputStream_l: no codec buffers available");
792        return;
793    }
794
795    // We reject any additional incoming graphic buffers, so there's no need
796    // to stick a placeholder into codecBuffer.mGraphicBuffer to mark it as
797    // in-use.
798    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
799
800    OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
801    status_t err = mNodeInstance->emptyGraphicBuffer(
802            header, NULL /* buffer */, OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_EOS,
803            0 /* timestamp */, -1 /* fenceFd */);
804    if (err != OK) {
805        ALOGW("emptyDirectBuffer EOS failed: 0x%x", err);
806    } else {
807        ALOGV("submitEndOfInputStream_l: buffer submitted, header=%p cbi=%d",
808                header, cbi);
809        mEndOfStreamSent = true;
810    }
811}
812
813int GraphicBufferSource::findAvailableCodecBuffer_l() {
814    CHECK(mCodecBuffers.size() > 0);
815
816    for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
817        if (mCodecBuffers[i].mGraphicBuffer == NULL) {
818            return i;
819        }
820    }
821    return -1;
822}
823
824int GraphicBufferSource::findMatchingCodecBuffer_l(
825        const OMX_BUFFERHEADERTYPE* header) {
826    for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
827        if (mCodecBuffers[i].mHeader == header) {
828            return i;
829        }
830    }
831    return -1;
832}
833
834/*
835 * Releases an acquired buffer back to the consumer for either persistent
836 * or non-persistent surfaces.
837 *
838 * id: buffer slot to release (in persistent case the id might be changed)
839 * frameNum: frame number of the frame being released
840 * buffer: GraphicBuffer pointer to release (note this must not be & as we
841 *         will clear the original mBufferSlot in persistent case)
842 * fence: fence of the frame being released
843 */
844void GraphicBufferSource::releaseBuffer(
845        int &id, uint64_t frameNum,
846        const sp<GraphicBuffer> buffer, const sp<Fence> &fence) {
847    if (mIsPersistent) {
848        mConsumer->detachBuffer(id);
849        mBufferSlot[id] = NULL;
850
851        mConsumer->attachBuffer(&id, buffer);
852        mConsumer->releaseBuffer(
853                id, 0, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
854    } else {
855        mConsumer->releaseBuffer(
856                id, frameNum, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
857    }
858    mNumBufferAcquired--;
859}
860
861// BufferQueue::ConsumerListener callback
862void GraphicBufferSource::onFrameAvailable(const BufferItem& /*item*/) {
863    Mutex::Autolock autoLock(mMutex);
864
865    ALOGV("onFrameAvailable exec=%d avail=%zu",
866            mExecuting, mNumFramesAvailable);
867
868    if (mEndOfStream || mSuspended) {
869        if (mEndOfStream) {
870            // This should only be possible if a new buffer was queued after
871            // EOS was signaled, i.e. the app is misbehaving.
872
873            ALOGW("onFrameAvailable: EOS is set, ignoring frame");
874        } else {
875            ALOGV("onFrameAvailable: suspended, ignoring frame");
876        }
877
878        BufferItem item;
879        status_t err = mConsumer->acquireBuffer(&item, 0);
880        if (err == OK) {
881            mNumBufferAcquired++;
882
883            // If this is the first time we're seeing this buffer, add it to our
884            // slot table.
885            if (item.mGraphicBuffer != NULL) {
886                ALOGV("onFrameAvailable: setting mBufferSlot %d", item.mBuf);
887                mBufferSlot[item.mBuf] = item.mGraphicBuffer;
888            }
889
890            releaseBuffer(item.mBuf, item.mFrameNumber,
891                    item.mGraphicBuffer, item.mFence);
892        }
893        return;
894    }
895
896    mNumFramesAvailable++;
897
898    mRepeatBufferDeferred = false;
899    ++mRepeatLastFrameGeneration;
900
901    if (mExecuting) {
902        fillCodecBuffer_l();
903    }
904}
905
906// BufferQueue::ConsumerListener callback
907void GraphicBufferSource::onBuffersReleased() {
908    Mutex::Autolock lock(mMutex);
909
910    uint64_t slotMask;
911    if (mConsumer->getReleasedBuffers(&slotMask) != NO_ERROR) {
912        ALOGW("onBuffersReleased: unable to get released buffer set");
913        slotMask = 0xffffffffffffffffULL;
914    }
915
916    ALOGV("onBuffersReleased: 0x%016" PRIx64, slotMask);
917
918    for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) {
919        if ((slotMask & 0x01) != 0) {
920            mBufferSlot[i] = NULL;
921        }
922        slotMask >>= 1;
923    }
924}
925
926// BufferQueue::ConsumerListener callback
927void GraphicBufferSource::onSidebandStreamChanged() {
928    ALOG_ASSERT(false, "GraphicBufferSource can't consume sideband streams");
929}
930
931status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
932        int64_t repeatAfterUs) {
933    Mutex::Autolock autoLock(mMutex);
934
935    if (mExecuting || repeatAfterUs <= 0ll) {
936        return INVALID_OPERATION;
937    }
938
939    mRepeatAfterUs = repeatAfterUs;
940
941    return OK;
942}
943
944status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) {
945    Mutex::Autolock autoLock(mMutex);
946
947    if (mExecuting || maxGapUs <= 0ll) {
948        return INVALID_OPERATION;
949    }
950
951    mMaxTimestampGapUs = maxGapUs;
952
953    return OK;
954}
955
956status_t GraphicBufferSource::setMaxFps(float maxFps) {
957    Mutex::Autolock autoLock(mMutex);
958
959    if (mExecuting) {
960        return INVALID_OPERATION;
961    }
962
963    mFrameDropper = new FrameDropper();
964    status_t err = mFrameDropper->setMaxFrameRate(maxFps);
965    if (err != OK) {
966        mFrameDropper.clear();
967        return err;
968    }
969
970    return OK;
971}
972
973void GraphicBufferSource::setSkipFramesBeforeUs(int64_t skipFramesBeforeUs) {
974    Mutex::Autolock autoLock(mMutex);
975
976    mSkipFramesBeforeNs =
977            (skipFramesBeforeUs > 0) ? (skipFramesBeforeUs * 1000) : -1ll;
978}
979
980status_t GraphicBufferSource::setTimeLapseUs(int64_t* data) {
981    Mutex::Autolock autoLock(mMutex);
982
983    if (mExecuting || data[0] <= 0ll || data[1] <= 0ll) {
984        return INVALID_OPERATION;
985    }
986
987    mTimePerFrameUs = data[0];
988    mTimePerCaptureUs = data[1];
989
990    return OK;
991}
992
993void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
994    switch (msg->what()) {
995        case kWhatRepeatLastFrame:
996        {
997            Mutex::Autolock autoLock(mMutex);
998
999            int32_t generation;
1000            CHECK(msg->findInt32("generation", &generation));
1001
1002            if (generation != mRepeatLastFrameGeneration) {
1003                // stale
1004                break;
1005            }
1006
1007            if (!mExecuting || mNumFramesAvailable > 0) {
1008                break;
1009            }
1010
1011            bool success = repeatLatestBuffer_l();
1012
1013            if (success) {
1014                ALOGV("repeatLatestBuffer_l SUCCESS");
1015            } else {
1016                ALOGV("repeatLatestBuffer_l FAILURE");
1017                mRepeatBufferDeferred = true;
1018            }
1019            break;
1020        }
1021
1022        default:
1023            TRESPASS();
1024    }
1025}
1026
1027}  // namespace android
1028