GraphicBufferSource.cpp revision 9700f5fe4b3becfe858cbf5aa7964296975081bb 
1/*
2 * Copyright (C) 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include <inttypes.h>
18
19#define LOG_TAG "GraphicBufferSource"
20//#define LOG_NDEBUG 0
21#include <utils/Log.h>
22
23#include "GraphicBufferSource.h"
24
25#include <OMX_Core.h>
26#include <media/stagefright/foundation/ADebug.h>
27#include <media/stagefright/foundation/AMessage.h>
28
29#include <media/hardware/MetadataBufferType.h>
30#include <ui/GraphicBuffer.h>
31#include <gui/BufferItem.h>
32#include <HardwareAPI.h>
33
34#include <inttypes.h>
35#include "FrameDropper.h"
36
37namespace android {
38
39static const bool EXTRA_CHECK = true;
40
41GraphicBufferSource::PersistentProxyListener::PersistentProxyListener(
42        const wp<IGraphicBufferConsumer> &consumer,
43        const wp<ConsumerListener>& consumerListener) :
44    mConsumerListener(consumerListener),
45    mConsumer(consumer) {}
46
47GraphicBufferSource::PersistentProxyListener::~PersistentProxyListener() {}
48
49void GraphicBufferSource::PersistentProxyListener::onFrameAvailable(
50        const BufferItem& item) {
51    sp<ConsumerListener> listener(mConsumerListener.promote());
52    if (listener != NULL) {
53        listener->onFrameAvailable(item);
54    } else {
55        sp<IGraphicBufferConsumer> consumer(mConsumer.promote());
56        if (consumer == NULL) {
57            return;
58        }
59        BufferItem bi;
60        status_t err = consumer->acquireBuffer(&bi, 0);
61        if (err != OK) {
62            ALOGE("PersistentProxyListener: acquireBuffer failed (%d)", err);
63            return;
64        }
65
66        err = consumer->detachBuffer(bi.mBuf);
67        if (err != OK) {
68            ALOGE("PersistentProxyListener: detachBuffer failed (%d)", err);
69            return;
70        }
71
72        err = consumer->attachBuffer(&bi.mBuf, bi.mGraphicBuffer);
73        if (err != OK) {
74            ALOGE("PersistentProxyListener: attachBuffer failed (%d)", err);
75            return;
76        }
77
78        err = consumer->releaseBuffer(bi.mBuf, 0,
79                EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, bi.mFence);
80        if (err != OK) {
81            ALOGE("PersistentProxyListener: releaseBuffer failed (%d)", err);
82        }
83    }
84}
85
86void GraphicBufferSource::PersistentProxyListener::onFrameReplaced(
87        const BufferItem& item) {
88    sp<ConsumerListener> listener(mConsumerListener.promote());
89    if (listener != NULL) {
90        listener->onFrameReplaced(item);
91    }
92}
93
94void GraphicBufferSource::PersistentProxyListener::onBuffersReleased() {
95    sp<ConsumerListener> listener(mConsumerListener.promote());
96    if (listener != NULL) {
97        listener->onBuffersReleased();
98    }
99}
100
101void GraphicBufferSource::PersistentProxyListener::onSidebandStreamChanged() {
102    sp<ConsumerListener> listener(mConsumerListener.promote());
103    if (listener != NULL) {
104        listener->onSidebandStreamChanged();
105    }
106}
107
108GraphicBufferSource::GraphicBufferSource(
109        OMXNodeInstance* nodeInstance,
110        uint32_t bufferWidth,
111        uint32_t bufferHeight,
112        uint32_t bufferCount,
113        const sp<IGraphicBufferConsumer> &consumer) :
114    mInitCheck(UNKNOWN_ERROR),
115    mNodeInstance(nodeInstance),
116    mExecuting(false),
117    mSuspended(false),
118    mIsPersistent(false),
119    mConsumer(consumer),
120    mNumFramesAvailable(0),
121    mNumBufferAcquired(0),
122    mEndOfStream(false),
123    mEndOfStreamSent(false),
124    mMaxTimestampGapUs(-1ll),
125    mPrevOriginalTimeUs(-1ll),
126    mPrevModifiedTimeUs(-1ll),
127    mSkipFramesBeforeNs(-1ll),
128    mRepeatAfterUs(-1ll),
129    mRepeatLastFrameGeneration(0),
130    mRepeatLastFrameTimestamp(-1ll),
131    mLatestBufferId(-1),
132    mLatestBufferFrameNum(0),
133    mLatestBufferUseCount(0),
134    mLatestBufferFence(Fence::NO_FENCE),
135    mRepeatBufferDeferred(false),
136    mTimePerCaptureUs(-1ll),
137    mTimePerFrameUs(-1ll),
138    mPrevCaptureUs(-1ll),
139    mPrevFrameUs(-1ll) {
140
141    ALOGV("GraphicBufferSource w=%u h=%u c=%u",
142            bufferWidth, bufferHeight, bufferCount);
143
144    if (bufferWidth == 0 || bufferHeight == 0) {
145        ALOGE("Invalid dimensions %ux%u", bufferWidth, bufferHeight);
146        mInitCheck = BAD_VALUE;
147        return;
148    }
149
150    if (mConsumer == NULL) {
151        String8 name("GraphicBufferSource");
152
153        BufferQueue::createBufferQueue(&mProducer, &mConsumer);
154        mConsumer->setConsumerName(name);
155        mConsumer->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER);
156        mInitCheck = mConsumer->setMaxAcquiredBufferCount(bufferCount);
157        if (mInitCheck != NO_ERROR) {
158            ALOGE("Unable to set BQ max acquired buffer count to %u: %d",
159                    bufferCount, mInitCheck);
160            return;
161        }
162    } else {
163        mIsPersistent = true;
164    }
165    mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight);
166    // Note that we can't create an sp<...>(this) in a ctor that will not keep a
167    // reference once the ctor ends, as that would cause the refcount of 'this'
168    // dropping to 0 at the end of the ctor.  Since all we need is a wp<...>
169    // that's what we create.
170    wp<BufferQueue::ConsumerListener> listener = static_cast<BufferQueue::ConsumerListener*>(this);
171    sp<IConsumerListener> proxy;
172    if (!mIsPersistent) {
173        proxy = new BufferQueue::ProxyConsumerListener(listener);
174    } else {
175        proxy = new PersistentProxyListener(mConsumer, listener);
176    }
177
178    mInitCheck = mConsumer->consumerConnect(proxy, false);
179    if (mInitCheck != NO_ERROR) {
180        ALOGE("Error connecting to BufferQueue: %s (%d)",
181                strerror(-mInitCheck), mInitCheck);
182        return;
183    }
184
185    CHECK(mInitCheck == NO_ERROR);
186}
187
188GraphicBufferSource::~GraphicBufferSource() {
189    if (mLatestBufferId >= 0) {
190        releaseBuffer(
191                mLatestBufferId, mLatestBufferFrameNum,
192                mBufferSlot[mLatestBufferId], mLatestBufferFence);
193    }
194    if (mNumBufferAcquired != 0) {
195        ALOGW("potential buffer leak (acquired %d)", mNumBufferAcquired);
196    }
197    if (mConsumer != NULL && !mIsPersistent) {
198        status_t err = mConsumer->consumerDisconnect();
199        if (err != NO_ERROR) {
200            ALOGW("consumerDisconnect failed: %d", err);
201        }
202    }
203}
204
205void GraphicBufferSource::omxExecuting() {
206    Mutex::Autolock autoLock(mMutex);
207    ALOGV("--> executing; avail=%zu, codec vec size=%zd",
208            mNumFramesAvailable, mCodecBuffers.size());
209    CHECK(!mExecuting);
210    mExecuting = true;
211
212    // Start by loading up as many buffers as possible.  We want to do this,
213    // rather than just submit the first buffer, to avoid a degenerate case:
214    // if all BQ buffers arrive before we start executing, and we only submit
215    // one here, the other BQ buffers will just sit until we get notified
216    // that the codec buffer has been released.  We'd then acquire and
217    // submit a single additional buffer, repeatedly, never using more than
218    // one codec buffer simultaneously.  (We could instead try to submit
219    // all BQ buffers whenever any codec buffer is freed, but if we get the
220    // initial conditions right that will never be useful.)
221    while (mNumFramesAvailable) {
222        if (!fillCodecBuffer_l()) {
223            ALOGV("stop load with frames available (codecAvail=%d)",
224                    isCodecBufferAvailable_l());
225            break;
226        }
227    }
228
229    ALOGV("done loading initial frames, avail=%zu", mNumFramesAvailable);
230
231    // If EOS has already been signaled, and there are no more frames to
232    // submit, try to send EOS now as well.
233    if (mEndOfStream && mNumFramesAvailable == 0) {
234        submitEndOfInputStream_l();
235    }
236
237    if (mRepeatAfterUs > 0ll && mLooper == NULL) {
238        mReflector = new AHandlerReflector<GraphicBufferSource>(this);
239
240        mLooper = new ALooper;
241        mLooper->registerHandler(mReflector);
242        mLooper->start();
243
244        if (mLatestBufferId >= 0) {
245            sp<AMessage> msg =
246                new AMessage(kWhatRepeatLastFrame, mReflector);
247
248            msg->setInt32("generation", ++mRepeatLastFrameGeneration);
249            msg->post(mRepeatAfterUs);
250        }
251    }
252}
253
254void GraphicBufferSource::omxIdle() {
255    ALOGV("omxIdle");
256
257    Mutex::Autolock autoLock(mMutex);
258
259    if (mExecuting) {
260        // We are only interested in the transition from executing->idle,
261        // not loaded->idle.
262        mExecuting = false;
263    }
264}
265
266void GraphicBufferSource::omxLoaded(){
267    Mutex::Autolock autoLock(mMutex);
268    if (!mExecuting) {
269        // This can happen if something failed very early.
270        ALOGW("Dropped back down to Loaded without Executing");
271    }
272
273    if (mLooper != NULL) {
274        mLooper->unregisterHandler(mReflector->id());
275        mReflector.clear();
276
277        mLooper->stop();
278        mLooper.clear();
279    }
280
281    ALOGV("--> loaded; avail=%zu eos=%d eosSent=%d",
282            mNumFramesAvailable, mEndOfStream, mEndOfStreamSent);
283
284    // Codec is no longer executing.  Discard all codec-related state.
285    mCodecBuffers.clear();
286    // TODO: scan mCodecBuffers to verify that all mGraphicBuffer entries
287    //       are null; complain if not
288
289    mExecuting = false;
290}
291
292void GraphicBufferSource::addCodecBuffer(OMX_BUFFERHEADERTYPE* header) {
293    Mutex::Autolock autoLock(mMutex);
294
295    if (mExecuting) {
296        // This should never happen -- buffers can only be allocated when
297        // transitioning from "loaded" to "idle".
298        ALOGE("addCodecBuffer: buffer added while executing");
299        return;
300    }
301
302    ALOGV("addCodecBuffer h=%p size=%" PRIu32 " p=%p",
303            header, header->nAllocLen, header->pBuffer);
304    CodecBuffer codecBuffer;
305    codecBuffer.mHeader = header;
306    mCodecBuffers.add(codecBuffer);
307}
308
309void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header, int fenceFd) {
310    Mutex::Autolock autoLock(mMutex);
311    if (!mExecuting) {
312        return;
313    }
314
315    int cbi = findMatchingCodecBuffer_l(header);
316    if (cbi < 0) {
317        // This should never happen.
318        ALOGE("codecBufferEmptied: buffer not recognized (h=%p)", header);
319        if (fenceFd >= 0) {
320            ::close(fenceFd);
321        }
322        return;
323    }
324
325    ALOGV("codecBufferEmptied h=%p size=%" PRIu32 " filled=%" PRIu32 " p=%p",
326            header, header->nAllocLen, header->nFilledLen,
327            header->pBuffer);
328    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
329
330    // header->nFilledLen may not be the original value, so we can't compare
331    // that to zero to see of this was the EOS buffer.  Instead we just
332    // see if the GraphicBuffer reference was null, which should only ever
333    // happen for EOS.
334    if (codecBuffer.mGraphicBuffer == NULL) {
335        if (!(mEndOfStream && mEndOfStreamSent)) {
336            // This can happen when broken code sends us the same buffer
337            // twice in a row.
338            ALOGE("ERROR: codecBufferEmptied on non-EOS null buffer "
339                    "(buffer emptied twice?)");
340        }
341        // No GraphicBuffer to deal with, no additional input or output is
342        // expected, so just return.
343        if (fenceFd >= 0) {
344            ::close(fenceFd);
345        }
346        return;
347    }
348
349    if (EXTRA_CHECK && header->nAllocLen >= sizeof(MetadataBufferType)) {
350        // Pull the graphic buffer handle back out of the buffer, and confirm
351        // that it matches expectations.
352        OMX_U8* data = header->pBuffer;
353        MetadataBufferType type = *(MetadataBufferType *)data;
354        if (type == kMetadataBufferTypeGrallocSource
355                && header->nAllocLen >= sizeof(VideoGrallocMetadata)) {
356            VideoGrallocMetadata &grallocMeta = *(VideoGrallocMetadata *)data;
357            if (grallocMeta.pHandle != codecBuffer.mGraphicBuffer->handle) {
358                // should never happen
359                ALOGE("codecBufferEmptied: buffer's handle is %p, expected %p",
360                        grallocMeta.pHandle, codecBuffer.mGraphicBuffer->handle);
361                CHECK(!"codecBufferEmptied: mismatched buffer");
362            }
363        } else if (type == kMetadataBufferTypeANWBuffer
364                && header->nAllocLen >= sizeof(VideoNativeMetadata)) {
365            VideoNativeMetadata &nativeMeta = *(VideoNativeMetadata *)data;
366            if (nativeMeta.pBuffer != codecBuffer.mGraphicBuffer->getNativeBuffer()) {
367                // should never happen
368                ALOGE("codecBufferEmptied: buffer is %p, expected %p",
369                        nativeMeta.pBuffer, codecBuffer.mGraphicBuffer->getNativeBuffer());
370                CHECK(!"codecBufferEmptied: mismatched buffer");
371            }
372        }
373    }
374
375    // Find matching entry in our cached copy of the BufferQueue slots.
376    // If we find a match, release that slot.  If we don't, the BufferQueue
377    // has dropped that GraphicBuffer, and there's nothing for us to release.
378    int id = codecBuffer.mBuf;
379    sp<Fence> fence = new Fence(fenceFd);
380    if (mBufferSlot[id] != NULL &&
381        mBufferSlot[id]->handle == codecBuffer.mGraphicBuffer->handle) {
382        ALOGV("cbi %d matches bq slot %d, handle=%p",
383                cbi, id, mBufferSlot[id]->handle);
384
385        if (id == mLatestBufferId) {
386            CHECK_GT(mLatestBufferUseCount--, 0);
387        } else {
388            releaseBuffer(id, codecBuffer.mFrameNumber, mBufferSlot[id], fence);
389        }
390    } else {
391        ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d",
392                cbi);
393        // we will not reuse codec buffer, so there is no need to wait for fence
394    }
395
396    // Mark the codec buffer as available by clearing the GraphicBuffer ref.
397    codecBuffer.mGraphicBuffer = NULL;
398
399    if (mNumFramesAvailable) {
400        // Fill this codec buffer.
401        CHECK(!mEndOfStreamSent);
402        ALOGV("buffer freed, %zu frames avail (eos=%d)",
403                mNumFramesAvailable, mEndOfStream);
404        fillCodecBuffer_l();
405    } else if (mEndOfStream) {
406        // No frames available, but EOS is pending, so use this buffer to
407        // send that.
408        ALOGV("buffer freed, EOS pending");
409        submitEndOfInputStream_l();
410    } else if (mRepeatBufferDeferred) {
411        bool success = repeatLatestBuffer_l();
412        if (success) {
413            ALOGV("deferred repeatLatestBuffer_l SUCCESS");
414        } else {
415            ALOGV("deferred repeatLatestBuffer_l FAILURE");
416        }
417        mRepeatBufferDeferred = false;
418    }
419
420    return;
421}
422
423void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) {
424    Mutex::Autolock autoLock(mMutex);
425
426    if (mMaxTimestampGapUs > 0ll
427            && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) {
428        ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp);
429        if (index >= 0) {
430            ALOGV("OUT timestamp: %lld -> %lld",
431                    static_cast<long long>(header->nTimeStamp),
432                    static_cast<long long>(mOriginalTimeUs[index]));
433            header->nTimeStamp = mOriginalTimeUs[index];
434            mOriginalTimeUs.removeItemsAt(index);
435        } else {
436            // giving up the effort as encoder doesn't appear to preserve pts
437            ALOGW("giving up limiting timestamp gap (pts = %lld)",
438                    header->nTimeStamp);
439            mMaxTimestampGapUs = -1ll;
440        }
441        if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) {
442            // something terribly wrong must have happened, giving up...
443            ALOGE("mOriginalTimeUs has too many entries (%zu)",
444                    mOriginalTimeUs.size());
445            mMaxTimestampGapUs = -1ll;
446        }
447    }
448}
449
450void GraphicBufferSource::suspend(bool suspend) {
451    Mutex::Autolock autoLock(mMutex);
452
453    if (suspend) {
454        mSuspended = true;
455
456        while (mNumFramesAvailable > 0) {
457            BufferItem item;
458            status_t err = mConsumer->acquireBuffer(&item, 0);
459
460            if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
461                // shouldn't happen.
462                ALOGW("suspend: frame was not available");
463                break;
464            } else if (err != OK) {
465                ALOGW("suspend: acquireBuffer returned err=%d", err);
466                break;
467            }
468
469            ++mNumBufferAcquired;
470            --mNumFramesAvailable;
471
472            releaseBuffer(item.mBuf, item.mFrameNumber,
473                    item.mGraphicBuffer, item.mFence);
474        }
475        return;
476    }
477
478    mSuspended = false;
479
480    if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) {
481        if (repeatLatestBuffer_l()) {
482            ALOGV("suspend/deferred repeatLatestBuffer_l SUCCESS");
483
484            mRepeatBufferDeferred = false;
485        } else {
486            ALOGV("suspend/deferred repeatLatestBuffer_l FAILURE");
487        }
488    }
489}
490
491bool GraphicBufferSource::fillCodecBuffer_l() {
492    CHECK(mExecuting && mNumFramesAvailable > 0);
493
494    if (mSuspended) {
495        return false;
496    }
497
498    int cbi = findAvailableCodecBuffer_l();
499    if (cbi < 0) {
500        // No buffers available, bail.
501        ALOGV("fillCodecBuffer_l: no codec buffers, avail now %zu",
502                mNumFramesAvailable);
503        return false;
504    }
505
506    ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%zu",
507            mNumFramesAvailable);
508    BufferItem item;
509    status_t err = mConsumer->acquireBuffer(&item, 0);
510    if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
511        // shouldn't happen
512        ALOGW("fillCodecBuffer_l: frame was not available");
513        return false;
514    } else if (err != OK) {
515        // now what? fake end-of-stream?
516        ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err);
517        return false;
518    }
519
520    mNumBufferAcquired++;
521    mNumFramesAvailable--;
522
523    // If this is the first time we're seeing this buffer, add it to our
524    // slot table.
525    if (item.mGraphicBuffer != NULL) {
526        ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf);
527        mBufferSlot[item.mBuf] = item.mGraphicBuffer;
528    }
529
530    err = UNKNOWN_ERROR;
531
532    // only submit sample if start time is unspecified, or sample
533    // is queued after the specified start time
534    bool dropped = false;
535    if (mSkipFramesBeforeNs < 0ll || item.mTimestamp >= mSkipFramesBeforeNs) {
536        // if start time is set, offset time stamp by start time
537        if (mSkipFramesBeforeNs > 0) {
538            item.mTimestamp -= mSkipFramesBeforeNs;
539        }
540
541        int64_t timeUs = item.mTimestamp / 1000;
542        if (mFrameDropper != NULL && mFrameDropper->shouldDrop(timeUs)) {
543            ALOGV("skipping frame (%lld) to meet max framerate", static_cast<long long>(timeUs));
544            // set err to OK so that the skipped frame can still be saved as the lastest frame
545            err = OK;
546            dropped = true;
547        } else {
548            err = submitBuffer_l(item, cbi);
549        }
550    }
551
552    if (err != OK) {
553        ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf);
554        releaseBuffer(item.mBuf, item.mFrameNumber, item.mGraphicBuffer, item.mFence);
555    } else {
556        ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi);
557        setLatestBuffer_l(item, dropped);
558    }
559
560    return true;
561}
562
563bool GraphicBufferSource::repeatLatestBuffer_l() {
564    CHECK(mExecuting && mNumFramesAvailable == 0);
565
566    if (mLatestBufferId < 0 || mSuspended) {
567        return false;
568    }
569    if (mBufferSlot[mLatestBufferId] == NULL) {
570        // This can happen if the remote side disconnects, causing
571        // onBuffersReleased() to NULL out our copy of the slots.  The
572        // buffer is gone, so we have nothing to show.
573        //
574        // To be on the safe side we try to release the buffer.
575        ALOGD("repeatLatestBuffer_l: slot was NULL");
576        mConsumer->releaseBuffer(
577                mLatestBufferId,
578                mLatestBufferFrameNum,
579                EGL_NO_DISPLAY,
580                EGL_NO_SYNC_KHR,
581                mLatestBufferFence);
582        mLatestBufferId = -1;
583        mLatestBufferFrameNum = 0;
584        mLatestBufferFence = Fence::NO_FENCE;
585        return false;
586    }
587
588    int cbi = findAvailableCodecBuffer_l();
589    if (cbi < 0) {
590        // No buffers available, bail.
591        ALOGV("repeatLatestBuffer_l: no codec buffers.");
592        return false;
593    }
594
595    BufferItem item;
596    item.mBuf = mLatestBufferId;
597    item.mFrameNumber = mLatestBufferFrameNum;
598    item.mTimestamp = mRepeatLastFrameTimestamp;
599    item.mFence = mLatestBufferFence;
600
601    status_t err = submitBuffer_l(item, cbi);
602
603    if (err != OK) {
604        return false;
605    }
606
607    ++mLatestBufferUseCount;
608
609    /* repeat last frame up to kRepeatLastFrameCount times.
610     * in case of static scene, a single repeat might not get rid of encoder
611     * ghosting completely, refresh a couple more times to get better quality
612     */
613    if (--mRepeatLastFrameCount > 0) {
614        mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
615
616        if (mReflector != NULL) {
617            sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector);
618            msg->setInt32("generation", ++mRepeatLastFrameGeneration);
619            msg->post(mRepeatAfterUs);
620        }
621    }
622
623    return true;
624}
625
626void GraphicBufferSource::setLatestBuffer_l(
627        const BufferItem &item, bool dropped) {
628    ALOGV("setLatestBuffer_l");
629
630    if (mLatestBufferId >= 0) {
631        if (mLatestBufferUseCount == 0) {
632            releaseBuffer(mLatestBufferId, mLatestBufferFrameNum,
633                    mBufferSlot[mLatestBufferId], mLatestBufferFence);
634            // mLatestBufferFence will be set to new fence just below
635        }
636    }
637
638    mLatestBufferId = item.mBuf;
639    mLatestBufferFrameNum = item.mFrameNumber;
640    mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
641
642    mLatestBufferUseCount = dropped ? 0 : 1;
643    mRepeatBufferDeferred = false;
644    mRepeatLastFrameCount = kRepeatLastFrameCount;
645    mLatestBufferFence = item.mFence;
646
647    if (mReflector != NULL) {
648        sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector);
649        msg->setInt32("generation", ++mRepeatLastFrameGeneration);
650        msg->post(mRepeatAfterUs);
651    }
652}
653
654status_t GraphicBufferSource::signalEndOfInputStream() {
655    Mutex::Autolock autoLock(mMutex);
656    ALOGV("signalEndOfInputStream: exec=%d avail=%zu eos=%d",
657            mExecuting, mNumFramesAvailable, mEndOfStream);
658
659    if (mEndOfStream) {
660        ALOGE("EOS was already signaled");
661        return INVALID_OPERATION;
662    }
663
664    // Set the end-of-stream flag.  If no frames are pending from the
665    // BufferQueue, and a codec buffer is available, and we're executing,
666    // we initiate the EOS from here.  Otherwise, we'll let
667    // codecBufferEmptied() (or omxExecuting) do it.
668    //
669    // Note: if there are no pending frames and all codec buffers are
670    // available, we *must* submit the EOS from here or we'll just
671    // stall since no future events are expected.
672    mEndOfStream = true;
673
674    if (mExecuting && mNumFramesAvailable == 0) {
675        submitEndOfInputStream_l();
676    }
677
678    return OK;
679}
680
681int64_t GraphicBufferSource::getTimestamp(const BufferItem &item) {
682    int64_t timeUs = item.mTimestamp / 1000;
683
684    if (mTimePerCaptureUs > 0ll) {
685        // Time lapse or slow motion mode
686        if (mPrevCaptureUs < 0ll) {
687            // first capture
688            mPrevCaptureUs = timeUs;
689            mPrevFrameUs = timeUs;
690        } else {
691            // snap to nearest capture point
692            int64_t nFrames = (timeUs + mTimePerCaptureUs / 2 - mPrevCaptureUs)
693                    / mTimePerCaptureUs;
694            if (nFrames <= 0) {
695                // skip this frame as it's too close to previous capture
696                ALOGV("skipping frame, timeUs %lld", static_cast<long long>(timeUs));
697                return -1;
698            }
699            mPrevCaptureUs = mPrevCaptureUs + nFrames * mTimePerCaptureUs;
700            mPrevFrameUs += mTimePerFrameUs * nFrames;
701        }
702
703        ALOGV("timeUs %lld, captureUs %lld, frameUs %lld",
704                static_cast<long long>(timeUs),
705                static_cast<long long>(mPrevCaptureUs),
706                static_cast<long long>(mPrevFrameUs));
707
708        return mPrevFrameUs;
709    } else if (mMaxTimestampGapUs > 0ll) {
710        /* Cap timestamp gap between adjacent frames to specified max
711         *
712         * In the scenario of cast mirroring, encoding could be suspended for
713         * prolonged periods. Limiting the pts gap to workaround the problem
714         * where encoder's rate control logic produces huge frames after a
715         * long period of suspension.
716         */
717
718        int64_t originalTimeUs = timeUs;
719        if (mPrevOriginalTimeUs >= 0ll) {
720            if (originalTimeUs < mPrevOriginalTimeUs) {
721                // Drop the frame if it's going backward in time. Bad timestamp
722                // could disrupt encoder's rate control completely.
723                ALOGW("Dropping frame that's going backward in time");
724                return -1;
725            }
726            int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs;
727            timeUs = (timestampGapUs < mMaxTimestampGapUs ?
728                    timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs;
729        }
730        mPrevOriginalTimeUs = originalTimeUs;
731        mPrevModifiedTimeUs = timeUs;
732        mOriginalTimeUs.add(timeUs, originalTimeUs);
733        ALOGV("IN  timestamp: %lld -> %lld",
734            static_cast<long long>(originalTimeUs),
735            static_cast<long long>(timeUs));
736    }
737
738    return timeUs;
739}
740
741status_t GraphicBufferSource::submitBuffer_l(const BufferItem &item, int cbi) {
742    ALOGV("submitBuffer_l cbi=%d", cbi);
743
744    int64_t timeUs = getTimestamp(item);
745    if (timeUs < 0ll) {
746        return UNKNOWN_ERROR;
747    }
748
749    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
750    codecBuffer.mGraphicBuffer = mBufferSlot[item.mBuf];
751    codecBuffer.mBuf = item.mBuf;
752    codecBuffer.mFrameNumber = item.mFrameNumber;
753
754    OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
755    sp<GraphicBuffer> buffer = codecBuffer.mGraphicBuffer;
756    status_t err = mNodeInstance->emptyGraphicBuffer(
757            header, buffer, OMX_BUFFERFLAG_ENDOFFRAME, timeUs,
758            item.mFence->isValid() ? item.mFence->dup() : -1);
759    if (err != OK) {
760        ALOGW("WARNING: emptyNativeWindowBuffer failed: 0x%x", err);
761        codecBuffer.mGraphicBuffer = NULL;
762        return err;
763    }
764
765    ALOGV("emptyNativeWindowBuffer succeeded, h=%p p=%p buf=%p bufhandle=%p",
766            header, header->pBuffer, buffer->getNativeBuffer(), buffer->handle);
767    return OK;
768}
769
770void GraphicBufferSource::submitEndOfInputStream_l() {
771    CHECK(mEndOfStream);
772    if (mEndOfStreamSent) {
773        ALOGV("EOS already sent");
774        return;
775    }
776
777    int cbi = findAvailableCodecBuffer_l();
778    if (cbi < 0) {
779        ALOGV("submitEndOfInputStream_l: no codec buffers available");
780        return;
781    }
782
783    // We reject any additional incoming graphic buffers, so there's no need
784    // to stick a placeholder into codecBuffer.mGraphicBuffer to mark it as
785    // in-use.
786    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
787
788    OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
789    status_t err = mNodeInstance->emptyGraphicBuffer(
790            header, NULL /* buffer */, OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_EOS,
791            0 /* timestamp */, -1 /* fenceFd */);
792    if (err != OK) {
793        ALOGW("emptyDirectBuffer EOS failed: 0x%x", err);
794    } else {
795        ALOGV("submitEndOfInputStream_l: buffer submitted, header=%p cbi=%d",
796                header, cbi);
797        mEndOfStreamSent = true;
798    }
799}
800
801int GraphicBufferSource::findAvailableCodecBuffer_l() {
802    CHECK(mCodecBuffers.size() > 0);
803
804    for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
805        if (mCodecBuffers[i].mGraphicBuffer == NULL) {
806            return i;
807        }
808    }
809    return -1;
810}
811
812int GraphicBufferSource::findMatchingCodecBuffer_l(
813        const OMX_BUFFERHEADERTYPE* header) {
814    for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
815        if (mCodecBuffers[i].mHeader == header) {
816            return i;
817        }
818    }
819    return -1;
820}
821
822/*
823 * Releases an acquired buffer back to the consumer for either persistent
824 * or non-persistent surfaces.
825 *
826 * id: buffer slot to release (in persistent case the id might be changed)
827 * frameNum: frame number of the frame being released
828 * buffer: GraphicBuffer pointer to release (note this must not be & as we
829 *         will clear the original mBufferSlot in persistent case)
830 * fence: fence of the frame being released
831 */
832void GraphicBufferSource::releaseBuffer(
833        int &id, uint64_t frameNum,
834        const sp<GraphicBuffer> buffer, const sp<Fence> &fence) {
835    if (mIsPersistent) {
836        mConsumer->detachBuffer(id);
837        mBufferSlot[id] = NULL;
838
839        mConsumer->attachBuffer(&id, buffer);
840        mConsumer->releaseBuffer(
841                id, 0, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
842    } else {
843        mConsumer->releaseBuffer(
844                id, frameNum, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
845    }
846    mNumBufferAcquired--;
847}
848
849// BufferQueue::ConsumerListener callback
850void GraphicBufferSource::onFrameAvailable(const BufferItem& /*item*/) {
851    Mutex::Autolock autoLock(mMutex);
852
853    ALOGV("onFrameAvailable exec=%d avail=%zu",
854            mExecuting, mNumFramesAvailable);
855
856    if (mEndOfStream || mSuspended) {
857        if (mEndOfStream) {
858            // This should only be possible if a new buffer was queued after
859            // EOS was signaled, i.e. the app is misbehaving.
860
861            ALOGW("onFrameAvailable: EOS is set, ignoring frame");
862        } else {
863            ALOGV("onFrameAvailable: suspended, ignoring frame");
864        }
865
866        BufferItem item;
867        status_t err = mConsumer->acquireBuffer(&item, 0);
868        if (err == OK) {
869            mNumBufferAcquired++;
870
871            // If this is the first time we're seeing this buffer, add it to our
872            // slot table.
873            if (item.mGraphicBuffer != NULL) {
874                ALOGV("onFrameAvailable: setting mBufferSlot %d", item.mBuf);
875                mBufferSlot[item.mBuf] = item.mGraphicBuffer;
876            }
877
878            releaseBuffer(item.mBuf, item.mFrameNumber,
879                    item.mGraphicBuffer, item.mFence);
880        }
881        return;
882    }
883
884    mNumFramesAvailable++;
885
886    mRepeatBufferDeferred = false;
887    ++mRepeatLastFrameGeneration;
888
889    if (mExecuting) {
890        fillCodecBuffer_l();
891    }
892}
893
894// BufferQueue::ConsumerListener callback
895void GraphicBufferSource::onBuffersReleased() {
896    Mutex::Autolock lock(mMutex);
897
898    uint64_t slotMask;
899    if (mConsumer->getReleasedBuffers(&slotMask) != NO_ERROR) {
900        ALOGW("onBuffersReleased: unable to get released buffer set");
901        slotMask = 0xffffffffffffffffULL;
902    }
903
904    ALOGV("onBuffersReleased: 0x%016" PRIx64, slotMask);
905
906    for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) {
907        if ((slotMask & 0x01) != 0) {
908            mBufferSlot[i] = NULL;
909        }
910        slotMask >>= 1;
911    }
912}
913
914// BufferQueue::ConsumerListener callback
915void GraphicBufferSource::onSidebandStreamChanged() {
916    ALOG_ASSERT(false, "GraphicBufferSource can't consume sideband streams");
917}
918
919status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
920        int64_t repeatAfterUs) {
921    Mutex::Autolock autoLock(mMutex);
922
923    if (mExecuting || repeatAfterUs <= 0ll) {
924        return INVALID_OPERATION;
925    }
926
927    mRepeatAfterUs = repeatAfterUs;
928
929    return OK;
930}
931
932status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) {
933    Mutex::Autolock autoLock(mMutex);
934
935    if (mExecuting || maxGapUs <= 0ll) {
936        return INVALID_OPERATION;
937    }
938
939    mMaxTimestampGapUs = maxGapUs;
940
941    return OK;
942}
943
944status_t GraphicBufferSource::setMaxFps(float maxFps) {
945    Mutex::Autolock autoLock(mMutex);
946
947    if (mExecuting) {
948        return INVALID_OPERATION;
949    }
950
951    mFrameDropper = new FrameDropper();
952    status_t err = mFrameDropper->setMaxFrameRate(maxFps);
953    if (err != OK) {
954        mFrameDropper.clear();
955        return err;
956    }
957
958    return OK;
959}
960
961void GraphicBufferSource::setSkipFramesBeforeUs(int64_t skipFramesBeforeUs) {
962    Mutex::Autolock autoLock(mMutex);
963
964    mSkipFramesBeforeNs =
965            (skipFramesBeforeUs > 0) ? (skipFramesBeforeUs * 1000) : -1ll;
966}
967
968status_t GraphicBufferSource::setTimeLapseUs(int64_t* data) {
969    Mutex::Autolock autoLock(mMutex);
970
971    if (mExecuting || data[0] <= 0ll || data[1] <= 0ll) {
972        return INVALID_OPERATION;
973    }
974
975    mTimePerFrameUs = data[0];
976    mTimePerCaptureUs = data[1];
977
978    return OK;
979}
980
981void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
982    switch (msg->what()) {
983        case kWhatRepeatLastFrame:
984        {
985            Mutex::Autolock autoLock(mMutex);
986
987            int32_t generation;
988            CHECK(msg->findInt32("generation", &generation));
989
990            if (generation != mRepeatLastFrameGeneration) {
991                // stale
992                break;
993            }
994
995            if (!mExecuting || mNumFramesAvailable > 0) {
996                break;
997            }
998
999            bool success = repeatLatestBuffer_l();
1000
1001            if (success) {
1002                ALOGV("repeatLatestBuffer_l SUCCESS");
1003            } else {
1004                ALOGV("repeatLatestBuffer_l FAILURE");
1005                mRepeatBufferDeferred = true;
1006            }
1007            break;
1008        }
1009
1010        default:
1011            TRESPASS();
1012    }
1013}
1014
1015}  // namespace android
1016