GraphicBufferSource.cpp revision 15ab4996019387f27a48b81cb4774c21502bc0e5 
1/*
2 * Copyright (C) 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include <inttypes.h>
18
19#define LOG_TAG "GraphicBufferSource"
20//#define LOG_NDEBUG 0
21#include <utils/Log.h>
22
23#include "GraphicBufferSource.h"
24
25#include <OMX_Core.h>
26#include <media/stagefright/foundation/ADebug.h>
27#include <media/stagefright/foundation/AMessage.h>
28
29#include <media/hardware/MetadataBufferType.h>
30#include <ui/GraphicBuffer.h>
31#include <gui/BufferItem.h>
32#include <HardwareAPI.h>
33
34#include <inttypes.h>
35#include "FrameDropper.h"
36
37namespace android {
38
39static const bool EXTRA_CHECK = true;
40
41
42GraphicBufferSource::GraphicBufferSource(
43        OMXNodeInstance* nodeInstance,
44        uint32_t bufferWidth,
45        uint32_t bufferHeight,
46        uint32_t bufferCount,
47        const sp<IGraphicBufferConsumer> &consumer) :
48    mInitCheck(UNKNOWN_ERROR),
49    mNodeInstance(nodeInstance),
50    mExecuting(false),
51    mSuspended(false),
52    mIsPersistent(false),
53    mConsumer(consumer),
54    mNumFramesAvailable(0),
55    mEndOfStream(false),
56    mEndOfStreamSent(false),
57    mMaxTimestampGapUs(-1ll),
58    mPrevOriginalTimeUs(-1ll),
59    mPrevModifiedTimeUs(-1ll),
60    mSkipFramesBeforeNs(-1ll),
61    mRepeatAfterUs(-1ll),
62    mRepeatLastFrameGeneration(0),
63    mRepeatLastFrameTimestamp(-1ll),
64    mLatestBufferId(-1),
65    mLatestBufferFrameNum(0),
66    mLatestBufferUseCount(0),
67    mLatestBufferFence(Fence::NO_FENCE),
68    mRepeatBufferDeferred(false),
69    mTimePerCaptureUs(-1ll),
70    mTimePerFrameUs(-1ll),
71    mPrevCaptureUs(-1ll),
72    mPrevFrameUs(-1ll) {
73
74    ALOGV("GraphicBufferSource w=%u h=%u c=%u",
75            bufferWidth, bufferHeight, bufferCount);
76
77    if (bufferWidth == 0 || bufferHeight == 0) {
78        ALOGE("Invalid dimensions %ux%u", bufferWidth, bufferHeight);
79        mInitCheck = BAD_VALUE;
80        return;
81    }
82
83    if (mConsumer == NULL) {
84        String8 name("GraphicBufferSource");
85
86        BufferQueue::createBufferQueue(&mProducer, &mConsumer);
87        mConsumer->setConsumerName(name);
88        mConsumer->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER);
89        mInitCheck = mConsumer->setMaxAcquiredBufferCount(bufferCount);
90        if (mInitCheck != NO_ERROR) {
91            ALOGE("Unable to set BQ max acquired buffer count to %u: %d",
92                    bufferCount, mInitCheck);
93            return;
94        }
95    } else {
96        mIsPersistent = true;
97    }
98    mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight);
99    // Note that we can't create an sp<...>(this) in a ctor that will not keep a
100    // reference once the ctor ends, as that would cause the refcount of 'this'
101    // dropping to 0 at the end of the ctor.  Since all we need is a wp<...>
102    // that's what we create.
103    wp<BufferQueue::ConsumerListener> listener = static_cast<BufferQueue::ConsumerListener*>(this);
104    sp<BufferQueue::ProxyConsumerListener> proxy = new BufferQueue::ProxyConsumerListener(listener);
105
106    mInitCheck = mConsumer->consumerConnect(proxy, false);
107    if (mInitCheck != NO_ERROR) {
108        ALOGE("Error connecting to BufferQueue: %s (%d)",
109                strerror(-mInitCheck), mInitCheck);
110        return;
111    }
112
113    CHECK(mInitCheck == NO_ERROR);
114}
115
116GraphicBufferSource::~GraphicBufferSource() {
117    ALOGV("~GraphicBufferSource");
118    if (mConsumer != NULL && !mIsPersistent) {
119        status_t err = mConsumer->consumerDisconnect();
120        if (err != NO_ERROR) {
121            ALOGW("consumerDisconnect failed: %d", err);
122        }
123    }
124}
125
126void GraphicBufferSource::omxExecuting() {
127    Mutex::Autolock autoLock(mMutex);
128    ALOGV("--> executing; avail=%zu, codec vec size=%zd",
129            mNumFramesAvailable, mCodecBuffers.size());
130    CHECK(!mExecuting);
131    mExecuting = true;
132
133    // Start by loading up as many buffers as possible.  We want to do this,
134    // rather than just submit the first buffer, to avoid a degenerate case:
135    // if all BQ buffers arrive before we start executing, and we only submit
136    // one here, the other BQ buffers will just sit until we get notified
137    // that the codec buffer has been released.  We'd then acquire and
138    // submit a single additional buffer, repeatedly, never using more than
139    // one codec buffer simultaneously.  (We could instead try to submit
140    // all BQ buffers whenever any codec buffer is freed, but if we get the
141    // initial conditions right that will never be useful.)
142    while (mNumFramesAvailable) {
143        if (!fillCodecBuffer_l()) {
144            ALOGV("stop load with frames available (codecAvail=%d)",
145                    isCodecBufferAvailable_l());
146            break;
147        }
148    }
149
150    ALOGV("done loading initial frames, avail=%zu", mNumFramesAvailable);
151
152    // If EOS has already been signaled, and there are no more frames to
153    // submit, try to send EOS now as well.
154    if (mEndOfStream && mNumFramesAvailable == 0) {
155        submitEndOfInputStream_l();
156    }
157
158    if (mRepeatAfterUs > 0ll && mLooper == NULL) {
159        mReflector = new AHandlerReflector<GraphicBufferSource>(this);
160
161        mLooper = new ALooper;
162        mLooper->registerHandler(mReflector);
163        mLooper->start();
164
165        if (mLatestBufferId >= 0) {
166            sp<AMessage> msg =
167                new AMessage(kWhatRepeatLastFrame, mReflector);
168
169            msg->setInt32("generation", ++mRepeatLastFrameGeneration);
170            msg->post(mRepeatAfterUs);
171        }
172    }
173}
174
175void GraphicBufferSource::omxIdle() {
176    ALOGV("omxIdle");
177
178    Mutex::Autolock autoLock(mMutex);
179
180    if (mExecuting) {
181        // We are only interested in the transition from executing->idle,
182        // not loaded->idle.
183        mExecuting = false;
184    }
185}
186
187void GraphicBufferSource::omxLoaded(){
188    Mutex::Autolock autoLock(mMutex);
189    if (!mExecuting) {
190        // This can happen if something failed very early.
191        ALOGW("Dropped back down to Loaded without Executing");
192    }
193
194    if (mLooper != NULL) {
195        mLooper->unregisterHandler(mReflector->id());
196        mReflector.clear();
197
198        mLooper->stop();
199        mLooper.clear();
200    }
201
202    ALOGV("--> loaded; avail=%zu eos=%d eosSent=%d",
203            mNumFramesAvailable, mEndOfStream, mEndOfStreamSent);
204
205    // Codec is no longer executing.  Discard all codec-related state.
206    mCodecBuffers.clear();
207    // TODO: scan mCodecBuffers to verify that all mGraphicBuffer entries
208    //       are null; complain if not
209
210    mExecuting = false;
211}
212
213void GraphicBufferSource::addCodecBuffer(OMX_BUFFERHEADERTYPE* header) {
214    Mutex::Autolock autoLock(mMutex);
215
216    if (mExecuting) {
217        // This should never happen -- buffers can only be allocated when
218        // transitioning from "loaded" to "idle".
219        ALOGE("addCodecBuffer: buffer added while executing");
220        return;
221    }
222
223    ALOGV("addCodecBuffer h=%p size=%" PRIu32 " p=%p",
224            header, header->nAllocLen, header->pBuffer);
225    CodecBuffer codecBuffer;
226    codecBuffer.mHeader = header;
227    mCodecBuffers.add(codecBuffer);
228}
229
230void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header, int fenceFd) {
231    Mutex::Autolock autoLock(mMutex);
232    if (!mExecuting) {
233        return;
234    }
235
236    int cbi = findMatchingCodecBuffer_l(header);
237    if (cbi < 0) {
238        // This should never happen.
239        ALOGE("codecBufferEmptied: buffer not recognized (h=%p)", header);
240        if (fenceFd >= 0) {
241            ::close(fenceFd);
242        }
243        return;
244    }
245
246    ALOGV("codecBufferEmptied h=%p size=%" PRIu32 " filled=%" PRIu32 " p=%p",
247            header, header->nAllocLen, header->nFilledLen,
248            header->pBuffer);
249    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
250
251    // header->nFilledLen may not be the original value, so we can't compare
252    // that to zero to see of this was the EOS buffer.  Instead we just
253    // see if the GraphicBuffer reference was null, which should only ever
254    // happen for EOS.
255    if (codecBuffer.mGraphicBuffer == NULL) {
256        if (!(mEndOfStream && mEndOfStreamSent)) {
257            // This can happen when broken code sends us the same buffer
258            // twice in a row.
259            ALOGE("ERROR: codecBufferEmptied on non-EOS null buffer "
260                    "(buffer emptied twice?)");
261        }
262        // No GraphicBuffer to deal with, no additional input or output is
263        // expected, so just return.
264        if (fenceFd >= 0) {
265            ::close(fenceFd);
266        }
267        return;
268    }
269
270    if (EXTRA_CHECK && header->nAllocLen >= sizeof(MetadataBufferType)) {
271        // Pull the graphic buffer handle back out of the buffer, and confirm
272        // that it matches expectations.
273        OMX_U8* data = header->pBuffer;
274        MetadataBufferType type = *(MetadataBufferType *)data;
275        if (type == kMetadataBufferTypeGrallocSource
276                && header->nAllocLen >= sizeof(VideoGrallocMetadata)) {
277            VideoGrallocMetadata &grallocMeta = *(VideoGrallocMetadata *)data;
278            if (grallocMeta.hHandle != codecBuffer.mGraphicBuffer->handle) {
279                // should never happen
280                ALOGE("codecBufferEmptied: buffer's handle is %p, expected %p",
281                        grallocMeta.hHandle, codecBuffer.mGraphicBuffer->handle);
282                CHECK(!"codecBufferEmptied: mismatched buffer");
283            }
284        } else if (type == kMetadataBufferTypeANWBuffer
285                && header->nAllocLen >= sizeof(VideoNativeMetadata)) {
286            VideoNativeMetadata &nativeMeta = *(VideoNativeMetadata *)data;
287            if (nativeMeta.pBuffer != codecBuffer.mGraphicBuffer->getNativeBuffer()) {
288                // should never happen
289                ALOGE("codecBufferEmptied: buffer is %p, expected %p",
290                        nativeMeta.pBuffer, codecBuffer.mGraphicBuffer->getNativeBuffer());
291                CHECK(!"codecBufferEmptied: mismatched buffer");
292            }
293        }
294    }
295
296    // Find matching entry in our cached copy of the BufferQueue slots.
297    // If we find a match, release that slot.  If we don't, the BufferQueue
298    // has dropped that GraphicBuffer, and there's nothing for us to release.
299    int id = codecBuffer.mBuf;
300    sp<Fence> fence = new Fence(fenceFd);
301    if (mBufferSlot[id] != NULL &&
302        mBufferSlot[id]->handle == codecBuffer.mGraphicBuffer->handle) {
303        ALOGV("cbi %d matches bq slot %d, handle=%p",
304                cbi, id, mBufferSlot[id]->handle);
305
306        if (id == mLatestBufferId) {
307            CHECK_GT(mLatestBufferUseCount--, 0);
308        } else {
309            if (mIsPersistent) {
310                mConsumer->detachBuffer(id);
311                int outSlot;
312                mConsumer->attachBuffer(&outSlot, mBufferSlot[id]);
313                mConsumer->releaseBuffer(outSlot, 0,
314                        EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
315            } else {
316                mConsumer->releaseBuffer(id, codecBuffer.mFrameNumber,
317                        EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
318            }
319        }
320    } else {
321        ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d",
322                cbi);
323        // we will not reuse codec buffer, so there is no need to wait for fence
324    }
325
326    // Mark the codec buffer as available by clearing the GraphicBuffer ref.
327    codecBuffer.mGraphicBuffer = NULL;
328
329    if (mNumFramesAvailable) {
330        // Fill this codec buffer.
331        CHECK(!mEndOfStreamSent);
332        ALOGV("buffer freed, %zu frames avail (eos=%d)",
333                mNumFramesAvailable, mEndOfStream);
334        fillCodecBuffer_l();
335    } else if (mEndOfStream) {
336        // No frames available, but EOS is pending, so use this buffer to
337        // send that.
338        ALOGV("buffer freed, EOS pending");
339        submitEndOfInputStream_l();
340    } else if (mRepeatBufferDeferred) {
341        bool success = repeatLatestBuffer_l();
342        if (success) {
343            ALOGV("deferred repeatLatestBuffer_l SUCCESS");
344        } else {
345            ALOGV("deferred repeatLatestBuffer_l FAILURE");
346        }
347        mRepeatBufferDeferred = false;
348    }
349
350    return;
351}
352
353void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) {
354    Mutex::Autolock autoLock(mMutex);
355
356    if (mMaxTimestampGapUs > 0ll
357            && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) {
358        ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp);
359        if (index >= 0) {
360            ALOGV("OUT timestamp: %lld -> %lld",
361                    static_cast<long long>(header->nTimeStamp),
362                    static_cast<long long>(mOriginalTimeUs[index]));
363            header->nTimeStamp = mOriginalTimeUs[index];
364            mOriginalTimeUs.removeItemsAt(index);
365        } else {
366            // giving up the effort as encoder doesn't appear to preserve pts
367            ALOGW("giving up limiting timestamp gap (pts = %lld)",
368                    header->nTimeStamp);
369            mMaxTimestampGapUs = -1ll;
370        }
371        if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) {
372            // something terribly wrong must have happened, giving up...
373            ALOGE("mOriginalTimeUs has too many entries (%zu)",
374                    mOriginalTimeUs.size());
375            mMaxTimestampGapUs = -1ll;
376        }
377    }
378}
379
380void GraphicBufferSource::suspend(bool suspend) {
381    Mutex::Autolock autoLock(mMutex);
382
383    if (suspend) {
384        mSuspended = true;
385
386        while (mNumFramesAvailable > 0) {
387            BufferItem item;
388            status_t err = mConsumer->acquireBuffer(&item, 0);
389
390            if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
391                // shouldn't happen.
392                ALOGW("suspend: frame was not available");
393                break;
394            } else if (err != OK) {
395                ALOGW("suspend: acquireBuffer returned err=%d", err);
396                break;
397            }
398
399            --mNumFramesAvailable;
400
401            if (mIsPersistent) {
402                mConsumer->detachBuffer(item.mBuf);
403                mConsumer->attachBuffer(&item.mBuf, item.mGraphicBuffer);
404                mConsumer->releaseBuffer(item.mBuf, 0,
405                        EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence);
406            } else {
407                mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber,
408                        EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence);
409            }
410        }
411        return;
412    }
413
414    mSuspended = false;
415
416    if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) {
417        if (repeatLatestBuffer_l()) {
418            ALOGV("suspend/deferred repeatLatestBuffer_l SUCCESS");
419
420            mRepeatBufferDeferred = false;
421        } else {
422            ALOGV("suspend/deferred repeatLatestBuffer_l FAILURE");
423        }
424    }
425}
426
427bool GraphicBufferSource::fillCodecBuffer_l() {
428    CHECK(mExecuting && mNumFramesAvailable > 0);
429
430    if (mSuspended) {
431        return false;
432    }
433
434    int cbi = findAvailableCodecBuffer_l();
435    if (cbi < 0) {
436        // No buffers available, bail.
437        ALOGV("fillCodecBuffer_l: no codec buffers, avail now %zu",
438                mNumFramesAvailable);
439        return false;
440    }
441
442    ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%zu",
443            mNumFramesAvailable);
444    BufferItem item;
445    status_t err = mConsumer->acquireBuffer(&item, 0);
446    if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
447        // shouldn't happen
448        ALOGW("fillCodecBuffer_l: frame was not available");
449        return false;
450    } else if (err != OK) {
451        // now what? fake end-of-stream?
452        ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err);
453        return false;
454    }
455
456    mNumFramesAvailable--;
457
458    // If this is the first time we're seeing this buffer, add it to our
459    // slot table.
460    if (item.mGraphicBuffer != NULL) {
461        ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf);
462        mBufferSlot[item.mBuf] = item.mGraphicBuffer;
463    }
464
465    err = UNKNOWN_ERROR;
466
467    // only submit sample if start time is unspecified, or sample
468    // is queued after the specified start time
469    bool dropped = false;
470    if (mSkipFramesBeforeNs < 0ll || item.mTimestamp >= mSkipFramesBeforeNs) {
471        // if start time is set, offset time stamp by start time
472        if (mSkipFramesBeforeNs > 0) {
473            item.mTimestamp -= mSkipFramesBeforeNs;
474        }
475
476        int64_t timeUs = item.mTimestamp / 1000;
477        if (mFrameDropper != NULL && mFrameDropper->shouldDrop(timeUs)) {
478            ALOGV("skipping frame (%lld) to meet max framerate", static_cast<long long>(timeUs));
479            // set err to OK so that the skipped frame can still be saved as the lastest frame
480            err = OK;
481            dropped = true;
482        } else {
483            err = submitBuffer_l(item, cbi);
484        }
485    }
486
487    if (err != OK) {
488        ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf);
489        if (mIsPersistent) {
490            mConsumer->detachBuffer(item.mBuf);
491            mConsumer->attachBuffer(&item.mBuf, item.mGraphicBuffer);
492            mConsumer->releaseBuffer(item.mBuf, 0,
493                    EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence);
494        } else {
495            mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber,
496                    EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence);
497        }
498        // item.mFence is released at the end of this method
499    } else {
500        ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi);
501        setLatestBuffer_l(item, dropped);
502    }
503
504    return true;
505}
506
507bool GraphicBufferSource::repeatLatestBuffer_l() {
508    CHECK(mExecuting && mNumFramesAvailable == 0);
509
510    if (mLatestBufferId < 0 || mSuspended) {
511        return false;
512    }
513    if (mBufferSlot[mLatestBufferId] == NULL) {
514        // This can happen if the remote side disconnects, causing
515        // onBuffersReleased() to NULL out our copy of the slots.  The
516        // buffer is gone, so we have nothing to show.
517        //
518        // To be on the safe side we try to release the buffer.
519        ALOGD("repeatLatestBuffer_l: slot was NULL");
520        mConsumer->releaseBuffer(
521                mLatestBufferId,
522                mLatestBufferFrameNum,
523                EGL_NO_DISPLAY,
524                EGL_NO_SYNC_KHR,
525                mLatestBufferFence);
526        mLatestBufferId = -1;
527        mLatestBufferFrameNum = 0;
528        mLatestBufferFence = Fence::NO_FENCE;
529        return false;
530    }
531
532    int cbi = findAvailableCodecBuffer_l();
533    if (cbi < 0) {
534        // No buffers available, bail.
535        ALOGV("repeatLatestBuffer_l: no codec buffers.");
536        return false;
537    }
538
539    BufferItem item;
540    item.mBuf = mLatestBufferId;
541    item.mFrameNumber = mLatestBufferFrameNum;
542    item.mTimestamp = mRepeatLastFrameTimestamp;
543    item.mFence = mLatestBufferFence;
544
545    status_t err = submitBuffer_l(item, cbi);
546
547    if (err != OK) {
548        return false;
549    }
550
551    ++mLatestBufferUseCount;
552
553    /* repeat last frame up to kRepeatLastFrameCount times.
554     * in case of static scene, a single repeat might not get rid of encoder
555     * ghosting completely, refresh a couple more times to get better quality
556     */
557    if (--mRepeatLastFrameCount > 0) {
558        mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
559
560        if (mReflector != NULL) {
561            sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector);
562            msg->setInt32("generation", ++mRepeatLastFrameGeneration);
563            msg->post(mRepeatAfterUs);
564        }
565    }
566
567    return true;
568}
569
570void GraphicBufferSource::setLatestBuffer_l(
571        const BufferItem &item, bool dropped) {
572    ALOGV("setLatestBuffer_l");
573
574    if (mLatestBufferId >= 0) {
575        if (mLatestBufferUseCount == 0) {
576            if (mIsPersistent) {
577                mConsumer->detachBuffer(mLatestBufferId);
578
579                int outSlot;
580                mConsumer->attachBuffer(&outSlot, mBufferSlot[mLatestBufferId]);
581
582                mConsumer->releaseBuffer(outSlot, 0,
583                        EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, mLatestBufferFence);
584            } else {
585                mConsumer->releaseBuffer(
586                        mLatestBufferId, mLatestBufferFrameNum,
587                        EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, mLatestBufferFence);
588            }
589            // mLatestBufferFence will be set to new fence just below
590        }
591    }
592
593    mLatestBufferId = item.mBuf;
594    mLatestBufferFrameNum = item.mFrameNumber;
595    mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
596
597    mLatestBufferUseCount = dropped ? 0 : 1;
598    mRepeatBufferDeferred = false;
599    mRepeatLastFrameCount = kRepeatLastFrameCount;
600    mLatestBufferFence = item.mFence;
601
602    if (mReflector != NULL) {
603        sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector);
604        msg->setInt32("generation", ++mRepeatLastFrameGeneration);
605        msg->post(mRepeatAfterUs);
606    }
607}
608
609status_t GraphicBufferSource::signalEndOfInputStream() {
610    Mutex::Autolock autoLock(mMutex);
611    ALOGV("signalEndOfInputStream: exec=%d avail=%zu eos=%d",
612            mExecuting, mNumFramesAvailable, mEndOfStream);
613
614    if (mEndOfStream) {
615        ALOGE("EOS was already signaled");
616        return INVALID_OPERATION;
617    }
618
619    // Set the end-of-stream flag.  If no frames are pending from the
620    // BufferQueue, and a codec buffer is available, and we're executing,
621    // we initiate the EOS from here.  Otherwise, we'll let
622    // codecBufferEmptied() (or omxExecuting) do it.
623    //
624    // Note: if there are no pending frames and all codec buffers are
625    // available, we *must* submit the EOS from here or we'll just
626    // stall since no future events are expected.
627    mEndOfStream = true;
628
629    if (mExecuting && mNumFramesAvailable == 0) {
630        submitEndOfInputStream_l();
631    }
632
633    return OK;
634}
635
636int64_t GraphicBufferSource::getTimestamp(const BufferItem &item) {
637    int64_t timeUs = item.mTimestamp / 1000;
638
639    if (mTimePerCaptureUs > 0ll) {
640        // Time lapse or slow motion mode
641        if (mPrevCaptureUs < 0ll) {
642            // first capture
643            mPrevCaptureUs = timeUs;
644            mPrevFrameUs = timeUs;
645        } else {
646            // snap to nearest capture point
647            int64_t nFrames = (timeUs + mTimePerCaptureUs / 2 - mPrevCaptureUs)
648                    / mTimePerCaptureUs;
649            if (nFrames <= 0) {
650                // skip this frame as it's too close to previous capture
651                ALOGV("skipping frame, timeUs %lld", static_cast<long long>(timeUs));
652                return -1;
653            }
654            mPrevCaptureUs = mPrevCaptureUs + nFrames * mTimePerCaptureUs;
655            mPrevFrameUs += mTimePerFrameUs * nFrames;
656        }
657
658        ALOGV("timeUs %lld, captureUs %lld, frameUs %lld",
659                static_cast<long long>(timeUs),
660                static_cast<long long>(mPrevCaptureUs),
661                static_cast<long long>(mPrevFrameUs));
662
663        return mPrevFrameUs;
664    } else if (mMaxTimestampGapUs > 0ll) {
665        /* Cap timestamp gap between adjacent frames to specified max
666         *
667         * In the scenario of cast mirroring, encoding could be suspended for
668         * prolonged periods. Limiting the pts gap to workaround the problem
669         * where encoder's rate control logic produces huge frames after a
670         * long period of suspension.
671         */
672
673        int64_t originalTimeUs = timeUs;
674        if (mPrevOriginalTimeUs >= 0ll) {
675            if (originalTimeUs < mPrevOriginalTimeUs) {
676                // Drop the frame if it's going backward in time. Bad timestamp
677                // could disrupt encoder's rate control completely.
678                ALOGW("Dropping frame that's going backward in time");
679                return -1;
680            }
681            int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs;
682            timeUs = (timestampGapUs < mMaxTimestampGapUs ?
683                    timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs;
684        }
685        mPrevOriginalTimeUs = originalTimeUs;
686        mPrevModifiedTimeUs = timeUs;
687        mOriginalTimeUs.add(timeUs, originalTimeUs);
688        ALOGV("IN  timestamp: %lld -> %lld",
689            static_cast<long long>(originalTimeUs),
690            static_cast<long long>(timeUs));
691    }
692
693    return timeUs;
694}
695
696status_t GraphicBufferSource::submitBuffer_l(const BufferItem &item, int cbi) {
697    ALOGV("submitBuffer_l cbi=%d", cbi);
698
699    int64_t timeUs = getTimestamp(item);
700    if (timeUs < 0ll) {
701        return UNKNOWN_ERROR;
702    }
703
704    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
705    codecBuffer.mGraphicBuffer = mBufferSlot[item.mBuf];
706    codecBuffer.mBuf = item.mBuf;
707    codecBuffer.mFrameNumber = item.mFrameNumber;
708
709    OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
710    sp<GraphicBuffer> buffer = codecBuffer.mGraphicBuffer;
711    status_t err = mNodeInstance->emptyGraphicBuffer(
712            header, buffer, OMX_BUFFERFLAG_ENDOFFRAME, timeUs,
713            item.mFence->isValid() ? item.mFence->dup() : -1);
714    if (err != OK) {
715        ALOGW("WARNING: emptyNativeWindowBuffer failed: 0x%x", err);
716        codecBuffer.mGraphicBuffer = NULL;
717        return err;
718    }
719
720    ALOGV("emptyNativeWindowBuffer succeeded, h=%p p=%p buf=%p bufhandle=%p",
721            header, header->pBuffer, buffer->getNativeBuffer(), buffer->handle);
722    return OK;
723}
724
725void GraphicBufferSource::submitEndOfInputStream_l() {
726    CHECK(mEndOfStream);
727    if (mEndOfStreamSent) {
728        ALOGV("EOS already sent");
729        return;
730    }
731
732    int cbi = findAvailableCodecBuffer_l();
733    if (cbi < 0) {
734        ALOGV("submitEndOfInputStream_l: no codec buffers available");
735        return;
736    }
737
738    // We reject any additional incoming graphic buffers, so there's no need
739    // to stick a placeholder into codecBuffer.mGraphicBuffer to mark it as
740    // in-use.
741    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
742
743    OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
744    status_t err = mNodeInstance->emptyGraphicBuffer(
745            header, NULL /* buffer */, OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_EOS,
746            0 /* timestamp */, -1 /* fenceFd */);
747    if (err != OK) {
748        ALOGW("emptyDirectBuffer EOS failed: 0x%x", err);
749    } else {
750        ALOGV("submitEndOfInputStream_l: buffer submitted, header=%p cbi=%d",
751                header, cbi);
752        mEndOfStreamSent = true;
753    }
754}
755
756int GraphicBufferSource::findAvailableCodecBuffer_l() {
757    CHECK(mCodecBuffers.size() > 0);
758
759    for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
760        if (mCodecBuffers[i].mGraphicBuffer == NULL) {
761            return i;
762        }
763    }
764    return -1;
765}
766
767int GraphicBufferSource::findMatchingCodecBuffer_l(
768        const OMX_BUFFERHEADERTYPE* header) {
769    for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
770        if (mCodecBuffers[i].mHeader == header) {
771            return i;
772        }
773    }
774    return -1;
775}
776
777// BufferQueue::ConsumerListener callback
778void GraphicBufferSource::onFrameAvailable(const BufferItem& /*item*/) {
779    Mutex::Autolock autoLock(mMutex);
780
781    ALOGV("onFrameAvailable exec=%d avail=%zu",
782            mExecuting, mNumFramesAvailable);
783
784    if (mEndOfStream || mSuspended) {
785        if (mEndOfStream) {
786            // This should only be possible if a new buffer was queued after
787            // EOS was signaled, i.e. the app is misbehaving.
788
789            ALOGW("onFrameAvailable: EOS is set, ignoring frame");
790        } else {
791            ALOGV("onFrameAvailable: suspended, ignoring frame");
792        }
793
794        BufferItem item;
795        status_t err = mConsumer->acquireBuffer(&item, 0);
796        if (err == OK) {
797            // If this is the first time we're seeing this buffer, add it to our
798            // slot table.
799            if (item.mGraphicBuffer != NULL) {
800                ALOGV("onFrameAvailable: setting mBufferSlot %d", item.mBuf);
801                mBufferSlot[item.mBuf] = item.mGraphicBuffer;
802            }
803
804            if (mIsPersistent) {
805                mConsumer->detachBuffer(item.mBuf);
806                mConsumer->attachBuffer(&item.mBuf, item.mGraphicBuffer);
807                mConsumer->releaseBuffer(item.mBuf, 0,
808                        EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence);
809            } else {
810                mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber,
811                        EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence);
812            }
813        }
814        return;
815    }
816
817    mNumFramesAvailable++;
818
819    mRepeatBufferDeferred = false;
820    ++mRepeatLastFrameGeneration;
821
822    if (mExecuting) {
823        fillCodecBuffer_l();
824    }
825}
826
827// BufferQueue::ConsumerListener callback
828void GraphicBufferSource::onBuffersReleased() {
829    Mutex::Autolock lock(mMutex);
830
831    uint64_t slotMask;
832    if (mConsumer->getReleasedBuffers(&slotMask) != NO_ERROR) {
833        ALOGW("onBuffersReleased: unable to get released buffer set");
834        slotMask = 0xffffffffffffffffULL;
835    }
836
837    ALOGV("onBuffersReleased: 0x%016" PRIx64, slotMask);
838
839    for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) {
840        if ((slotMask & 0x01) != 0) {
841            mBufferSlot[i] = NULL;
842        }
843        slotMask >>= 1;
844    }
845}
846
847// BufferQueue::ConsumerListener callback
848void GraphicBufferSource::onSidebandStreamChanged() {
849    ALOG_ASSERT(false, "GraphicBufferSource can't consume sideband streams");
850}
851
852status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
853        int64_t repeatAfterUs) {
854    Mutex::Autolock autoLock(mMutex);
855
856    if (mExecuting || repeatAfterUs <= 0ll) {
857        return INVALID_OPERATION;
858    }
859
860    mRepeatAfterUs = repeatAfterUs;
861
862    return OK;
863}
864
865status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) {
866    Mutex::Autolock autoLock(mMutex);
867
868    if (mExecuting || maxGapUs <= 0ll) {
869        return INVALID_OPERATION;
870    }
871
872    mMaxTimestampGapUs = maxGapUs;
873
874    return OK;
875}
876
877status_t GraphicBufferSource::setMaxFps(float maxFps) {
878    Mutex::Autolock autoLock(mMutex);
879
880    if (mExecuting) {
881        return INVALID_OPERATION;
882    }
883
884    mFrameDropper = new FrameDropper();
885    status_t err = mFrameDropper->setMaxFrameRate(maxFps);
886    if (err != OK) {
887        mFrameDropper.clear();
888        return err;
889    }
890
891    return OK;
892}
893
894void GraphicBufferSource::setSkipFramesBeforeUs(int64_t skipFramesBeforeUs) {
895    Mutex::Autolock autoLock(mMutex);
896
897    mSkipFramesBeforeNs =
898            (skipFramesBeforeUs > 0) ? (skipFramesBeforeUs * 1000) : -1ll;
899}
900
901status_t GraphicBufferSource::setTimeLapseUs(int64_t* data) {
902    Mutex::Autolock autoLock(mMutex);
903
904    if (mExecuting || data[0] <= 0ll || data[1] <= 0ll) {
905        return INVALID_OPERATION;
906    }
907
908    mTimePerFrameUs = data[0];
909    mTimePerCaptureUs = data[1];
910
911    return OK;
912}
913
914void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
915    switch (msg->what()) {
916        case kWhatRepeatLastFrame:
917        {
918            Mutex::Autolock autoLock(mMutex);
919
920            int32_t generation;
921            CHECK(msg->findInt32("generation", &generation));
922
923            if (generation != mRepeatLastFrameGeneration) {
924                // stale
925                break;
926            }
927
928            if (!mExecuting || mNumFramesAvailable > 0) {
929                break;
930            }
931
932            bool success = repeatLatestBuffer_l();
933
934            if (success) {
935                ALOGV("repeatLatestBuffer_l SUCCESS");
936            } else {
937                ALOGV("repeatLatestBuffer_l FAILURE");
938                mRepeatBufferDeferred = true;
939            }
940            break;
941        }
942
943        default:
944            TRESPASS();
945    }
946}
947
948}  // namespace android
949