GraphicBufferSource.cpp revision 61fcfd1b0b58dff9284ede8dc49749ca7395856d 
1/*
2 * Copyright (C) 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include <inttypes.h>
18
19#define LOG_TAG "GraphicBufferSource"
20//#define LOG_NDEBUG 0
21#include <utils/Log.h>
22
23#define STRINGIFY_ENUMS // for asString in HardwareAPI.h/VideoAPI.h
24
25#include "GraphicBufferSource.h"
26#include "OMXUtils.h"
27
28#include <OMX_Core.h>
29#include <OMX_IndexExt.h>
30#include <media/stagefright/foundation/ADebug.h>
31#include <media/stagefright/foundation/AMessage.h>
32#include <media/stagefright/foundation/ColorUtils.h>
33
34#include <media/hardware/MetadataBufferType.h>
35#include <ui/GraphicBuffer.h>
36#include <gui/BufferItem.h>
37#include <HardwareAPI.h>
38
39#include <inttypes.h>
40#include "FrameDropper.h"
41
42namespace android {
43
44static const bool EXTRA_CHECK = true;
45
46static const OMX_U32 kPortIndexInput = 0;
47
48GraphicBufferSource::PersistentProxyListener::PersistentProxyListener(
49        const wp<IGraphicBufferConsumer> &consumer,
50        const wp<ConsumerListener>& consumerListener) :
51    mConsumerListener(consumerListener),
52    mConsumer(consumer) {}
53
54GraphicBufferSource::PersistentProxyListener::~PersistentProxyListener() {}
55
56void GraphicBufferSource::PersistentProxyListener::onFrameAvailable(
57        const BufferItem& item) {
58    sp<ConsumerListener> listener(mConsumerListener.promote());
59    if (listener != NULL) {
60        listener->onFrameAvailable(item);
61    } else {
62        sp<IGraphicBufferConsumer> consumer(mConsumer.promote());
63        if (consumer == NULL) {
64            return;
65        }
66        BufferItem bi;
67        status_t err = consumer->acquireBuffer(&bi, 0);
68        if (err != OK) {
69            ALOGE("PersistentProxyListener: acquireBuffer failed (%d)", err);
70            return;
71        }
72
73        err = consumer->detachBuffer(bi.mSlot);
74        if (err != OK) {
75            ALOGE("PersistentProxyListener: detachBuffer failed (%d)", err);
76            return;
77        }
78
79        err = consumer->attachBuffer(&bi.mSlot, bi.mGraphicBuffer);
80        if (err != OK) {
81            ALOGE("PersistentProxyListener: attachBuffer failed (%d)", err);
82            return;
83        }
84
85        err = consumer->releaseBuffer(bi.mSlot, 0,
86                EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, bi.mFence);
87        if (err != OK) {
88            ALOGE("PersistentProxyListener: releaseBuffer failed (%d)", err);
89        }
90    }
91}
92
93void GraphicBufferSource::PersistentProxyListener::onFrameReplaced(
94        const BufferItem& item) {
95    sp<ConsumerListener> listener(mConsumerListener.promote());
96    if (listener != NULL) {
97        listener->onFrameReplaced(item);
98    }
99}
100
101void GraphicBufferSource::PersistentProxyListener::onBuffersReleased() {
102    sp<ConsumerListener> listener(mConsumerListener.promote());
103    if (listener != NULL) {
104        listener->onBuffersReleased();
105    }
106}
107
108void GraphicBufferSource::PersistentProxyListener::onSidebandStreamChanged() {
109    sp<ConsumerListener> listener(mConsumerListener.promote());
110    if (listener != NULL) {
111        listener->onSidebandStreamChanged();
112    }
113}
114
115GraphicBufferSource::GraphicBufferSource(
116        OMXNodeInstance* nodeInstance,
117        uint32_t bufferWidth,
118        uint32_t bufferHeight,
119        uint32_t bufferCount,
120        uint32_t consumerUsage,
121        const sp<IGraphicBufferConsumer> &consumer) :
122    mInitCheck(UNKNOWN_ERROR),
123    mNodeInstance(nodeInstance),
124    mExecuting(false),
125    mSuspended(false),
126    mLastDataSpace(HAL_DATASPACE_UNKNOWN),
127    mIsPersistent(false),
128    mConsumer(consumer),
129    mNumFramesAvailable(0),
130    mNumBufferAcquired(0),
131    mEndOfStream(false),
132    mEndOfStreamSent(false),
133    mMaxTimestampGapUs(-1ll),
134    mPrevOriginalTimeUs(-1ll),
135    mPrevModifiedTimeUs(-1ll),
136    mSkipFramesBeforeNs(-1ll),
137    mRepeatAfterUs(-1ll),
138    mRepeatLastFrameGeneration(0),
139    mRepeatLastFrameTimestamp(-1ll),
140    mLatestBufferId(-1),
141    mLatestBufferFrameNum(0),
142    mLatestBufferUseCount(0),
143    mLatestBufferFence(Fence::NO_FENCE),
144    mRepeatBufferDeferred(false),
145    mTimePerCaptureUs(-1ll),
146    mTimePerFrameUs(-1ll),
147    mPrevCaptureUs(-1ll),
148    mPrevFrameUs(-1ll),
149    mInputBufferTimeOffsetUs(0ll) {
150
151    ALOGV("GraphicBufferSource w=%u h=%u c=%u",
152            bufferWidth, bufferHeight, bufferCount);
153
154    if (bufferWidth == 0 || bufferHeight == 0) {
155        ALOGE("Invalid dimensions %ux%u", bufferWidth, bufferHeight);
156        mInitCheck = BAD_VALUE;
157        return;
158    }
159
160    if (mConsumer == NULL) {
161        String8 name("GraphicBufferSource");
162
163        BufferQueue::createBufferQueue(&mProducer, &mConsumer);
164        mConsumer->setConsumerName(name);
165
166        // use consumer usage bits queried from encoder, but always add HW_VIDEO_ENCODER
167        // for backward compatibility.
168        consumerUsage |= GRALLOC_USAGE_HW_VIDEO_ENCODER;
169        mConsumer->setConsumerUsageBits(consumerUsage);
170
171        mInitCheck = mConsumer->setMaxAcquiredBufferCount(bufferCount);
172        if (mInitCheck != NO_ERROR) {
173            ALOGE("Unable to set BQ max acquired buffer count to %u: %d",
174                    bufferCount, mInitCheck);
175            return;
176        }
177    } else {
178        mIsPersistent = true;
179    }
180    mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight);
181    // Note that we can't create an sp<...>(this) in a ctor that will not keep a
182    // reference once the ctor ends, as that would cause the refcount of 'this'
183    // dropping to 0 at the end of the ctor.  Since all we need is a wp<...>
184    // that's what we create.
185    wp<BufferQueue::ConsumerListener> listener = static_cast<BufferQueue::ConsumerListener*>(this);
186    sp<IConsumerListener> proxy;
187    if (!mIsPersistent) {
188        proxy = new BufferQueue::ProxyConsumerListener(listener);
189    } else {
190        proxy = new PersistentProxyListener(mConsumer, listener);
191    }
192
193    mInitCheck = mConsumer->consumerConnect(proxy, false);
194    if (mInitCheck != NO_ERROR) {
195        ALOGE("Error connecting to BufferQueue: %s (%d)",
196                strerror(-mInitCheck), mInitCheck);
197        return;
198    }
199
200    memset(&mColorAspects, 0, sizeof(mColorAspects));
201
202    CHECK(mInitCheck == NO_ERROR);
203}
204
205GraphicBufferSource::~GraphicBufferSource() {
206    if (mLatestBufferId >= 0) {
207        releaseBuffer(
208                mLatestBufferId, mLatestBufferFrameNum,
209                mBufferSlot[mLatestBufferId], mLatestBufferFence);
210    }
211    if (mNumBufferAcquired != 0) {
212        ALOGW("potential buffer leak (acquired %d)", mNumBufferAcquired);
213    }
214    if (mConsumer != NULL && !mIsPersistent) {
215        status_t err = mConsumer->consumerDisconnect();
216        if (err != NO_ERROR) {
217            ALOGW("consumerDisconnect failed: %d", err);
218        }
219    }
220}
221
222void GraphicBufferSource::omxExecuting() {
223    Mutex::Autolock autoLock(mMutex);
224    ALOGV("--> executing; avail=%zu, codec vec size=%zd",
225            mNumFramesAvailable, mCodecBuffers.size());
226    CHECK(!mExecuting);
227    mExecuting = true;
228    mLastDataSpace = HAL_DATASPACE_UNKNOWN;
229    ALOGV("clearing last dataSpace");
230
231    // Start by loading up as many buffers as possible.  We want to do this,
232    // rather than just submit the first buffer, to avoid a degenerate case:
233    // if all BQ buffers arrive before we start executing, and we only submit
234    // one here, the other BQ buffers will just sit until we get notified
235    // that the codec buffer has been released.  We'd then acquire and
236    // submit a single additional buffer, repeatedly, never using more than
237    // one codec buffer simultaneously.  (We could instead try to submit
238    // all BQ buffers whenever any codec buffer is freed, but if we get the
239    // initial conditions right that will never be useful.)
240    while (mNumFramesAvailable) {
241        if (!fillCodecBuffer_l()) {
242            ALOGV("stop load with frames available (codecAvail=%d)",
243                    isCodecBufferAvailable_l());
244            break;
245        }
246    }
247
248    ALOGV("done loading initial frames, avail=%zu", mNumFramesAvailable);
249
250    // If EOS has already been signaled, and there are no more frames to
251    // submit, try to send EOS now as well.
252    if (mEndOfStream && mNumFramesAvailable == 0) {
253        submitEndOfInputStream_l();
254    }
255
256    if (mRepeatAfterUs > 0ll && mLooper == NULL) {
257        mReflector = new AHandlerReflector<GraphicBufferSource>(this);
258
259        mLooper = new ALooper;
260        mLooper->registerHandler(mReflector);
261        mLooper->start();
262
263        if (mLatestBufferId >= 0) {
264            sp<AMessage> msg =
265                new AMessage(kWhatRepeatLastFrame, mReflector);
266
267            msg->setInt32("generation", ++mRepeatLastFrameGeneration);
268            msg->post(mRepeatAfterUs);
269        }
270    }
271}
272
273void GraphicBufferSource::omxIdle() {
274    ALOGV("omxIdle");
275
276    Mutex::Autolock autoLock(mMutex);
277
278    if (mExecuting) {
279        // We are only interested in the transition from executing->idle,
280        // not loaded->idle.
281        mExecuting = false;
282    }
283}
284
285void GraphicBufferSource::omxLoaded(){
286    Mutex::Autolock autoLock(mMutex);
287    if (!mExecuting) {
288        // This can happen if something failed very early.
289        ALOGW("Dropped back down to Loaded without Executing");
290    }
291
292    if (mLooper != NULL) {
293        mLooper->unregisterHandler(mReflector->id());
294        mReflector.clear();
295
296        mLooper->stop();
297        mLooper.clear();
298    }
299
300    ALOGV("--> loaded; avail=%zu eos=%d eosSent=%d",
301            mNumFramesAvailable, mEndOfStream, mEndOfStreamSent);
302
303    // Codec is no longer executing.  Discard all codec-related state.
304    mCodecBuffers.clear();
305    // TODO: scan mCodecBuffers to verify that all mGraphicBuffer entries
306    //       are null; complain if not
307
308    mExecuting = false;
309}
310
311void GraphicBufferSource::addCodecBuffer(OMX_BUFFERHEADERTYPE* header) {
312    Mutex::Autolock autoLock(mMutex);
313
314    if (mExecuting) {
315        // This should never happen -- buffers can only be allocated when
316        // transitioning from "loaded" to "idle".
317        ALOGE("addCodecBuffer: buffer added while executing");
318        return;
319    }
320
321    ALOGV("addCodecBuffer h=%p size=%" PRIu32 " p=%p",
322            header, header->nAllocLen, header->pBuffer);
323    CodecBuffer codecBuffer;
324    codecBuffer.mHeader = header;
325    mCodecBuffers.add(codecBuffer);
326}
327
328void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header, int fenceFd) {
329    Mutex::Autolock autoLock(mMutex);
330    if (!mExecuting) {
331        return;
332    }
333
334    int cbi = findMatchingCodecBuffer_l(header);
335    if (cbi < 0) {
336        // This should never happen.
337        ALOGE("codecBufferEmptied: buffer not recognized (h=%p)", header);
338        if (fenceFd >= 0) {
339            ::close(fenceFd);
340        }
341        return;
342    }
343
344    ALOGV("codecBufferEmptied h=%p size=%" PRIu32 " filled=%" PRIu32 " p=%p",
345            header, header->nAllocLen, header->nFilledLen,
346            header->pBuffer);
347    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
348
349    // header->nFilledLen may not be the original value, so we can't compare
350    // that to zero to see of this was the EOS buffer.  Instead we just
351    // see if the GraphicBuffer reference was null, which should only ever
352    // happen for EOS.
353    if (codecBuffer.mGraphicBuffer == NULL) {
354        if (!(mEndOfStream && mEndOfStreamSent)) {
355            // This can happen when broken code sends us the same buffer
356            // twice in a row.
357            ALOGE("ERROR: codecBufferEmptied on non-EOS null buffer "
358                    "(buffer emptied twice?)");
359        }
360        // No GraphicBuffer to deal with, no additional input or output is
361        // expected, so just return.
362        if (fenceFd >= 0) {
363            ::close(fenceFd);
364        }
365        return;
366    }
367
368    if (EXTRA_CHECK && header->nAllocLen >= sizeof(MetadataBufferType)) {
369        // Pull the graphic buffer handle back out of the buffer, and confirm
370        // that it matches expectations.
371        OMX_U8* data = header->pBuffer;
372        MetadataBufferType type = *(MetadataBufferType *)data;
373        if (type == kMetadataBufferTypeGrallocSource
374                && header->nAllocLen >= sizeof(VideoGrallocMetadata)) {
375            VideoGrallocMetadata &grallocMeta = *(VideoGrallocMetadata *)data;
376            if (grallocMeta.pHandle != codecBuffer.mGraphicBuffer->handle) {
377                // should never happen
378                ALOGE("codecBufferEmptied: buffer's handle is %p, expected %p",
379                        grallocMeta.pHandle, codecBuffer.mGraphicBuffer->handle);
380                CHECK(!"codecBufferEmptied: mismatched buffer");
381            }
382        } else if (type == kMetadataBufferTypeANWBuffer
383                && header->nAllocLen >= sizeof(VideoNativeMetadata)) {
384            VideoNativeMetadata &nativeMeta = *(VideoNativeMetadata *)data;
385            if (nativeMeta.pBuffer != codecBuffer.mGraphicBuffer->getNativeBuffer()) {
386                // should never happen
387                ALOGE("codecBufferEmptied: buffer is %p, expected %p",
388                        nativeMeta.pBuffer, codecBuffer.mGraphicBuffer->getNativeBuffer());
389                CHECK(!"codecBufferEmptied: mismatched buffer");
390            }
391        }
392    }
393
394    // Find matching entry in our cached copy of the BufferQueue slots.
395    // If we find a match, release that slot.  If we don't, the BufferQueue
396    // has dropped that GraphicBuffer, and there's nothing for us to release.
397    int id = codecBuffer.mSlot;
398    sp<Fence> fence = new Fence(fenceFd);
399    if (mBufferSlot[id] != NULL &&
400        mBufferSlot[id]->handle == codecBuffer.mGraphicBuffer->handle) {
401        ALOGV("cbi %d matches bq slot %d, handle=%p",
402                cbi, id, mBufferSlot[id]->handle);
403
404        if (id == mLatestBufferId) {
405            CHECK_GT(mLatestBufferUseCount--, 0);
406        } else {
407            releaseBuffer(id, codecBuffer.mFrameNumber, mBufferSlot[id], fence);
408        }
409    } else {
410        ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d",
411                cbi);
412        // we will not reuse codec buffer, so there is no need to wait for fence
413    }
414
415    // Mark the codec buffer as available by clearing the GraphicBuffer ref.
416    codecBuffer.mGraphicBuffer = NULL;
417
418    if (mNumFramesAvailable) {
419        // Fill this codec buffer.
420        CHECK(!mEndOfStreamSent);
421        ALOGV("buffer freed, %zu frames avail (eos=%d)",
422                mNumFramesAvailable, mEndOfStream);
423        fillCodecBuffer_l();
424    } else if (mEndOfStream) {
425        // No frames available, but EOS is pending, so use this buffer to
426        // send that.
427        ALOGV("buffer freed, EOS pending");
428        submitEndOfInputStream_l();
429    } else if (mRepeatBufferDeferred) {
430        bool success = repeatLatestBuffer_l();
431        if (success) {
432            ALOGV("deferred repeatLatestBuffer_l SUCCESS");
433        } else {
434            ALOGV("deferred repeatLatestBuffer_l FAILURE");
435        }
436        mRepeatBufferDeferred = false;
437    }
438
439    return;
440}
441
442void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) {
443    Mutex::Autolock autoLock(mMutex);
444
445    if (mMaxTimestampGapUs > 0ll
446            && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) {
447        ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp);
448        if (index >= 0) {
449            ALOGV("OUT timestamp: %lld -> %lld",
450                    static_cast<long long>(header->nTimeStamp),
451                    static_cast<long long>(mOriginalTimeUs[index]));
452            header->nTimeStamp = mOriginalTimeUs[index];
453            mOriginalTimeUs.removeItemsAt(index);
454        } else {
455            // giving up the effort as encoder doesn't appear to preserve pts
456            ALOGW("giving up limiting timestamp gap (pts = %lld)",
457                    header->nTimeStamp);
458            mMaxTimestampGapUs = -1ll;
459        }
460        if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) {
461            // something terribly wrong must have happened, giving up...
462            ALOGE("mOriginalTimeUs has too many entries (%zu)",
463                    mOriginalTimeUs.size());
464            mMaxTimestampGapUs = -1ll;
465        }
466    }
467}
468
469void GraphicBufferSource::suspend(bool suspend) {
470    Mutex::Autolock autoLock(mMutex);
471
472    if (suspend) {
473        mSuspended = true;
474
475        while (mNumFramesAvailable > 0) {
476            BufferItem item;
477            status_t err = mConsumer->acquireBuffer(&item, 0);
478
479            if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
480                // shouldn't happen.
481                ALOGW("suspend: frame was not available");
482                break;
483            } else if (err != OK) {
484                ALOGW("suspend: acquireBuffer returned err=%d", err);
485                break;
486            }
487
488            ++mNumBufferAcquired;
489            --mNumFramesAvailable;
490
491            releaseBuffer(item.mSlot, item.mFrameNumber,
492                    item.mGraphicBuffer, item.mFence);
493        }
494        return;
495    }
496
497    mSuspended = false;
498
499    if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) {
500        if (repeatLatestBuffer_l()) {
501            ALOGV("suspend/deferred repeatLatestBuffer_l SUCCESS");
502
503            mRepeatBufferDeferred = false;
504        } else {
505            ALOGV("suspend/deferred repeatLatestBuffer_l FAILURE");
506        }
507    }
508}
509
510void GraphicBufferSource::onDataSpaceChanged_l(
511        android_dataspace dataSpace, android_pixel_format pixelFormat) {
512    ALOGD("got buffer with new dataSpace #%x", dataSpace);
513    mLastDataSpace = dataSpace;
514
515    if (ColorUtils::convertDataSpaceToV0(dataSpace)) {
516        ColorAspects aspects = mColorAspects; // initially requested aspects
517
518        // request color aspects to encode
519        OMX_INDEXTYPE index;
520        status_t err = mNodeInstance->getExtensionIndex(
521                "OMX.google.android.index.describeColorAspects", &index);
522        if (err == OK) {
523            // V0 dataspace
524            DescribeColorAspectsParams params;
525            InitOMXParams(&params);
526            params.nPortIndex = kPortIndexInput;
527            params.nDataSpace = mLastDataSpace;
528            params.nPixelFormat = pixelFormat;
529            params.bDataSpaceChanged = OMX_TRUE;
530            params.sAspects = mColorAspects;
531
532            err = mNodeInstance->getConfig(index, &params, sizeof(params));
533            if (err == OK) {
534                aspects = params.sAspects;
535                ALOGD("Codec resolved it to (R:%d(%s), P:%d(%s), M:%d(%s), T:%d(%s)) err=%d(%s)",
536                        params.sAspects.mRange, asString(params.sAspects.mRange),
537                        params.sAspects.mPrimaries, asString(params.sAspects.mPrimaries),
538                        params.sAspects.mMatrixCoeffs, asString(params.sAspects.mMatrixCoeffs),
539                        params.sAspects.mTransfer, asString(params.sAspects.mTransfer),
540                        err, asString(err));
541            } else {
542                params.sAspects = aspects;
543                err = OK;
544            }
545            params.bDataSpaceChanged = OMX_FALSE;
546            for (int triesLeft = 2; --triesLeft >= 0; ) {
547                status_t err = mNodeInstance->setConfig(index, &params, sizeof(params));
548                if (err == OK) {
549                    err = mNodeInstance->getConfig(index, &params, sizeof(params));
550                }
551                if (err != OK || !ColorUtils::checkIfAspectsChangedAndUnspecifyThem(
552                        params.sAspects, aspects)) {
553                    // if we can't set or get color aspects, still communicate dataspace to client
554                    break;
555                }
556
557                ALOGW_IF(triesLeft == 0, "Codec repeatedly changed requested ColorAspects.");
558            }
559        }
560
561        ALOGV("Set color aspects to (R:%d(%s), P:%d(%s), M:%d(%s), T:%d(%s)) err=%d(%s)",
562                aspects.mRange, asString(aspects.mRange),
563                aspects.mPrimaries, asString(aspects.mPrimaries),
564                aspects.mMatrixCoeffs, asString(aspects.mMatrixCoeffs),
565                aspects.mTransfer, asString(aspects.mTransfer),
566                err, asString(err));
567
568        // signal client that the dataspace has changed; this will update the output format
569        // TODO: we should tie this to an output buffer somehow, and signal the change
570        // just before the output buffer is returned to the client, but there are many
571        // ways this could fail (e.g. flushing), and we are not yet supporting this scenario.
572
573        mNodeInstance->signalEvent(
574                OMX_EventDataSpaceChanged, dataSpace,
575                (aspects.mRange << 24) | (aspects.mPrimaries << 16)
576                        | (aspects.mMatrixCoeffs << 8) | aspects.mTransfer);
577    }
578}
579
580bool GraphicBufferSource::fillCodecBuffer_l() {
581    CHECK(mExecuting && mNumFramesAvailable > 0);
582
583    if (mSuspended) {
584        return false;
585    }
586
587    int cbi = findAvailableCodecBuffer_l();
588    if (cbi < 0) {
589        // No buffers available, bail.
590        ALOGV("fillCodecBuffer_l: no codec buffers, avail now %zu",
591                mNumFramesAvailable);
592        return false;
593    }
594
595    ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%zu",
596            mNumFramesAvailable);
597    BufferItem item;
598    status_t err = mConsumer->acquireBuffer(&item, 0);
599    if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
600        // shouldn't happen
601        ALOGW("fillCodecBuffer_l: frame was not available");
602        return false;
603    } else if (err != OK) {
604        // now what? fake end-of-stream?
605        ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err);
606        return false;
607    }
608
609    mNumBufferAcquired++;
610    mNumFramesAvailable--;
611
612    // If this is the first time we're seeing this buffer, add it to our
613    // slot table.
614    if (item.mGraphicBuffer != NULL) {
615        ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mSlot);
616        mBufferSlot[item.mSlot] = item.mGraphicBuffer;
617    }
618
619    if (item.mDataSpace != mLastDataSpace) {
620        onDataSpaceChanged_l(
621                item.mDataSpace, (android_pixel_format)mBufferSlot[item.mSlot]->getPixelFormat());
622    }
623
624
625    err = UNKNOWN_ERROR;
626
627    // only submit sample if start time is unspecified, or sample
628    // is queued after the specified start time
629    bool dropped = false;
630    if (mSkipFramesBeforeNs < 0ll || item.mTimestamp >= mSkipFramesBeforeNs) {
631        // if start time is set, offset time stamp by start time
632        if (mSkipFramesBeforeNs > 0) {
633            item.mTimestamp -= mSkipFramesBeforeNs;
634        }
635
636        int64_t timeUs = item.mTimestamp / 1000;
637        if (mFrameDropper != NULL && mFrameDropper->shouldDrop(timeUs)) {
638            ALOGV("skipping frame (%lld) to meet max framerate", static_cast<long long>(timeUs));
639            // set err to OK so that the skipped frame can still be saved as the lastest frame
640            err = OK;
641            dropped = true;
642        } else {
643            err = submitBuffer_l(item, cbi);
644        }
645    }
646
647    if (err != OK) {
648        ALOGV("submitBuffer_l failed, releasing bq slot %d", item.mSlot);
649        releaseBuffer(item.mSlot, item.mFrameNumber, item.mGraphicBuffer, item.mFence);
650    } else {
651        ALOGV("buffer submitted (bq %d, cbi %d)", item.mSlot, cbi);
652        setLatestBuffer_l(item, dropped);
653    }
654
655    return true;
656}
657
658bool GraphicBufferSource::repeatLatestBuffer_l() {
659    CHECK(mExecuting && mNumFramesAvailable == 0);
660
661    if (mLatestBufferId < 0 || mSuspended) {
662        return false;
663    }
664    if (mBufferSlot[mLatestBufferId] == NULL) {
665        // This can happen if the remote side disconnects, causing
666        // onBuffersReleased() to NULL out our copy of the slots.  The
667        // buffer is gone, so we have nothing to show.
668        //
669        // To be on the safe side we try to release the buffer.
670        ALOGD("repeatLatestBuffer_l: slot was NULL");
671        mConsumer->releaseBuffer(
672                mLatestBufferId,
673                mLatestBufferFrameNum,
674                EGL_NO_DISPLAY,
675                EGL_NO_SYNC_KHR,
676                mLatestBufferFence);
677        mLatestBufferId = -1;
678        mLatestBufferFrameNum = 0;
679        mLatestBufferFence = Fence::NO_FENCE;
680        return false;
681    }
682
683    int cbi = findAvailableCodecBuffer_l();
684    if (cbi < 0) {
685        // No buffers available, bail.
686        ALOGV("repeatLatestBuffer_l: no codec buffers.");
687        return false;
688    }
689
690    BufferItem item;
691    item.mSlot = mLatestBufferId;
692    item.mFrameNumber = mLatestBufferFrameNum;
693    item.mTimestamp = mRepeatLastFrameTimestamp;
694    item.mFence = mLatestBufferFence;
695
696    status_t err = submitBuffer_l(item, cbi);
697
698    if (err != OK) {
699        return false;
700    }
701
702    ++mLatestBufferUseCount;
703
704    /* repeat last frame up to kRepeatLastFrameCount times.
705     * in case of static scene, a single repeat might not get rid of encoder
706     * ghosting completely, refresh a couple more times to get better quality
707     */
708    if (--mRepeatLastFrameCount > 0) {
709        mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
710
711        if (mReflector != NULL) {
712            sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector);
713            msg->setInt32("generation", ++mRepeatLastFrameGeneration);
714            msg->post(mRepeatAfterUs);
715        }
716    }
717
718    return true;
719}
720
721void GraphicBufferSource::setLatestBuffer_l(
722        const BufferItem &item, bool dropped) {
723    ALOGV("setLatestBuffer_l");
724
725    if (mLatestBufferId >= 0) {
726        if (mLatestBufferUseCount == 0) {
727            releaseBuffer(mLatestBufferId, mLatestBufferFrameNum,
728                    mBufferSlot[mLatestBufferId], mLatestBufferFence);
729            // mLatestBufferFence will be set to new fence just below
730        }
731    }
732
733    mLatestBufferId = item.mSlot;
734    mLatestBufferFrameNum = item.mFrameNumber;
735    mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
736
737    mLatestBufferUseCount = dropped ? 0 : 1;
738    mRepeatBufferDeferred = false;
739    mRepeatLastFrameCount = kRepeatLastFrameCount;
740    mLatestBufferFence = item.mFence;
741
742    if (mReflector != NULL) {
743        sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector);
744        msg->setInt32("generation", ++mRepeatLastFrameGeneration);
745        msg->post(mRepeatAfterUs);
746    }
747}
748
749status_t GraphicBufferSource::signalEndOfInputStream() {
750    Mutex::Autolock autoLock(mMutex);
751    ALOGV("signalEndOfInputStream: exec=%d avail=%zu eos=%d",
752            mExecuting, mNumFramesAvailable, mEndOfStream);
753
754    if (mEndOfStream) {
755        ALOGE("EOS was already signaled");
756        return INVALID_OPERATION;
757    }
758
759    // Set the end-of-stream flag.  If no frames are pending from the
760    // BufferQueue, and a codec buffer is available, and we're executing,
761    // we initiate the EOS from here.  Otherwise, we'll let
762    // codecBufferEmptied() (or omxExecuting) do it.
763    //
764    // Note: if there are no pending frames and all codec buffers are
765    // available, we *must* submit the EOS from here or we'll just
766    // stall since no future events are expected.
767    mEndOfStream = true;
768
769    if (mExecuting && mNumFramesAvailable == 0) {
770        submitEndOfInputStream_l();
771    }
772
773    return OK;
774}
775
776int64_t GraphicBufferSource::getTimestamp(const BufferItem &item) {
777    int64_t timeUs = item.mTimestamp / 1000;
778    timeUs += mInputBufferTimeOffsetUs;
779
780    if (mTimePerCaptureUs > 0ll
781            && (mTimePerCaptureUs > 2 * mTimePerFrameUs
782            || mTimePerFrameUs > 2 * mTimePerCaptureUs)) {
783        // Time lapse or slow motion mode
784        if (mPrevCaptureUs < 0ll) {
785            // first capture
786            mPrevCaptureUs = timeUs;
787            mPrevFrameUs = timeUs;
788        } else {
789            // snap to nearest capture point
790            int64_t nFrames = (timeUs + mTimePerCaptureUs / 2 - mPrevCaptureUs)
791                    / mTimePerCaptureUs;
792            if (nFrames <= 0) {
793                // skip this frame as it's too close to previous capture
794                ALOGV("skipping frame, timeUs %lld", static_cast<long long>(timeUs));
795                return -1;
796            }
797            mPrevCaptureUs = mPrevCaptureUs + nFrames * mTimePerCaptureUs;
798            mPrevFrameUs += mTimePerFrameUs * nFrames;
799        }
800
801        ALOGV("timeUs %lld, captureUs %lld, frameUs %lld",
802                static_cast<long long>(timeUs),
803                static_cast<long long>(mPrevCaptureUs),
804                static_cast<long long>(mPrevFrameUs));
805
806        return mPrevFrameUs;
807    } else {
808        int64_t originalTimeUs = timeUs;
809        if (originalTimeUs <= mPrevOriginalTimeUs) {
810                // Drop the frame if it's going backward in time. Bad timestamp
811                // could disrupt encoder's rate control completely.
812            ALOGW("Dropping frame that's going backward in time");
813            return -1;
814        }
815
816        if (mMaxTimestampGapUs > 0ll) {
817            //TODO: Fix the case when mMaxTimestampGapUs and mTimePerCaptureUs are both set.
818
819            /* Cap timestamp gap between adjacent frames to specified max
820             *
821             * In the scenario of cast mirroring, encoding could be suspended for
822             * prolonged periods. Limiting the pts gap to workaround the problem
823             * where encoder's rate control logic produces huge frames after a
824             * long period of suspension.
825             */
826            if (mPrevOriginalTimeUs >= 0ll) {
827                int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs;
828                timeUs = (timestampGapUs < mMaxTimestampGapUs ?
829                    timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs;
830                mOriginalTimeUs.add(timeUs, originalTimeUs);
831                ALOGV("IN  timestamp: %lld -> %lld",
832                    static_cast<long long>(originalTimeUs),
833                    static_cast<long long>(timeUs));
834            }
835        }
836
837        mPrevOriginalTimeUs = originalTimeUs;
838        mPrevModifiedTimeUs = timeUs;
839    }
840
841    return timeUs;
842}
843
844status_t GraphicBufferSource::submitBuffer_l(const BufferItem &item, int cbi) {
845    ALOGV("submitBuffer_l cbi=%d", cbi);
846
847    int64_t timeUs = getTimestamp(item);
848    if (timeUs < 0ll) {
849        return UNKNOWN_ERROR;
850    }
851
852    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
853    codecBuffer.mGraphicBuffer = mBufferSlot[item.mSlot];
854    codecBuffer.mSlot = item.mSlot;
855    codecBuffer.mFrameNumber = item.mFrameNumber;
856
857    OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
858    sp<GraphicBuffer> buffer = codecBuffer.mGraphicBuffer;
859    status_t err = mNodeInstance->emptyGraphicBuffer(
860            header, buffer, OMX_BUFFERFLAG_ENDOFFRAME, timeUs,
861            item.mFence->isValid() ? item.mFence->dup() : -1);
862    if (err != OK) {
863        ALOGW("WARNING: emptyNativeWindowBuffer failed: 0x%x", err);
864        codecBuffer.mGraphicBuffer = NULL;
865        return err;
866    }
867
868    ALOGV("emptyNativeWindowBuffer succeeded, h=%p p=%p buf=%p bufhandle=%p",
869            header, header->pBuffer, buffer->getNativeBuffer(), buffer->handle);
870    return OK;
871}
872
873void GraphicBufferSource::submitEndOfInputStream_l() {
874    CHECK(mEndOfStream);
875    if (mEndOfStreamSent) {
876        ALOGV("EOS already sent");
877        return;
878    }
879
880    int cbi = findAvailableCodecBuffer_l();
881    if (cbi < 0) {
882        ALOGV("submitEndOfInputStream_l: no codec buffers available");
883        return;
884    }
885
886    // We reject any additional incoming graphic buffers, so there's no need
887    // to stick a placeholder into codecBuffer.mGraphicBuffer to mark it as
888    // in-use.
889    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
890
891    OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
892    status_t err = mNodeInstance->emptyGraphicBuffer(
893            header, NULL /* buffer */, OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_EOS,
894            0 /* timestamp */, -1 /* fenceFd */);
895    if (err != OK) {
896        ALOGW("emptyDirectBuffer EOS failed: 0x%x", err);
897    } else {
898        ALOGV("submitEndOfInputStream_l: buffer submitted, header=%p cbi=%d",
899                header, cbi);
900        mEndOfStreamSent = true;
901    }
902}
903
904int GraphicBufferSource::findAvailableCodecBuffer_l() {
905    CHECK(mCodecBuffers.size() > 0);
906
907    for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
908        if (mCodecBuffers[i].mGraphicBuffer == NULL) {
909            return i;
910        }
911    }
912    return -1;
913}
914
915int GraphicBufferSource::findMatchingCodecBuffer_l(
916        const OMX_BUFFERHEADERTYPE* header) {
917    for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
918        if (mCodecBuffers[i].mHeader == header) {
919            return i;
920        }
921    }
922    return -1;
923}
924
925/*
926 * Releases an acquired buffer back to the consumer for either persistent
927 * or non-persistent surfaces.
928 *
929 * id: buffer slot to release (in persistent case the id might be changed)
930 * frameNum: frame number of the frame being released
931 * buffer: GraphicBuffer pointer to release (note this must not be & as we
932 *         will clear the original mBufferSlot in persistent case)
933 * fence: fence of the frame being released
934 */
935void GraphicBufferSource::releaseBuffer(
936        int &id, uint64_t frameNum,
937        const sp<GraphicBuffer> buffer, const sp<Fence> &fence) {
938    if (mIsPersistent) {
939        mConsumer->detachBuffer(id);
940        mBufferSlot[id] = NULL;
941
942        if (mConsumer->attachBuffer(&id, buffer) == OK) {
943            mConsumer->releaseBuffer(
944                    id, 0, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
945        }
946    } else {
947        mConsumer->releaseBuffer(
948                id, frameNum, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
949    }
950    id = -1; // invalidate id
951    mNumBufferAcquired--;
952}
953
954// BufferQueue::ConsumerListener callback
955void GraphicBufferSource::onFrameAvailable(const BufferItem& /*item*/) {
956    Mutex::Autolock autoLock(mMutex);
957
958    ALOGV("onFrameAvailable exec=%d avail=%zu",
959            mExecuting, mNumFramesAvailable);
960
961    if (mEndOfStream || mSuspended) {
962        if (mEndOfStream) {
963            // This should only be possible if a new buffer was queued after
964            // EOS was signaled, i.e. the app is misbehaving.
965
966            ALOGW("onFrameAvailable: EOS is set, ignoring frame");
967        } else {
968            ALOGV("onFrameAvailable: suspended, ignoring frame");
969        }
970
971        BufferItem item;
972        status_t err = mConsumer->acquireBuffer(&item, 0);
973        if (err == OK) {
974            mNumBufferAcquired++;
975
976            // If this is the first time we're seeing this buffer, add it to our
977            // slot table.
978            if (item.mGraphicBuffer != NULL) {
979                ALOGV("onFrameAvailable: setting mBufferSlot %d", item.mSlot);
980                mBufferSlot[item.mSlot] = item.mGraphicBuffer;
981            }
982
983            releaseBuffer(item.mSlot, item.mFrameNumber,
984                    item.mGraphicBuffer, item.mFence);
985        }
986        return;
987    }
988
989    mNumFramesAvailable++;
990
991    mRepeatBufferDeferred = false;
992    ++mRepeatLastFrameGeneration;
993
994    if (mExecuting) {
995        fillCodecBuffer_l();
996    }
997}
998
999// BufferQueue::ConsumerListener callback
1000void GraphicBufferSource::onBuffersReleased() {
1001    Mutex::Autolock lock(mMutex);
1002
1003    uint64_t slotMask;
1004    if (mConsumer->getReleasedBuffers(&slotMask) != NO_ERROR) {
1005        ALOGW("onBuffersReleased: unable to get released buffer set");
1006        slotMask = 0xffffffffffffffffULL;
1007    }
1008
1009    ALOGV("onBuffersReleased: 0x%016" PRIx64, slotMask);
1010
1011    for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) {
1012        if ((slotMask & 0x01) != 0) {
1013            mBufferSlot[i] = NULL;
1014        }
1015        slotMask >>= 1;
1016    }
1017}
1018
1019// BufferQueue::ConsumerListener callback
1020void GraphicBufferSource::onSidebandStreamChanged() {
1021    ALOG_ASSERT(false, "GraphicBufferSource can't consume sideband streams");
1022}
1023
1024void GraphicBufferSource::setDefaultDataSpace(android_dataspace dataSpace) {
1025    // no need for mutex as we are not yet running
1026    ALOGD("setting dataspace: %#x", dataSpace);
1027    mConsumer->setDefaultBufferDataSpace(dataSpace);
1028    mLastDataSpace = dataSpace;
1029}
1030
1031status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
1032        int64_t repeatAfterUs) {
1033    Mutex::Autolock autoLock(mMutex);
1034
1035    if (mExecuting || repeatAfterUs <= 0ll) {
1036        return INVALID_OPERATION;
1037    }
1038
1039    mRepeatAfterUs = repeatAfterUs;
1040
1041    return OK;
1042}
1043
1044status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) {
1045    Mutex::Autolock autoLock(mMutex);
1046
1047    if (mExecuting || maxGapUs <= 0ll) {
1048        return INVALID_OPERATION;
1049    }
1050
1051    mMaxTimestampGapUs = maxGapUs;
1052
1053    return OK;
1054}
1055
1056status_t GraphicBufferSource::setInputBufferTimeOffset(int64_t timeOffsetUs) {
1057    Mutex::Autolock autoLock(mMutex);
1058
1059    // timeOffsetUs must be negative for adjustment.
1060    if (timeOffsetUs >= 0ll) {
1061        return INVALID_OPERATION;
1062    }
1063
1064    mInputBufferTimeOffsetUs = timeOffsetUs;
1065    return OK;
1066}
1067
1068status_t GraphicBufferSource::setMaxFps(float maxFps) {
1069    Mutex::Autolock autoLock(mMutex);
1070
1071    if (mExecuting) {
1072        return INVALID_OPERATION;
1073    }
1074
1075    mFrameDropper = new FrameDropper();
1076    status_t err = mFrameDropper->setMaxFrameRate(maxFps);
1077    if (err != OK) {
1078        mFrameDropper.clear();
1079        return err;
1080    }
1081
1082    return OK;
1083}
1084
1085void GraphicBufferSource::setSkipFramesBeforeUs(int64_t skipFramesBeforeUs) {
1086    Mutex::Autolock autoLock(mMutex);
1087
1088    mSkipFramesBeforeNs =
1089            (skipFramesBeforeUs > 0) ? (skipFramesBeforeUs * 1000) : -1ll;
1090}
1091
1092status_t GraphicBufferSource::setTimeLapseConfig(const TimeLapseConfig &config) {
1093    Mutex::Autolock autoLock(mMutex);
1094
1095    if (mExecuting || config.mTimePerFrameUs <= 0ll || config.mTimePerCaptureUs <= 0ll) {
1096        return INVALID_OPERATION;
1097    }
1098
1099    mTimePerFrameUs = config.mTimePerFrameUs;
1100    mTimePerCaptureUs = config.mTimePerCaptureUs;
1101
1102    return OK;
1103}
1104
1105void GraphicBufferSource::setColorAspects(const ColorAspects &aspects) {
1106    Mutex::Autolock autoLock(mMutex);
1107    mColorAspects = aspects;
1108    ALOGD("requesting color aspects (R:%d(%s), P:%d(%s), M:%d(%s), T:%d(%s))",
1109            aspects.mRange, asString(aspects.mRange),
1110            aspects.mPrimaries, asString(aspects.mPrimaries),
1111            aspects.mMatrixCoeffs, asString(aspects.mMatrixCoeffs),
1112            aspects.mTransfer, asString(aspects.mTransfer));
1113}
1114
1115void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
1116    switch (msg->what()) {
1117        case kWhatRepeatLastFrame:
1118        {
1119            Mutex::Autolock autoLock(mMutex);
1120
1121            int32_t generation;
1122            CHECK(msg->findInt32("generation", &generation));
1123
1124            if (generation != mRepeatLastFrameGeneration) {
1125                // stale
1126                break;
1127            }
1128
1129            if (!mExecuting || mNumFramesAvailable > 0) {
1130                break;
1131            }
1132
1133            bool success = repeatLatestBuffer_l();
1134
1135            if (success) {
1136                ALOGV("repeatLatestBuffer_l SUCCESS");
1137            } else {
1138                ALOGV("repeatLatestBuffer_l FAILURE");
1139                mRepeatBufferDeferred = true;
1140            }
1141            break;
1142        }
1143
1144        default:
1145            TRESPASS();
1146    }
1147}
1148
1149}  // namespace android
1150