1/*
2 * Copyright 2014,2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include <inttypes.h>
18
19#define LOG_TAG "Camera3StreamSplitter"
20#define ATRACE_TAG ATRACE_TAG_CAMERA
21//#define LOG_NDEBUG 0
22
23#include <gui/BufferItem.h>
24#include <gui/IGraphicBufferConsumer.h>
25#include <gui/IGraphicBufferProducer.h>
26#include <gui/BufferQueue.h>
27#include <gui/Surface.h>
28
29#include <ui/GraphicBuffer.h>
30
31#include <binder/ProcessState.h>
32
33#include <utils/Trace.h>
34
35#include <cutils/atomic.h>
36
37#include "Camera3StreamSplitter.h"
38
39namespace android {
40
41status_t Camera3StreamSplitter::connect(const std::vector<sp<Surface> >& surfaces,
42        uint32_t consumerUsage, size_t halMaxBuffers, sp<Surface>* consumer) {
43    ATRACE_CALL();
44    if (consumer == nullptr) {
45        SP_LOGE("%s: consumer pointer is NULL", __FUNCTION__);
46        return BAD_VALUE;
47    }
48
49    Mutex::Autolock lock(mMutex);
50    status_t res = OK;
51
52    if (mOutputs.size() > 0 || mConsumer != nullptr) {
53        SP_LOGE("%s: already connected", __FUNCTION__);
54        return BAD_VALUE;
55    }
56    if (mBuffers.size() > 0) {
57        SP_LOGE("%s: still has %zu pending buffers", __FUNCTION__, mBuffers.size());
58        return BAD_VALUE;
59    }
60
61    mMaxHalBuffers = halMaxBuffers;
62    mConsumerName = getUniqueConsumerName();
63    // Add output surfaces. This has to be before creating internal buffer queue
64    // in order to get max consumer side buffers.
65    for (size_t i = 0; i < surfaces.size(); i++) {
66        if (surfaces[i] == nullptr) {
67            SP_LOGE("%s: Fatal: surface is NULL", __FUNCTION__);
68            return BAD_VALUE;
69        }
70        res = addOutputLocked(surfaces[i]);
71        if (res != OK) {
72            SP_LOGE("%s: Failed to add output surface: %s(%d)",
73                    __FUNCTION__, strerror(-res), res);
74            return res;
75        }
76    }
77
78    // Create BufferQueue for input
79    BufferQueue::createBufferQueue(&mProducer, &mConsumer);
80
81    // Allocate 1 extra buffer to handle the case where all buffers are detached
82    // from input, and attached to the outputs. In this case, the input queue's
83    // dequeueBuffer can still allocate 1 extra buffer before being blocked by
84    // the output's attachBuffer().
85    mBufferItemConsumer = new BufferItemConsumer(mConsumer, consumerUsage,
86                                                 mMaxConsumerBuffers+1);
87    if (mBufferItemConsumer == nullptr) {
88        return NO_MEMORY;
89    }
90    mConsumer->setConsumerName(mConsumerName);
91
92    *consumer = new Surface(mProducer);
93    if (*consumer == nullptr) {
94        return NO_MEMORY;
95    }
96
97    res = mConsumer->consumerConnect(this, /* controlledByApp */ false);
98
99    SP_LOGV("%s: connected", __FUNCTION__);
100    return res;
101}
102
103status_t Camera3StreamSplitter::getOnFrameAvailableResult() {
104    ATRACE_CALL();
105    return mOnFrameAvailableRes.load();
106}
107
108void Camera3StreamSplitter::disconnect() {
109    ATRACE_CALL();
110    Mutex::Autolock lock(mMutex);
111
112    for (auto& notifier : mNotifiers) {
113        sp<IGraphicBufferProducer> producer = notifier.first;
114        sp<OutputListener> listener = notifier.second;
115        IInterface::asBinder(producer)->unlinkToDeath(listener);
116    }
117    mNotifiers.clear();
118
119    for (auto& output : mOutputs) {
120        output->disconnect(NATIVE_WINDOW_API_CAMERA);
121    }
122    mOutputs.clear();
123    mOutputSlots.clear();
124
125    mConsumer->consumerDisconnect();
126
127    if (mBuffers.size() > 0) {
128        SP_LOGW("%zu buffers still being tracked", mBuffers.size());
129        mBuffers.clear();
130    }
131
132    mMaxHalBuffers = 0;
133    mMaxConsumerBuffers = 0;
134    SP_LOGV("%s: Disconnected", __FUNCTION__);
135}
136
137
138Camera3StreamSplitter::~Camera3StreamSplitter() {
139    disconnect();
140}
141
142status_t Camera3StreamSplitter::addOutput(const sp<Surface>& outputQueue) {
143    ATRACE_CALL();
144    Mutex::Autolock lock(mMutex);
145    status_t res = addOutputLocked(outputQueue);
146
147    if (res != OK) {
148        SP_LOGE("%s: addOutputLocked failed %d", __FUNCTION__, res);
149        return res;
150    }
151
152    res = mConsumer->setMaxAcquiredBufferCount(mMaxConsumerBuffers+1);
153
154    return res;
155}
156
157status_t Camera3StreamSplitter::addOutputLocked(const sp<Surface>& outputQueue) {
158    ATRACE_CALL();
159    if (outputQueue == nullptr) {
160        SP_LOGE("addOutput: outputQueue must not be NULL");
161        return BAD_VALUE;
162    }
163
164    sp<IGraphicBufferProducer> gbp = outputQueue->getIGraphicBufferProducer();
165    // Connect to the buffer producer
166    sp<OutputListener> listener(new OutputListener(this, gbp));
167    IInterface::asBinder(gbp)->linkToDeath(listener);
168    status_t res = outputQueue->connect(NATIVE_WINDOW_API_CAMERA, listener);
169    if (res != NO_ERROR) {
170        SP_LOGE("addOutput: failed to connect (%d)", res);
171        return res;
172    }
173
174    // Query consumer side buffer count, and update overall buffer count
175    int maxConsumerBuffers = 0;
176    res = static_cast<ANativeWindow*>(outputQueue.get())->query(
177            outputQueue.get(),
178            NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &maxConsumerBuffers);
179    if (res != OK) {
180        SP_LOGE("%s: Unable to query consumer undequeued buffer count"
181              " for surface", __FUNCTION__);
182        return res;
183    }
184
185    SP_LOGV("%s: Consumer wants %d buffers, Producer wants %zu", __FUNCTION__,
186            maxConsumerBuffers, mMaxHalBuffers);
187    size_t totalBufferCount = maxConsumerBuffers + mMaxHalBuffers;
188    res = native_window_set_buffer_count(outputQueue.get(),
189            totalBufferCount);
190    if (res != OK) {
191        SP_LOGE("%s: Unable to set buffer count for surface %p",
192                __FUNCTION__, outputQueue.get());
193        return res;
194    }
195
196    // Set dequeueBuffer/attachBuffer timeout if the consumer is not hw composer or hw texture.
197    // We need skip these cases as timeout will disable the non-blocking (async) mode.
198    int32_t usage = 0;
199    static_cast<ANativeWindow*>(outputQueue.get())->query(
200            outputQueue.get(),
201            NATIVE_WINDOW_CONSUMER_USAGE_BITS, &usage);
202    if (!(usage & (GRALLOC_USAGE_HW_COMPOSER | GRALLOC_USAGE_HW_TEXTURE))) {
203        outputQueue->setDequeueTimeout(kDequeueBufferTimeout);
204    }
205
206    res = gbp->allowAllocation(false);
207    if (res != OK) {
208        SP_LOGE("%s: Failed to turn off allocation for outputQueue", __FUNCTION__);
209        return res;
210    }
211
212    // Add new entry into mOutputs
213    mOutputs.push_back(gbp);
214    mNotifiers[gbp] = listener;
215    mOutputSlots[gbp] = std::make_unique<OutputSlots>(totalBufferCount);
216
217    mMaxConsumerBuffers += maxConsumerBuffers;
218    return NO_ERROR;
219}
220
221status_t Camera3StreamSplitter::outputBufferLocked(const sp<IGraphicBufferProducer>& output,
222        const BufferItem& bufferItem) {
223    ATRACE_CALL();
224    status_t res;
225    IGraphicBufferProducer::QueueBufferInput queueInput(
226            bufferItem.mTimestamp, bufferItem.mIsAutoTimestamp,
227            bufferItem.mDataSpace, bufferItem.mCrop,
228            static_cast<int32_t>(bufferItem.mScalingMode),
229            bufferItem.mTransform, bufferItem.mFence);
230
231    IGraphicBufferProducer::QueueBufferOutput queueOutput;
232
233    uint64_t bufferId = bufferItem.mGraphicBuffer->getId();
234    const BufferTracker& tracker = *(mBuffers[bufferId]);
235    int slot = getSlotForOutputLocked(output, tracker.getBuffer());
236
237    // In case the output BufferQueue has its own lock, if we hold splitter lock while calling
238    // queueBuffer (which will try to acquire the output lock), the output could be holding its
239    // own lock calling releaseBuffer (which  will try to acquire the splitter lock), running into
240    // circular lock situation.
241    mMutex.unlock();
242    res = output->queueBuffer(slot, queueInput, &queueOutput);
243    mMutex.lock();
244
245    SP_LOGV("%s: Queuing buffer to buffer queue %p slot %d returns %d",
246            __FUNCTION__, output.get(), slot, res);
247    if (res != OK) {
248        if (res != NO_INIT && res != DEAD_OBJECT) {
249            SP_LOGE("Queuing buffer to output failed (%d)", res);
250        }
251        // If we just discovered that this output has been abandoned, note
252        // that, increment the release count so that we still release this
253        // buffer eventually, and move on to the next output
254        onAbandonedLocked();
255        decrementBufRefCountLocked(bufferItem.mGraphicBuffer->getId(), output);
256        return res;
257    }
258
259    // If the queued buffer replaces a pending buffer in the async
260    // queue, no onBufferReleased is called by the buffer queue.
261    // Proactively trigger the callback to avoid buffer loss.
262    if (queueOutput.bufferReplaced) {
263        onBufferReleasedByOutputLocked(output);
264    }
265
266    return res;
267}
268
269String8 Camera3StreamSplitter::getUniqueConsumerName() {
270    static volatile int32_t counter = 0;
271    return String8::format("Camera3StreamSplitter-%d", android_atomic_inc(&counter));
272}
273
274status_t Camera3StreamSplitter::notifyBufferReleased(const sp<GraphicBuffer>& buffer) {
275    ATRACE_CALL();
276    status_t res = OK;
277
278    Mutex::Autolock lock(mMutex);
279
280    uint64_t bufferId = buffer->getId();
281    std::unique_ptr<BufferTracker> tracker_ptr = std::move(mBuffers[bufferId]);
282    mBuffers.erase(bufferId);
283
284    for (const auto surface : tracker_ptr->requestedSurfaces()) {
285        sp<IGraphicBufferProducer>& gbp = mOutputs[surface];
286        OutputSlots& outputSlots = *(mOutputSlots[gbp]);
287        int slot = getSlotForOutputLocked(gbp, buffer);
288        if (slot != BufferItem::INVALID_BUFFER_SLOT) {
289             gbp->detachBuffer(slot);
290             outputSlots[slot].clear();
291        }
292    }
293
294    return res;
295}
296
297status_t Camera3StreamSplitter::attachBufferToOutputs(ANativeWindowBuffer* anb,
298        const std::vector<size_t>& surface_ids) {
299    ATRACE_CALL();
300    status_t res = OK;
301
302    Mutex::Autolock lock(mMutex);
303
304    sp<GraphicBuffer> gb(static_cast<GraphicBuffer*>(anb));
305    uint64_t bufferId = gb->getId();
306
307    // Initialize buffer tracker for this input buffer
308    auto tracker = std::make_unique<BufferTracker>(gb, surface_ids);
309
310    for (auto& surface_id : surface_ids) {
311        sp<IGraphicBufferProducer>& gbp = mOutputs[surface_id];
312        int slot = BufferItem::INVALID_BUFFER_SLOT;
313        //Temporarly Unlock the mutex when trying to attachBuffer to the output
314        //queue, because attachBuffer could block in case of a slow consumer. If
315        //we block while holding the lock, onFrameAvailable and onBufferReleased
316        //will block as well because they need to acquire the same lock.
317        mMutex.unlock();
318        res = gbp->attachBuffer(&slot, gb);
319        mMutex.lock();
320        if (res != OK) {
321            SP_LOGE("%s: Cannot acquireBuffer from GraphicBufferProducer %p: %s (%d)",
322                    __FUNCTION__, gbp.get(), strerror(-res), res);
323            return res;
324        }
325        auto& outputSlots = *mOutputSlots[gbp];
326        if (outputSlots[slot] != nullptr) {
327            // If the buffer is attached to a slot which already contains a buffer,
328            // the previous buffer will be removed from the output queue. Decrement
329            // the reference count accordingly.
330            decrementBufRefCountLocked(outputSlots[slot]->getId(), gbp);
331        }
332        SP_LOGV("%s: Attached buffer %p to slot %d on output %p.",__FUNCTION__, gb.get(),
333                slot, gbp.get());
334        outputSlots[slot] = gb;
335    }
336
337    mBuffers[bufferId] = std::move(tracker);
338
339    return res;
340}
341
342void Camera3StreamSplitter::onFrameAvailable(const BufferItem& /*item*/) {
343    ATRACE_CALL();
344    Mutex::Autolock lock(mMutex);
345
346    // Acquire and detach the buffer from the input
347    BufferItem bufferItem;
348    status_t res = mConsumer->acquireBuffer(&bufferItem, /* presentWhen */ 0);
349    if (res != NO_ERROR) {
350        SP_LOGE("%s: Acquiring buffer from input failed (%d)", __FUNCTION__, res);
351        mOnFrameAvailableRes.store(res);
352        return;
353    }
354    if (mBuffers.find(bufferItem.mGraphicBuffer->getId()) == mBuffers.end()) {
355        SP_LOGE("%s: Acquired buffer doesn't exist in attached buffer map",
356                __FUNCTION__);
357        mOnFrameAvailableRes.store(INVALID_OPERATION);
358        return;
359    }
360
361    SP_LOGV("acquired buffer %" PRId64 " from input at slot %d",
362            bufferItem.mGraphicBuffer->getId(), bufferItem.mSlot);
363
364    res = mConsumer->detachBuffer(bufferItem.mSlot);
365    if (res != NO_ERROR) {
366        SP_LOGE("%s: detaching buffer from input failed (%d)", __FUNCTION__, res);
367        mOnFrameAvailableRes.store(res);
368        return;
369    }
370
371    // Attach and queue the buffer to each of the outputs
372    BufferTracker& tracker = *(mBuffers[bufferItem.mGraphicBuffer->getId()]);
373
374    SP_LOGV("%s: BufferTracker for buffer %" PRId64 ", number of requests %zu",
375           __FUNCTION__, bufferItem.mGraphicBuffer->getId(), tracker.requestedSurfaces().size());
376    for (const auto id : tracker.requestedSurfaces()) {
377
378        LOG_ALWAYS_FATAL_IF(id >= mOutputs.size(),
379                "requested surface id exceeding max registered ids");
380
381        res = outputBufferLocked(mOutputs[id], bufferItem);
382        if (res != OK) {
383            SP_LOGE("%s: outputBufferLocked failed %d", __FUNCTION__, res);
384            mOnFrameAvailableRes.store(res);
385            // If we fail to send buffer to certain output, keep sending to
386            // other outputs.
387            continue;
388        }
389    }
390
391    mOnFrameAvailableRes.store(res);
392}
393
394void Camera3StreamSplitter::decrementBufRefCountLocked(uint64_t id,
395        const sp<IGraphicBufferProducer>& from) {
396    ATRACE_CALL();
397    size_t referenceCount = mBuffers[id]->decrementReferenceCountLocked();
398
399    removeSlotForOutputLocked(from, mBuffers[id]->getBuffer());
400    if (referenceCount > 0) {
401        return;
402    }
403
404    // We no longer need to track the buffer now that it is being returned to the
405    // input. Note that this should happen before we unlock the mutex and call
406    // releaseBuffer, to avoid the case where the same bufferId is acquired in
407    // attachBufferToOutputs resulting in a new BufferTracker with same bufferId
408    // overwrites the current one.
409    std::unique_ptr<BufferTracker> tracker_ptr = std::move(mBuffers[id]);
410    mBuffers.erase(id);
411
412    // Attach and release the buffer back to the input
413    int consumerSlot = BufferItem::INVALID_BUFFER_SLOT;
414    status_t res = mConsumer->attachBuffer(&consumerSlot, tracker_ptr->getBuffer());
415    if (res != NO_ERROR) {
416        SP_LOGE("%s: attaching buffer to input failed (%d)", __FUNCTION__, res);
417        return;
418    }
419
420    // Temporarily unlock mutex to avoid circular lock:
421    // 1. This function holds splitter lock, calls releaseBuffer which triggers
422    // onBufferReleased in Camera3OutputStream. onBufferReleased waits on the
423    // OutputStream lock
424    // 2. Camera3SharedOutputStream::getBufferLocked calls
425    // attachBufferToOutputs, which holds the stream lock, and waits for the
426    // splitter lock.
427    sp<IGraphicBufferConsumer> consumer(mConsumer);
428    mMutex.unlock();
429    if (consumer != nullptr) {
430        res = consumer->releaseBuffer(consumerSlot, /* frameNumber */ 0,
431                EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, tracker_ptr->getMergedFence());
432    } else {
433        SP_LOGE("%s: consumer has become null!", __FUNCTION__);
434    }
435    mMutex.lock();
436    // If the producer of this queue is disconnected, -22 error will occur
437    if (res != NO_ERROR) {
438        SP_LOGE("%s: releaseBuffer returns %d", __FUNCTION__, res);
439    }
440}
441
442void Camera3StreamSplitter::onBufferReleasedByOutput(
443        const sp<IGraphicBufferProducer>& from) {
444    ATRACE_CALL();
445    Mutex::Autolock lock(mMutex);
446
447    onBufferReleasedByOutputLocked(from);
448}
449
450void Camera3StreamSplitter::onBufferReleasedByOutputLocked(
451        const sp<IGraphicBufferProducer>& from) {
452    ATRACE_CALL();
453    sp<GraphicBuffer> buffer;
454    sp<Fence> fence;
455    status_t res = from->detachNextBuffer(&buffer, &fence);
456    if (res == NO_INIT) {
457        // If we just discovered that this output has been abandoned, note that,
458        // but we can't do anything else, since buffer is invalid
459        onAbandonedLocked();
460        return;
461    } else if (res == NO_MEMORY) {
462        SP_LOGV("%s: No free buffers", __FUNCTION__);
463        return;
464    } else if (res != OK) {
465        SP_LOGE("%s: detaching buffer from output failed (%d)", __FUNCTION__, res);
466        return;
467    }
468
469    BufferTracker& tracker = *(mBuffers[buffer->getId()]);
470    // Merge the release fence of the incoming buffer so that the fence we send
471    // back to the input includes all of the outputs' fences
472    if (fence != nullptr && fence->isValid()) {
473        tracker.mergeFence(fence);
474    }
475    SP_LOGV("detached buffer %" PRId64 " %p from output %p",
476            buffer->getId(), buffer.get(), from.get());
477
478    // Check to see if this is the last outstanding reference to this buffer
479    decrementBufRefCountLocked(buffer->getId(), from);
480}
481
482void Camera3StreamSplitter::onAbandonedLocked() {
483    // If this is called from binderDied callback, it means the app process
484    // holding the binder has died. CameraService will be notified of the binder
485    // death, and camera device will be closed, which in turn calls
486    // disconnect().
487    //
488    // If this is called from onBufferReleasedByOutput or onFrameAvailable, one
489    // consumer being abanoned shouldn't impact the other consumer. So we won't
490    // stop the buffer flow.
491    //
492    // In both cases, we don't need to do anything here.
493    SP_LOGV("One of my outputs has abandoned me");
494}
495
496int Camera3StreamSplitter::getSlotForOutputLocked(const sp<IGraphicBufferProducer>& gbp,
497        const sp<GraphicBuffer>& gb) {
498    auto& outputSlots = *mOutputSlots[gbp];
499
500    for (size_t i = 0; i < outputSlots.size(); i++) {
501        if (outputSlots[i] == gb) {
502            return (int)i;
503        }
504    }
505
506    SP_LOGE("%s: Cannot find slot for gb %p on output %p", __FUNCTION__, gb.get(),
507            gbp.get());
508    return BufferItem::INVALID_BUFFER_SLOT;
509}
510
511status_t Camera3StreamSplitter::removeSlotForOutputLocked(const sp<IGraphicBufferProducer>& gbp,
512        const sp<GraphicBuffer>& gb) {
513    auto& outputSlots = *mOutputSlots[gbp];
514
515    for (size_t i = 0; i < outputSlots.size(); i++) {
516        if (outputSlots[i] == gb) {
517           outputSlots[i].clear();
518           return NO_ERROR;
519        }
520    }
521
522    SP_LOGE("%s: Cannot find slot for gb %p on output %p", __FUNCTION__, gb.get(),
523            gbp.get());
524    return BAD_VALUE;
525}
526
527Camera3StreamSplitter::OutputListener::OutputListener(
528        wp<Camera3StreamSplitter> splitter,
529        wp<IGraphicBufferProducer> output)
530      : mSplitter(splitter), mOutput(output) {}
531
532void Camera3StreamSplitter::OutputListener::onBufferReleased() {
533    ATRACE_CALL();
534    sp<Camera3StreamSplitter> splitter = mSplitter.promote();
535    sp<IGraphicBufferProducer> output = mOutput.promote();
536    if (splitter != nullptr && output != nullptr) {
537        splitter->onBufferReleasedByOutput(output);
538    }
539}
540
541void Camera3StreamSplitter::OutputListener::binderDied(const wp<IBinder>& /* who */) {
542    sp<Camera3StreamSplitter> splitter = mSplitter.promote();
543    if (splitter != nullptr) {
544        Mutex::Autolock lock(splitter->mMutex);
545        splitter->onAbandonedLocked();
546    }
547}
548
549Camera3StreamSplitter::BufferTracker::BufferTracker(
550        const sp<GraphicBuffer>& buffer, const std::vector<size_t>& requestedSurfaces)
551      : mBuffer(buffer), mMergedFence(Fence::NO_FENCE), mRequestedSurfaces(requestedSurfaces),
552        mReferenceCount(requestedSurfaces.size()) {}
553
554void Camera3StreamSplitter::BufferTracker::mergeFence(const sp<Fence>& with) {
555    mMergedFence = Fence::merge(String8("Camera3StreamSplitter"), mMergedFence, with);
556}
557
558size_t Camera3StreamSplitter::BufferTracker::decrementReferenceCountLocked() {
559    if (mReferenceCount > 0)
560        --mReferenceCount;
561    return mReferenceCount;
562}
563
564} // namespace android
565