1/* 2 * Copyright 2014,2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include <inttypes.h> 18 19#define LOG_TAG "Camera3StreamSplitter" 20#define ATRACE_TAG ATRACE_TAG_CAMERA 21//#define LOG_NDEBUG 0 22 23#include <gui/BufferItem.h> 24#include <gui/IGraphicBufferConsumer.h> 25#include <gui/IGraphicBufferProducer.h> 26#include <gui/BufferQueue.h> 27#include <gui/Surface.h> 28 29#include <ui/GraphicBuffer.h> 30 31#include <binder/ProcessState.h> 32 33#include <utils/Trace.h> 34 35#include <cutils/atomic.h> 36 37#include "Camera3StreamSplitter.h" 38 39namespace android { 40 41status_t Camera3StreamSplitter::connect(const std::vector<sp<Surface> >& surfaces, 42 uint32_t consumerUsage, size_t halMaxBuffers, sp<Surface>* consumer) { 43 ATRACE_CALL(); 44 if (consumer == nullptr) { 45 SP_LOGE("%s: consumer pointer is NULL", __FUNCTION__); 46 return BAD_VALUE; 47 } 48 49 Mutex::Autolock lock(mMutex); 50 status_t res = OK; 51 52 if (mOutputs.size() > 0 || mConsumer != nullptr) { 53 SP_LOGE("%s: already connected", __FUNCTION__); 54 return BAD_VALUE; 55 } 56 if (mBuffers.size() > 0) { 57 SP_LOGE("%s: still has %zu pending buffers", __FUNCTION__, mBuffers.size()); 58 return BAD_VALUE; 59 } 60 61 mMaxHalBuffers = halMaxBuffers; 62 mConsumerName = getUniqueConsumerName(); 63 // Add output surfaces. This has to be before creating internal buffer queue 64 // in order to get max consumer side buffers. 65 for (size_t i = 0; i < surfaces.size(); i++) { 66 if (surfaces[i] == nullptr) { 67 SP_LOGE("%s: Fatal: surface is NULL", __FUNCTION__); 68 return BAD_VALUE; 69 } 70 res = addOutputLocked(surfaces[i]); 71 if (res != OK) { 72 SP_LOGE("%s: Failed to add output surface: %s(%d)", 73 __FUNCTION__, strerror(-res), res); 74 return res; 75 } 76 } 77 78 // Create BufferQueue for input 79 BufferQueue::createBufferQueue(&mProducer, &mConsumer); 80 81 // Allocate 1 extra buffer to handle the case where all buffers are detached 82 // from input, and attached to the outputs. In this case, the input queue's 83 // dequeueBuffer can still allocate 1 extra buffer before being blocked by 84 // the output's attachBuffer(). 85 mBufferItemConsumer = new BufferItemConsumer(mConsumer, consumerUsage, 86 mMaxConsumerBuffers+1); 87 if (mBufferItemConsumer == nullptr) { 88 return NO_MEMORY; 89 } 90 mConsumer->setConsumerName(mConsumerName); 91 92 *consumer = new Surface(mProducer); 93 if (*consumer == nullptr) { 94 return NO_MEMORY; 95 } 96 97 res = mConsumer->consumerConnect(this, /* controlledByApp */ false); 98 99 SP_LOGV("%s: connected", __FUNCTION__); 100 return res; 101} 102 103status_t Camera3StreamSplitter::getOnFrameAvailableResult() { 104 ATRACE_CALL(); 105 return mOnFrameAvailableRes.load(); 106} 107 108void Camera3StreamSplitter::disconnect() { 109 ATRACE_CALL(); 110 Mutex::Autolock lock(mMutex); 111 112 for (auto& notifier : mNotifiers) { 113 sp<IGraphicBufferProducer> producer = notifier.first; 114 sp<OutputListener> listener = notifier.second; 115 IInterface::asBinder(producer)->unlinkToDeath(listener); 116 } 117 mNotifiers.clear(); 118 119 for (auto& output : mOutputs) { 120 output->disconnect(NATIVE_WINDOW_API_CAMERA); 121 } 122 mOutputs.clear(); 123 mOutputSlots.clear(); 124 125 mConsumer->consumerDisconnect(); 126 127 if (mBuffers.size() > 0) { 128 SP_LOGW("%zu buffers still being tracked", mBuffers.size()); 129 mBuffers.clear(); 130 } 131 132 mMaxHalBuffers = 0; 133 mMaxConsumerBuffers = 0; 134 SP_LOGV("%s: Disconnected", __FUNCTION__); 135} 136 137 138Camera3StreamSplitter::~Camera3StreamSplitter() { 139 disconnect(); 140} 141 142status_t Camera3StreamSplitter::addOutput(const sp<Surface>& outputQueue) { 143 ATRACE_CALL(); 144 Mutex::Autolock lock(mMutex); 145 status_t res = addOutputLocked(outputQueue); 146 147 if (res != OK) { 148 SP_LOGE("%s: addOutputLocked failed %d", __FUNCTION__, res); 149 return res; 150 } 151 152 res = mConsumer->setMaxAcquiredBufferCount(mMaxConsumerBuffers+1); 153 154 return res; 155} 156 157status_t Camera3StreamSplitter::addOutputLocked(const sp<Surface>& outputQueue) { 158 ATRACE_CALL(); 159 if (outputQueue == nullptr) { 160 SP_LOGE("addOutput: outputQueue must not be NULL"); 161 return BAD_VALUE; 162 } 163 164 sp<IGraphicBufferProducer> gbp = outputQueue->getIGraphicBufferProducer(); 165 // Connect to the buffer producer 166 sp<OutputListener> listener(new OutputListener(this, gbp)); 167 IInterface::asBinder(gbp)->linkToDeath(listener); 168 status_t res = outputQueue->connect(NATIVE_WINDOW_API_CAMERA, listener); 169 if (res != NO_ERROR) { 170 SP_LOGE("addOutput: failed to connect (%d)", res); 171 return res; 172 } 173 174 // Query consumer side buffer count, and update overall buffer count 175 int maxConsumerBuffers = 0; 176 res = static_cast<ANativeWindow*>(outputQueue.get())->query( 177 outputQueue.get(), 178 NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &maxConsumerBuffers); 179 if (res != OK) { 180 SP_LOGE("%s: Unable to query consumer undequeued buffer count" 181 " for surface", __FUNCTION__); 182 return res; 183 } 184 185 SP_LOGV("%s: Consumer wants %d buffers, Producer wants %zu", __FUNCTION__, 186 maxConsumerBuffers, mMaxHalBuffers); 187 size_t totalBufferCount = maxConsumerBuffers + mMaxHalBuffers; 188 res = native_window_set_buffer_count(outputQueue.get(), 189 totalBufferCount); 190 if (res != OK) { 191 SP_LOGE("%s: Unable to set buffer count for surface %p", 192 __FUNCTION__, outputQueue.get()); 193 return res; 194 } 195 196 // Set dequeueBuffer/attachBuffer timeout if the consumer is not hw composer or hw texture. 197 // We need skip these cases as timeout will disable the non-blocking (async) mode. 198 int32_t usage = 0; 199 static_cast<ANativeWindow*>(outputQueue.get())->query( 200 outputQueue.get(), 201 NATIVE_WINDOW_CONSUMER_USAGE_BITS, &usage); 202 if (!(usage & (GRALLOC_USAGE_HW_COMPOSER | GRALLOC_USAGE_HW_TEXTURE))) { 203 outputQueue->setDequeueTimeout(kDequeueBufferTimeout); 204 } 205 206 res = gbp->allowAllocation(false); 207 if (res != OK) { 208 SP_LOGE("%s: Failed to turn off allocation for outputQueue", __FUNCTION__); 209 return res; 210 } 211 212 // Add new entry into mOutputs 213 mOutputs.push_back(gbp); 214 mNotifiers[gbp] = listener; 215 mOutputSlots[gbp] = std::make_unique<OutputSlots>(totalBufferCount); 216 217 mMaxConsumerBuffers += maxConsumerBuffers; 218 return NO_ERROR; 219} 220 221status_t Camera3StreamSplitter::outputBufferLocked(const sp<IGraphicBufferProducer>& output, 222 const BufferItem& bufferItem) { 223 ATRACE_CALL(); 224 status_t res; 225 IGraphicBufferProducer::QueueBufferInput queueInput( 226 bufferItem.mTimestamp, bufferItem.mIsAutoTimestamp, 227 bufferItem.mDataSpace, bufferItem.mCrop, 228 static_cast<int32_t>(bufferItem.mScalingMode), 229 bufferItem.mTransform, bufferItem.mFence); 230 231 IGraphicBufferProducer::QueueBufferOutput queueOutput; 232 233 uint64_t bufferId = bufferItem.mGraphicBuffer->getId(); 234 const BufferTracker& tracker = *(mBuffers[bufferId]); 235 int slot = getSlotForOutputLocked(output, tracker.getBuffer()); 236 237 // In case the output BufferQueue has its own lock, if we hold splitter lock while calling 238 // queueBuffer (which will try to acquire the output lock), the output could be holding its 239 // own lock calling releaseBuffer (which will try to acquire the splitter lock), running into 240 // circular lock situation. 241 mMutex.unlock(); 242 res = output->queueBuffer(slot, queueInput, &queueOutput); 243 mMutex.lock(); 244 245 SP_LOGV("%s: Queuing buffer to buffer queue %p slot %d returns %d", 246 __FUNCTION__, output.get(), slot, res); 247 if (res != OK) { 248 if (res != NO_INIT && res != DEAD_OBJECT) { 249 SP_LOGE("Queuing buffer to output failed (%d)", res); 250 } 251 // If we just discovered that this output has been abandoned, note 252 // that, increment the release count so that we still release this 253 // buffer eventually, and move on to the next output 254 onAbandonedLocked(); 255 decrementBufRefCountLocked(bufferItem.mGraphicBuffer->getId(), output); 256 return res; 257 } 258 259 // If the queued buffer replaces a pending buffer in the async 260 // queue, no onBufferReleased is called by the buffer queue. 261 // Proactively trigger the callback to avoid buffer loss. 262 if (queueOutput.bufferReplaced) { 263 onBufferReleasedByOutputLocked(output); 264 } 265 266 return res; 267} 268 269String8 Camera3StreamSplitter::getUniqueConsumerName() { 270 static volatile int32_t counter = 0; 271 return String8::format("Camera3StreamSplitter-%d", android_atomic_inc(&counter)); 272} 273 274status_t Camera3StreamSplitter::notifyBufferReleased(const sp<GraphicBuffer>& buffer) { 275 ATRACE_CALL(); 276 status_t res = OK; 277 278 Mutex::Autolock lock(mMutex); 279 280 uint64_t bufferId = buffer->getId(); 281 std::unique_ptr<BufferTracker> tracker_ptr = std::move(mBuffers[bufferId]); 282 mBuffers.erase(bufferId); 283 284 for (const auto surface : tracker_ptr->requestedSurfaces()) { 285 sp<IGraphicBufferProducer>& gbp = mOutputs[surface]; 286 OutputSlots& outputSlots = *(mOutputSlots[gbp]); 287 int slot = getSlotForOutputLocked(gbp, buffer); 288 if (slot != BufferItem::INVALID_BUFFER_SLOT) { 289 gbp->detachBuffer(slot); 290 outputSlots[slot].clear(); 291 } 292 } 293 294 return res; 295} 296 297status_t Camera3StreamSplitter::attachBufferToOutputs(ANativeWindowBuffer* anb, 298 const std::vector<size_t>& surface_ids) { 299 ATRACE_CALL(); 300 status_t res = OK; 301 302 Mutex::Autolock lock(mMutex); 303 304 sp<GraphicBuffer> gb(static_cast<GraphicBuffer*>(anb)); 305 uint64_t bufferId = gb->getId(); 306 307 // Initialize buffer tracker for this input buffer 308 auto tracker = std::make_unique<BufferTracker>(gb, surface_ids); 309 310 for (auto& surface_id : surface_ids) { 311 sp<IGraphicBufferProducer>& gbp = mOutputs[surface_id]; 312 int slot = BufferItem::INVALID_BUFFER_SLOT; 313 //Temporarly Unlock the mutex when trying to attachBuffer to the output 314 //queue, because attachBuffer could block in case of a slow consumer. If 315 //we block while holding the lock, onFrameAvailable and onBufferReleased 316 //will block as well because they need to acquire the same lock. 317 mMutex.unlock(); 318 res = gbp->attachBuffer(&slot, gb); 319 mMutex.lock(); 320 if (res != OK) { 321 SP_LOGE("%s: Cannot acquireBuffer from GraphicBufferProducer %p: %s (%d)", 322 __FUNCTION__, gbp.get(), strerror(-res), res); 323 return res; 324 } 325 auto& outputSlots = *mOutputSlots[gbp]; 326 if (outputSlots[slot] != nullptr) { 327 // If the buffer is attached to a slot which already contains a buffer, 328 // the previous buffer will be removed from the output queue. Decrement 329 // the reference count accordingly. 330 decrementBufRefCountLocked(outputSlots[slot]->getId(), gbp); 331 } 332 SP_LOGV("%s: Attached buffer %p to slot %d on output %p.",__FUNCTION__, gb.get(), 333 slot, gbp.get()); 334 outputSlots[slot] = gb; 335 } 336 337 mBuffers[bufferId] = std::move(tracker); 338 339 return res; 340} 341 342void Camera3StreamSplitter::onFrameAvailable(const BufferItem& /*item*/) { 343 ATRACE_CALL(); 344 Mutex::Autolock lock(mMutex); 345 346 // Acquire and detach the buffer from the input 347 BufferItem bufferItem; 348 status_t res = mConsumer->acquireBuffer(&bufferItem, /* presentWhen */ 0); 349 if (res != NO_ERROR) { 350 SP_LOGE("%s: Acquiring buffer from input failed (%d)", __FUNCTION__, res); 351 mOnFrameAvailableRes.store(res); 352 return; 353 } 354 if (mBuffers.find(bufferItem.mGraphicBuffer->getId()) == mBuffers.end()) { 355 SP_LOGE("%s: Acquired buffer doesn't exist in attached buffer map", 356 __FUNCTION__); 357 mOnFrameAvailableRes.store(INVALID_OPERATION); 358 return; 359 } 360 361 SP_LOGV("acquired buffer %" PRId64 " from input at slot %d", 362 bufferItem.mGraphicBuffer->getId(), bufferItem.mSlot); 363 364 res = mConsumer->detachBuffer(bufferItem.mSlot); 365 if (res != NO_ERROR) { 366 SP_LOGE("%s: detaching buffer from input failed (%d)", __FUNCTION__, res); 367 mOnFrameAvailableRes.store(res); 368 return; 369 } 370 371 // Attach and queue the buffer to each of the outputs 372 BufferTracker& tracker = *(mBuffers[bufferItem.mGraphicBuffer->getId()]); 373 374 SP_LOGV("%s: BufferTracker for buffer %" PRId64 ", number of requests %zu", 375 __FUNCTION__, bufferItem.mGraphicBuffer->getId(), tracker.requestedSurfaces().size()); 376 for (const auto id : tracker.requestedSurfaces()) { 377 378 LOG_ALWAYS_FATAL_IF(id >= mOutputs.size(), 379 "requested surface id exceeding max registered ids"); 380 381 res = outputBufferLocked(mOutputs[id], bufferItem); 382 if (res != OK) { 383 SP_LOGE("%s: outputBufferLocked failed %d", __FUNCTION__, res); 384 mOnFrameAvailableRes.store(res); 385 // If we fail to send buffer to certain output, keep sending to 386 // other outputs. 387 continue; 388 } 389 } 390 391 mOnFrameAvailableRes.store(res); 392} 393 394void Camera3StreamSplitter::decrementBufRefCountLocked(uint64_t id, 395 const sp<IGraphicBufferProducer>& from) { 396 ATRACE_CALL(); 397 size_t referenceCount = mBuffers[id]->decrementReferenceCountLocked(); 398 399 removeSlotForOutputLocked(from, mBuffers[id]->getBuffer()); 400 if (referenceCount > 0) { 401 return; 402 } 403 404 // We no longer need to track the buffer now that it is being returned to the 405 // input. Note that this should happen before we unlock the mutex and call 406 // releaseBuffer, to avoid the case where the same bufferId is acquired in 407 // attachBufferToOutputs resulting in a new BufferTracker with same bufferId 408 // overwrites the current one. 409 std::unique_ptr<BufferTracker> tracker_ptr = std::move(mBuffers[id]); 410 mBuffers.erase(id); 411 412 // Attach and release the buffer back to the input 413 int consumerSlot = BufferItem::INVALID_BUFFER_SLOT; 414 status_t res = mConsumer->attachBuffer(&consumerSlot, tracker_ptr->getBuffer()); 415 if (res != NO_ERROR) { 416 SP_LOGE("%s: attaching buffer to input failed (%d)", __FUNCTION__, res); 417 return; 418 } 419 420 // Temporarily unlock mutex to avoid circular lock: 421 // 1. This function holds splitter lock, calls releaseBuffer which triggers 422 // onBufferReleased in Camera3OutputStream. onBufferReleased waits on the 423 // OutputStream lock 424 // 2. Camera3SharedOutputStream::getBufferLocked calls 425 // attachBufferToOutputs, which holds the stream lock, and waits for the 426 // splitter lock. 427 sp<IGraphicBufferConsumer> consumer(mConsumer); 428 mMutex.unlock(); 429 if (consumer != nullptr) { 430 res = consumer->releaseBuffer(consumerSlot, /* frameNumber */ 0, 431 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, tracker_ptr->getMergedFence()); 432 } else { 433 SP_LOGE("%s: consumer has become null!", __FUNCTION__); 434 } 435 mMutex.lock(); 436 // If the producer of this queue is disconnected, -22 error will occur 437 if (res != NO_ERROR) { 438 SP_LOGE("%s: releaseBuffer returns %d", __FUNCTION__, res); 439 } 440} 441 442void Camera3StreamSplitter::onBufferReleasedByOutput( 443 const sp<IGraphicBufferProducer>& from) { 444 ATRACE_CALL(); 445 Mutex::Autolock lock(mMutex); 446 447 onBufferReleasedByOutputLocked(from); 448} 449 450void Camera3StreamSplitter::onBufferReleasedByOutputLocked( 451 const sp<IGraphicBufferProducer>& from) { 452 ATRACE_CALL(); 453 sp<GraphicBuffer> buffer; 454 sp<Fence> fence; 455 status_t res = from->detachNextBuffer(&buffer, &fence); 456 if (res == NO_INIT) { 457 // If we just discovered that this output has been abandoned, note that, 458 // but we can't do anything else, since buffer is invalid 459 onAbandonedLocked(); 460 return; 461 } else if (res == NO_MEMORY) { 462 SP_LOGV("%s: No free buffers", __FUNCTION__); 463 return; 464 } else if (res != OK) { 465 SP_LOGE("%s: detaching buffer from output failed (%d)", __FUNCTION__, res); 466 return; 467 } 468 469 BufferTracker& tracker = *(mBuffers[buffer->getId()]); 470 // Merge the release fence of the incoming buffer so that the fence we send 471 // back to the input includes all of the outputs' fences 472 if (fence != nullptr && fence->isValid()) { 473 tracker.mergeFence(fence); 474 } 475 SP_LOGV("detached buffer %" PRId64 " %p from output %p", 476 buffer->getId(), buffer.get(), from.get()); 477 478 // Check to see if this is the last outstanding reference to this buffer 479 decrementBufRefCountLocked(buffer->getId(), from); 480} 481 482void Camera3StreamSplitter::onAbandonedLocked() { 483 // If this is called from binderDied callback, it means the app process 484 // holding the binder has died. CameraService will be notified of the binder 485 // death, and camera device will be closed, which in turn calls 486 // disconnect(). 487 // 488 // If this is called from onBufferReleasedByOutput or onFrameAvailable, one 489 // consumer being abanoned shouldn't impact the other consumer. So we won't 490 // stop the buffer flow. 491 // 492 // In both cases, we don't need to do anything here. 493 SP_LOGV("One of my outputs has abandoned me"); 494} 495 496int Camera3StreamSplitter::getSlotForOutputLocked(const sp<IGraphicBufferProducer>& gbp, 497 const sp<GraphicBuffer>& gb) { 498 auto& outputSlots = *mOutputSlots[gbp]; 499 500 for (size_t i = 0; i < outputSlots.size(); i++) { 501 if (outputSlots[i] == gb) { 502 return (int)i; 503 } 504 } 505 506 SP_LOGE("%s: Cannot find slot for gb %p on output %p", __FUNCTION__, gb.get(), 507 gbp.get()); 508 return BufferItem::INVALID_BUFFER_SLOT; 509} 510 511status_t Camera3StreamSplitter::removeSlotForOutputLocked(const sp<IGraphicBufferProducer>& gbp, 512 const sp<GraphicBuffer>& gb) { 513 auto& outputSlots = *mOutputSlots[gbp]; 514 515 for (size_t i = 0; i < outputSlots.size(); i++) { 516 if (outputSlots[i] == gb) { 517 outputSlots[i].clear(); 518 return NO_ERROR; 519 } 520 } 521 522 SP_LOGE("%s: Cannot find slot for gb %p on output %p", __FUNCTION__, gb.get(), 523 gbp.get()); 524 return BAD_VALUE; 525} 526 527Camera3StreamSplitter::OutputListener::OutputListener( 528 wp<Camera3StreamSplitter> splitter, 529 wp<IGraphicBufferProducer> output) 530 : mSplitter(splitter), mOutput(output) {} 531 532void Camera3StreamSplitter::OutputListener::onBufferReleased() { 533 ATRACE_CALL(); 534 sp<Camera3StreamSplitter> splitter = mSplitter.promote(); 535 sp<IGraphicBufferProducer> output = mOutput.promote(); 536 if (splitter != nullptr && output != nullptr) { 537 splitter->onBufferReleasedByOutput(output); 538 } 539} 540 541void Camera3StreamSplitter::OutputListener::binderDied(const wp<IBinder>& /* who */) { 542 sp<Camera3StreamSplitter> splitter = mSplitter.promote(); 543 if (splitter != nullptr) { 544 Mutex::Autolock lock(splitter->mMutex); 545 splitter->onAbandonedLocked(); 546 } 547} 548 549Camera3StreamSplitter::BufferTracker::BufferTracker( 550 const sp<GraphicBuffer>& buffer, const std::vector<size_t>& requestedSurfaces) 551 : mBuffer(buffer), mMergedFence(Fence::NO_FENCE), mRequestedSurfaces(requestedSurfaces), 552 mReferenceCount(requestedSurfaces.size()) {} 553 554void Camera3StreamSplitter::BufferTracker::mergeFence(const sp<Fence>& with) { 555 mMergedFence = Fence::merge(String8("Camera3StreamSplitter"), mMergedFence, with); 556} 557 558size_t Camera3StreamSplitter::BufferTracker::decrementReferenceCountLocked() { 559 if (mReferenceCount > 0) 560 --mReferenceCount; 561 return mReferenceCount; 562} 563 564} // namespace android 565