Camera3StreamSplitter.cpp revision a141c5f3cc2214a96f250999edacc3bd4d454601
1/* 2 * Copyright 2014,2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include <inttypes.h> 18 19#define LOG_TAG "Camera3StreamSplitter" 20#define ATRACE_TAG ATRACE_TAG_CAMERA 21//#define LOG_NDEBUG 0 22 23#include <gui/BufferItem.h> 24#include <gui/IGraphicBufferConsumer.h> 25#include <gui/IGraphicBufferProducer.h> 26#include <gui/BufferQueue.h> 27#include <gui/Surface.h> 28 29#include <ui/GraphicBuffer.h> 30 31#include <binder/ProcessState.h> 32 33#include <utils/Trace.h> 34 35#include "Camera3StreamSplitter.h" 36 37namespace android { 38 39status_t Camera3StreamSplitter::connect(const std::vector<sp<Surface> >& surfaces, 40 uint32_t consumerUsage, size_t hal_max_buffers, 41 sp<Surface>& consumer) { 42 if (consumer != nullptr) { 43 ALOGE("%s: output Surface is not NULL", __FUNCTION__); 44 return BAD_VALUE; 45 } 46 47 Mutex::Autolock lock(mMutex); 48 status_t res = OK; 49 50 if (mOutputs.size() > 0 || mConsumer != nullptr) { 51 ALOGE("%s: StreamSplitter already connected", __FUNCTION__); 52 return BAD_VALUE; 53 } 54 55 // Add output surfaces. This has to be before creating internal buffer queue 56 // in order to get max consumer side buffers. 57 for (size_t i = 0; i < surfaces.size(); i++) { 58 if (surfaces[i] == nullptr) { 59 ALOGE("%s: Fatal: surface is NULL", __FUNCTION__); 60 return BAD_VALUE; 61 } 62 res = addOutputLocked(surfaces[i], hal_max_buffers, OutputType::NonDeferred); 63 if (res != OK) { 64 ALOGE("%s: Failed to add output surface: %s(%d)", 65 __FUNCTION__, strerror(-res), res); 66 return res; 67 } 68 } 69 70 // Create buffer queue for input 71 BufferQueue::createBufferQueue(&mProducer, &mConsumer); 72 73 mBufferItemConsumer = new BufferItemConsumer(mConsumer, consumerUsage, 74 mMaxConsumerBuffers); 75 if (mBufferItemConsumer == nullptr) { 76 return NO_MEMORY; 77 } 78 mConsumer->setConsumerName(getUniqueConsumerName()); 79 80 mSurface = new Surface(mProducer); 81 if (mSurface == nullptr) { 82 return NO_MEMORY; 83 } 84 consumer = mSurface; 85 86 res = mConsumer->consumerConnect(this, /* controlledByApp */ false); 87 88 return res; 89} 90 91void Camera3StreamSplitter::disconnect() { 92 Mutex::Autolock lock(mMutex); 93 94 for (auto& output : mOutputs) { 95 output->disconnect(NATIVE_WINDOW_API_CAMERA); 96 } 97 mOutputs.clear(); 98 99 if (mConsumer != nullptr) { 100 mConsumer->consumerDisconnect(); 101 mConsumer.clear(); 102 } 103 104 if (mBuffers.size() > 0) { 105 ALOGI("%zu buffers still being tracked", mBuffers.size()); 106 } 107} 108 109Camera3StreamSplitter::~Camera3StreamSplitter() { 110 disconnect(); 111} 112 113status_t Camera3StreamSplitter::addOutput( 114 const sp<Surface>& outputQueue, size_t hal_max_buffers) { 115 Mutex::Autolock lock(mMutex); 116 return addOutputLocked(outputQueue, hal_max_buffers, OutputType::Deferred); 117} 118 119status_t Camera3StreamSplitter::addOutputLocked( 120 const sp<Surface>& outputQueue, size_t hal_max_buffers, 121 OutputType outputType) { 122 if (outputQueue == nullptr) { 123 ALOGE("addOutput: outputQueue must not be NULL"); 124 return BAD_VALUE; 125 } 126 if (hal_max_buffers < 1) { 127 ALOGE("%s: Camera HAL requested max_buffer count: %zu, requires at least 1", 128 __FUNCTION__, hal_max_buffers); 129 return BAD_VALUE; 130 } 131 132 sp<IGraphicBufferProducer> gbp = outputQueue->getIGraphicBufferProducer(); 133 // Connect to the buffer producer 134 IGraphicBufferProducer::QueueBufferOutput queueBufferOutput; 135 sp<OutputListener> listener(new OutputListener(this, gbp)); 136 IInterface::asBinder(gbp)->linkToDeath(listener); 137 status_t status = gbp->connect(listener, NATIVE_WINDOW_API_CAMERA, 138 /* producerControlledByApp */ true, &queueBufferOutput); 139 if (status != NO_ERROR) { 140 ALOGE("addOutput: failed to connect (%d)", status); 141 return status; 142 } 143 144 // Query consumer side buffer count, and update overall buffer count 145 int maxConsumerBuffers = 0; 146 status = static_cast<ANativeWindow*>(outputQueue.get())->query( 147 outputQueue.get(), 148 NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &maxConsumerBuffers); 149 if (status != OK) { 150 ALOGE("%s: Unable to query consumer undequeued buffer count" 151 " for surface", __FUNCTION__); 152 return status; 153 } 154 155 if (maxConsumerBuffers > mMaxConsumerBuffers) { 156 if (outputType == OutputType::Deferred) { 157 ALOGE("%s: Fatal: Deferred surface has higher consumer buffer count" 158 " %d than what's already configured %d", __FUNCTION__, 159 maxConsumerBuffers, mMaxConsumerBuffers); 160 return BAD_VALUE; 161 } 162 mMaxConsumerBuffers = maxConsumerBuffers; 163 } 164 165 ALOGV("%s: Consumer wants %d buffers, HAL wants %zu", __FUNCTION__, 166 maxConsumerBuffers, hal_max_buffers); 167 size_t totalBufferCount = maxConsumerBuffers + hal_max_buffers; 168 status = native_window_set_buffer_count(outputQueue.get(), 169 totalBufferCount); 170 if (status != OK) { 171 ALOGE("%s: Unable to set buffer count for surface %p", 172 __FUNCTION__, outputQueue.get()); 173 return status; 174 } 175 176 // Set dequeueBuffer/attachBuffer timeout if the consumer is not hw composer or hw texture. 177 // We need skip these cases as timeout will disable the non-blocking (async) mode. 178 int32_t usage = 0; 179 static_cast<ANativeWindow*>(outputQueue.get())->query( 180 outputQueue.get(), 181 NATIVE_WINDOW_CONSUMER_USAGE_BITS, &usage); 182 if (!(usage & (GRALLOC_USAGE_HW_COMPOSER | GRALLOC_USAGE_HW_TEXTURE))) { 183 outputQueue->setDequeueTimeout(kDequeueBufferTimeout); 184 } 185 186 status = gbp->allowAllocation(false); 187 if (status != OK) { 188 ALOGE("%s: Failed to turn off allocation for outputQueue", __FUNCTION__); 189 return status; 190 } 191 192 // Add new entry into mOutputs 193 mOutputs.push_back(gbp); 194 return NO_ERROR; 195} 196 197String8 Camera3StreamSplitter::getUniqueConsumerName() { 198 static volatile int32_t counter = 0; 199 return String8::format("Camera3StreamSplitter-%d", android_atomic_inc(&counter)); 200} 201 202status_t Camera3StreamSplitter::notifyRequestedSurfaces( 203 const std::vector<size_t>& surfaces) { 204 ATRACE_CALL(); 205 Mutex::Autolock lock(mMutex); 206 207 mRequestedSurfaces.push_back(surfaces); 208 return OK; 209} 210 211 212void Camera3StreamSplitter::onFrameAvailable(const BufferItem& /* item */) { 213 ATRACE_CALL(); 214 Mutex::Autolock lock(mMutex); 215 216 // The current policy is that if any one consumer is consuming buffers too 217 // slowly, the splitter will stall the rest of the outputs by not acquiring 218 // any more buffers from the input. This will cause back pressure on the 219 // input queue, slowing down its producer. 220 221 // If there are too many outstanding buffers, we block until a buffer is 222 // released back to the input in onBufferReleased 223 while (mOutstandingBuffers >= mMaxConsumerBuffers) { 224 mReleaseCondition.wait(mMutex); 225 226 // If the splitter is abandoned while we are waiting, the release 227 // condition variable will be broadcast, and we should just return 228 // without attempting to do anything more (since the input queue will 229 // also be abandoned). 230 if (mIsAbandoned) { 231 return; 232 } 233 } 234 // If the splitter is abandoned without reaching mMaxConsumerBuffers, just 235 // return without attempting to do anything more. 236 if (mIsAbandoned) { 237 return; 238 } 239 240 ++mOutstandingBuffers; 241 242 // Acquire and detach the buffer from the input 243 BufferItem bufferItem; 244 status_t status = mConsumer->acquireBuffer(&bufferItem, /* presentWhen */ 0); 245 LOG_ALWAYS_FATAL_IF(status != NO_ERROR, 246 "acquiring buffer from input failed (%d)", status); 247 248 ALOGV("acquired buffer %#" PRIx64 " from input", 249 bufferItem.mGraphicBuffer->getId()); 250 251 status = mConsumer->detachBuffer(bufferItem.mSlot); 252 LOG_ALWAYS_FATAL_IF(status != NO_ERROR, 253 "detaching buffer from input failed (%d)", status); 254 255 IGraphicBufferProducer::QueueBufferInput queueInput( 256 bufferItem.mTimestamp, bufferItem.mIsAutoTimestamp, 257 bufferItem.mDataSpace, bufferItem.mCrop, 258 static_cast<int32_t>(bufferItem.mScalingMode), 259 bufferItem.mTransform, bufferItem.mFence); 260 261 // Attach and queue the buffer to each of the outputs 262 std::vector<std::vector<size_t> >::iterator surfaces = mRequestedSurfaces.begin(); 263 if (surfaces != mRequestedSurfaces.end()) { 264 265 LOG_ALWAYS_FATAL_IF(surfaces->size() == 0, 266 "requested surface ids shouldn't be empty"); 267 268 // Initialize our reference count for this buffer 269 mBuffers[bufferItem.mGraphicBuffer->getId()] = 270 std::unique_ptr<BufferTracker>( 271 new BufferTracker(bufferItem.mGraphicBuffer, surfaces->size())); 272 273 for (auto id : *surfaces) { 274 275 LOG_ALWAYS_FATAL_IF(id >= mOutputs.size(), 276 "requested surface id exceeding max registered ids"); 277 278 int slot = BufferItem::INVALID_BUFFER_SLOT; 279 status = mOutputs[id]->attachBuffer(&slot, bufferItem.mGraphicBuffer); 280 if (status == NO_INIT) { 281 // If we just discovered that this output has been abandoned, note 282 // that, decrement the reference count so that we still release this 283 // buffer eventually, and move on to the next output 284 onAbandonedLocked(); 285 mBuffers[bufferItem.mGraphicBuffer->getId()]-> 286 decrementReferenceCountLocked(); 287 continue; 288 } else if (status == WOULD_BLOCK) { 289 // If the output is async, attachBuffer may return WOULD_BLOCK 290 // indicating number of dequeued buffers has reached limit. In 291 // this case, simply decrement the reference count, and move on 292 // to the next output. 293 // TODO: Do we need to report BUFFER_ERROR for this result? 294 mBuffers[bufferItem.mGraphicBuffer->getId()]-> 295 decrementReferenceCountLocked(); 296 continue; 297 } else if (status == TIMED_OUT) { 298 // If attachBuffer times out due to the value set by 299 // setDequeueTimeout, simply decrement the reference count, and 300 // move on to the next output. 301 // TODO: Do we need to report BUFFER_ERROR for this result? 302 mBuffers[bufferItem.mGraphicBuffer->getId()]-> 303 decrementReferenceCountLocked(); 304 continue; 305 } else { 306 LOG_ALWAYS_FATAL_IF(status != NO_ERROR, 307 "attaching buffer to output failed (%d)", status); 308 } 309 310 IGraphicBufferProducer::QueueBufferOutput queueOutput; 311 status = mOutputs[id]->queueBuffer(slot, queueInput, &queueOutput); 312 if (status == NO_INIT) { 313 // If we just discovered that this output has been abandoned, note 314 // that, increment the release count so that we still release this 315 // buffer eventually, and move on to the next output 316 onAbandonedLocked(); 317 mBuffers[bufferItem.mGraphicBuffer->getId()]-> 318 decrementReferenceCountLocked(); 319 continue; 320 } else { 321 LOG_ALWAYS_FATAL_IF(status != NO_ERROR, 322 "queueing buffer to output failed (%d)", status); 323 } 324 325 // If the queued buffer replaces a pending buffer in the async 326 // queue, no onBufferReleased is called by the buffer queue. 327 // Proactively trigger the callback to avoid buffer loss. 328 if (queueOutput.bufferReplaced) { 329 onBufferReleasedByOutputLocked(mOutputs[id]); 330 } 331 332 ALOGV("queued buffer %#" PRIx64 " to output %p", 333 bufferItem.mGraphicBuffer->getId(), mOutputs[id].get()); 334 } 335 336 mRequestedSurfaces.erase(surfaces); 337 } 338} 339 340void Camera3StreamSplitter::onBufferReleasedByOutput( 341 const sp<IGraphicBufferProducer>& from) { 342 ATRACE_CALL(); 343 Mutex::Autolock lock(mMutex); 344 345 onBufferReleasedByOutputLocked(from); 346} 347 348void Camera3StreamSplitter::onBufferReleasedByOutputLocked( 349 const sp<IGraphicBufferProducer>& from) { 350 351 sp<GraphicBuffer> buffer; 352 sp<Fence> fence; 353 status_t status = from->detachNextBuffer(&buffer, &fence); 354 if (status == NO_INIT) { 355 // If we just discovered that this output has been abandoned, note that, 356 // but we can't do anything else, since buffer is invalid 357 onAbandonedLocked(); 358 return; 359 } else { 360 LOG_ALWAYS_FATAL_IF(status != NO_ERROR, 361 "detaching buffer from output failed (%d)", status); 362 } 363 364 ALOGV("detached buffer %#" PRIx64 " from output %p", 365 buffer->getId(), from.get()); 366 367 BufferTracker& tracker = *(mBuffers[buffer->getId()]); 368 369 // Merge the release fence of the incoming buffer so that the fence we send 370 // back to the input includes all of the outputs' fences 371 tracker.mergeFence(fence); 372 373 // Check to see if this is the last outstanding reference to this buffer 374 size_t referenceCount = tracker.decrementReferenceCountLocked(); 375 ALOGV("buffer %#" PRIx64 " reference count %zu", buffer->getId(), 376 referenceCount); 377 if (referenceCount > 0) { 378 return; 379 } 380 381 // If we've been abandoned, we can't return the buffer to the input, so just 382 // stop tracking it and move on 383 if (mIsAbandoned) { 384 mBuffers.erase(buffer->getId()); 385 return; 386 } 387 388 // Attach and release the buffer back to the input 389 int consumerSlot = BufferItem::INVALID_BUFFER_SLOT; 390 status = mConsumer->attachBuffer(&consumerSlot, tracker.getBuffer()); 391 LOG_ALWAYS_FATAL_IF(status != NO_ERROR, 392 "attaching buffer to input failed (%d)", status); 393 394 status = mConsumer->releaseBuffer(consumerSlot, /* frameNumber */ 0, 395 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, tracker.getMergedFence()); 396 LOG_ALWAYS_FATAL_IF(status != NO_ERROR, 397 "releasing buffer to input failed (%d)", status); 398 399 ALOGV("released buffer %#" PRIx64 " to input", buffer->getId()); 400 401 // We no longer need to track the buffer once it has been returned to the 402 // input 403 mBuffers.erase(buffer->getId()); 404 405 // Notify any waiting onFrameAvailable calls 406 --mOutstandingBuffers; 407 mReleaseCondition.signal(); 408} 409 410void Camera3StreamSplitter::onAbandonedLocked() { 411 ALOGE("one of my outputs has abandoned me"); 412 if (!mIsAbandoned && mConsumer != nullptr) { 413 mConsumer->consumerDisconnect(); 414 } 415 mIsAbandoned = true; 416 mReleaseCondition.broadcast(); 417} 418 419Camera3StreamSplitter::OutputListener::OutputListener( 420 wp<Camera3StreamSplitter> splitter, 421 wp<IGraphicBufferProducer> output) 422 : mSplitter(splitter), mOutput(output) {} 423 424void Camera3StreamSplitter::OutputListener::onBufferReleased() { 425 sp<Camera3StreamSplitter> splitter = mSplitter.promote(); 426 sp<IGraphicBufferProducer> output = mOutput.promote(); 427 if (splitter != nullptr && output != nullptr) { 428 splitter->onBufferReleasedByOutput(output); 429 } 430} 431 432void Camera3StreamSplitter::OutputListener::binderDied(const wp<IBinder>& /* who */) { 433 sp<Camera3StreamSplitter> splitter = mSplitter.promote(); 434 if (splitter != nullptr) { 435 Mutex::Autolock lock(splitter->mMutex); 436 splitter->onAbandonedLocked(); 437 } 438} 439 440Camera3StreamSplitter::BufferTracker::BufferTracker( 441 const sp<GraphicBuffer>& buffer, size_t referenceCount) 442 : mBuffer(buffer), mMergedFence(Fence::NO_FENCE), 443 mReferenceCount(referenceCount) {} 444 445void Camera3StreamSplitter::BufferTracker::mergeFence(const sp<Fence>& with) { 446 mMergedFence = Fence::merge(String8("Camera3StreamSplitter"), mMergedFence, with); 447} 448 449size_t Camera3StreamSplitter::BufferTracker::decrementReferenceCountLocked() { 450 if (mReferenceCount > 0) 451 --mReferenceCount; 452 return mReferenceCount; 453} 454 455} // namespace android 456