ZslProcessor3.cpp revision 204e3295e2814052aef7e45ee9edd60128efbbd0
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define LOG_TAG "Camera2-ZslProcessor3" 18#define ATRACE_TAG ATRACE_TAG_CAMERA 19//#define LOG_NDEBUG 0 20//#define LOG_NNDEBUG 0 21 22#ifdef LOG_NNDEBUG 23#define ALOGVV(...) ALOGV(__VA_ARGS__) 24#else 25#define ALOGVV(...) ((void)0) 26#endif 27 28#include <inttypes.h> 29 30#include <utils/Log.h> 31#include <utils/Trace.h> 32#include <gui/Surface.h> 33 34#include "common/CameraDeviceBase.h" 35#include "api1/Camera2Client.h" 36#include "api1/client2/CaptureSequencer.h" 37#include "api1/client2/ZslProcessor3.h" 38#include "device3/Camera3Device.h" 39 40namespace android { 41namespace camera2 { 42 43ZslProcessor3::ZslProcessor3( 44 sp<Camera2Client> client, 45 wp<CaptureSequencer> sequencer): 46 Thread(false), 47 mState(RUNNING), 48 mClient(client), 49 mSequencer(sequencer), 50 mId(client->getCameraId()), 51 mZslStreamId(NO_STREAM), 52 mFrameListHead(0), 53 mZslQueueHead(0), 54 mZslQueueTail(0) { 55 // Initialize buffer queue and frame list based on pipeline max depth. 56 size_t pipelineMaxDepth = kDefaultMaxPipelineDepth; 57 if (client != 0) { 58 sp<Camera3Device> device = 59 static_cast<Camera3Device*>(client->getCameraDevice().get()); 60 if (device != 0) { 61 camera_metadata_ro_entry_t entry = 62 device->info().find(ANDROID_REQUEST_PIPELINE_MAX_DEPTH); 63 if (entry.count == 1) { 64 pipelineMaxDepth = entry.data.u8[0]; 65 } else { 66 ALOGW("%s: Unable to find the android.request.pipelineMaxDepth," 67 " use default pipeline max depth %zu", __FUNCTION__, 68 kDefaultMaxPipelineDepth); 69 } 70 } 71 } 72 73 ALOGV("%s: Initialize buffer queue and frame list depth based on max pipeline depth (%d)", 74 __FUNCTION__, pipelineMaxDepth); 75 mBufferQueueDepth = pipelineMaxDepth + 1; 76 mFrameListDepth = pipelineMaxDepth + 1; 77 78 mZslQueue.insertAt(0, mBufferQueueDepth); 79 mFrameList.insertAt(0, mFrameListDepth); 80 sp<CaptureSequencer> captureSequencer = mSequencer.promote(); 81 if (captureSequencer != 0) captureSequencer->setZslProcessor(this); 82} 83 84ZslProcessor3::~ZslProcessor3() { 85 ALOGV("%s: Exit", __FUNCTION__); 86 deleteStream(); 87} 88 89void ZslProcessor3::onResultAvailable(const CaptureResult &result) { 90 ATRACE_CALL(); 91 ALOGV("%s:", __FUNCTION__); 92 Mutex::Autolock l(mInputMutex); 93 camera_metadata_ro_entry_t entry; 94 entry = result.mMetadata.find(ANDROID_SENSOR_TIMESTAMP); 95 nsecs_t timestamp = entry.data.i64[0]; 96 if (entry.count == 0) { 97 ALOGE("%s: metadata doesn't have timestamp, skip this result", __FUNCTION__); 98 return; 99 } 100 (void)timestamp; 101 102 entry = result.mMetadata.find(ANDROID_REQUEST_FRAME_COUNT); 103 if (entry.count == 0) { 104 ALOGE("%s: metadata doesn't have frame number, skip this result", __FUNCTION__); 105 return; 106 } 107 int32_t frameNumber = entry.data.i32[0]; 108 109 ALOGVV("Got preview metadata for frame %d with timestamp %" PRId64, frameNumber, timestamp); 110 111 if (mState != RUNNING) return; 112 113 mFrameList.editItemAt(mFrameListHead) = result.mMetadata; 114 mFrameListHead = (mFrameListHead + 1) % mFrameListDepth; 115} 116 117status_t ZslProcessor3::updateStream(const Parameters ¶ms) { 118 ATRACE_CALL(); 119 ALOGV("%s: Configuring ZSL streams", __FUNCTION__); 120 status_t res; 121 122 Mutex::Autolock l(mInputMutex); 123 124 sp<Camera2Client> client = mClient.promote(); 125 if (client == 0) { 126 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 127 return INVALID_OPERATION; 128 } 129 sp<Camera3Device> device = 130 static_cast<Camera3Device*>(client->getCameraDevice().get()); 131 if (device == 0) { 132 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 133 return INVALID_OPERATION; 134 } 135 136 if (mZslStreamId != NO_STREAM) { 137 // Check if stream parameters have to change 138 uint32_t currentWidth, currentHeight; 139 res = device->getStreamInfo(mZslStreamId, 140 ¤tWidth, ¤tHeight, 0); 141 if (res != OK) { 142 ALOGE("%s: Camera %d: Error querying capture output stream info: " 143 "%s (%d)", __FUNCTION__, 144 client->getCameraId(), strerror(-res), res); 145 return res; 146 } 147 if (currentWidth != (uint32_t)params.fastInfo.arrayWidth || 148 currentHeight != (uint32_t)params.fastInfo.arrayHeight) { 149 ALOGV("%s: Camera %d: Deleting stream %d since the buffer " 150 "dimensions changed", 151 __FUNCTION__, client->getCameraId(), mZslStreamId); 152 res = device->deleteStream(mZslStreamId); 153 if (res == -EBUSY) { 154 ALOGV("%s: Camera %d: Device is busy, call updateStream again " 155 " after it becomes idle", __FUNCTION__, mId); 156 return res; 157 } else if(res != OK) { 158 ALOGE("%s: Camera %d: Unable to delete old output stream " 159 "for ZSL: %s (%d)", __FUNCTION__, 160 client->getCameraId(), strerror(-res), res); 161 return res; 162 } 163 mZslStreamId = NO_STREAM; 164 } 165 } 166 167 if (mZslStreamId == NO_STREAM) { 168 // Create stream for HAL production 169 // TODO: Sort out better way to select resolution for ZSL 170 171 // Note that format specified internally in Camera3ZslStream 172 res = device->createZslStream( 173 params.fastInfo.arrayWidth, params.fastInfo.arrayHeight, 174 mBufferQueueDepth, 175 &mZslStreamId, 176 &mZslStream); 177 if (res != OK) { 178 ALOGE("%s: Camera %d: Can't create ZSL stream: " 179 "%s (%d)", __FUNCTION__, client->getCameraId(), 180 strerror(-res), res); 181 return res; 182 } 183 184 // Only add the camera3 buffer listener when the stream is created. 185 mZslStream->addBufferListener(this); 186 } 187 188 client->registerFrameListener(Camera2Client::kPreviewRequestIdStart, 189 Camera2Client::kPreviewRequestIdEnd, 190 this, 191 /*sendPartials*/false); 192 193 return OK; 194} 195 196status_t ZslProcessor3::deleteStream() { 197 ATRACE_CALL(); 198 status_t res; 199 200 Mutex::Autolock l(mInputMutex); 201 202 if (mZslStreamId != NO_STREAM) { 203 sp<Camera2Client> client = mClient.promote(); 204 if (client == 0) { 205 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 206 return INVALID_OPERATION; 207 } 208 209 sp<Camera3Device> device = 210 reinterpret_cast<Camera3Device*>(client->getCameraDevice().get()); 211 if (device == 0) { 212 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 213 return INVALID_OPERATION; 214 } 215 216 res = device->deleteStream(mZslStreamId); 217 if (res != OK) { 218 ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: " 219 "%s (%d)", __FUNCTION__, client->getCameraId(), 220 mZslStreamId, strerror(-res), res); 221 return res; 222 } 223 224 mZslStreamId = NO_STREAM; 225 } 226 return OK; 227} 228 229int ZslProcessor3::getStreamId() const { 230 Mutex::Autolock l(mInputMutex); 231 return mZslStreamId; 232} 233 234status_t ZslProcessor3::pushToReprocess(int32_t requestId) { 235 ALOGV("%s: Send in reprocess request with id %d", 236 __FUNCTION__, requestId); 237 Mutex::Autolock l(mInputMutex); 238 status_t res; 239 sp<Camera2Client> client = mClient.promote(); 240 241 if (client == 0) { 242 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 243 return INVALID_OPERATION; 244 } 245 246 IF_ALOGV() { 247 dumpZslQueue(-1); 248 } 249 250 size_t metadataIdx; 251 nsecs_t candidateTimestamp = getCandidateTimestampLocked(&metadataIdx); 252 253 if (candidateTimestamp == -1) { 254 ALOGE("%s: Could not find good candidate for ZSL reprocessing", 255 __FUNCTION__); 256 return NOT_ENOUGH_DATA; 257 } 258 259 res = mZslStream->enqueueInputBufferByTimestamp(candidateTimestamp, 260 /*actualTimestamp*/NULL); 261 262 if (res == mZslStream->NO_BUFFER_AVAILABLE) { 263 ALOGV("%s: No ZSL buffers yet", __FUNCTION__); 264 return NOT_ENOUGH_DATA; 265 } else if (res != OK) { 266 ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)", 267 __FUNCTION__, strerror(-res), res); 268 return res; 269 } 270 271 { 272 CameraMetadata request = mFrameList[metadataIdx]; 273 274 // Verify that the frame is reasonable for reprocessing 275 276 camera_metadata_entry_t entry; 277 entry = request.find(ANDROID_CONTROL_AE_STATE); 278 if (entry.count == 0) { 279 ALOGE("%s: ZSL queue frame has no AE state field!", 280 __FUNCTION__); 281 return BAD_VALUE; 282 } 283 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 284 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 285 ALOGV("%s: ZSL queue frame AE state is %d, need full capture", 286 __FUNCTION__, entry.data.u8[0]); 287 return NOT_ENOUGH_DATA; 288 } 289 290 uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS; 291 res = request.update(ANDROID_REQUEST_TYPE, 292 &requestType, 1); 293 if (res != OK) { 294 ALOGE("%s: Unable to update request type", 295 __FUNCTION__); 296 return INVALID_OPERATION; 297 } 298 299 int32_t inputStreams[1] = 300 { mZslStreamId }; 301 res = request.update(ANDROID_REQUEST_INPUT_STREAMS, 302 inputStreams, 1); 303 if (res != OK) { 304 ALOGE("%s: Unable to update request input streams", 305 __FUNCTION__); 306 return INVALID_OPERATION; 307 } 308 309 uint8_t captureIntent = 310 static_cast<uint8_t>(ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE); 311 res = request.update(ANDROID_CONTROL_CAPTURE_INTENT, 312 &captureIntent, 1); 313 if (res != OK ) { 314 ALOGE("%s: Unable to update request capture intent", 315 __FUNCTION__); 316 return INVALID_OPERATION; 317 } 318 319 // TODO: Shouldn't we also update the latest preview frame? 320 int32_t outputStreams[1] = 321 { client->getCaptureStreamId() }; 322 res = request.update(ANDROID_REQUEST_OUTPUT_STREAMS, 323 outputStreams, 1); 324 if (res != OK) { 325 ALOGE("%s: Unable to update request output streams", 326 __FUNCTION__); 327 return INVALID_OPERATION; 328 } 329 330 res = request.update(ANDROID_REQUEST_ID, 331 &requestId, 1); 332 if (res != OK ) { 333 ALOGE("%s: Unable to update frame to a reprocess request", 334 __FUNCTION__); 335 return INVALID_OPERATION; 336 } 337 338 res = client->stopStream(); 339 if (res != OK) { 340 ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: " 341 "%s (%d)", 342 __FUNCTION__, client->getCameraId(), strerror(-res), res); 343 return INVALID_OPERATION; 344 } 345 346 // Update JPEG settings 347 { 348 SharedParameters::Lock l(client->getParameters()); 349 res = l.mParameters.updateRequestJpeg(&request); 350 if (res != OK) { 351 ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL " 352 "capture request: %s (%d)", __FUNCTION__, 353 client->getCameraId(), 354 strerror(-res), res); 355 return res; 356 } 357 } 358 359 mLatestCapturedRequest = request; 360 res = client->getCameraDevice()->capture(request); 361 if (res != OK ) { 362 ALOGE("%s: Unable to send ZSL reprocess request to capture: %s" 363 " (%d)", __FUNCTION__, strerror(-res), res); 364 return res; 365 } 366 367 mState = LOCKED; 368 } 369 370 return OK; 371} 372 373status_t ZslProcessor3::clearZslQueue() { 374 Mutex::Autolock l(mInputMutex); 375 // If in middle of capture, can't clear out queue 376 if (mState == LOCKED) return OK; 377 378 return clearZslQueueLocked(); 379} 380 381status_t ZslProcessor3::clearZslQueueLocked() { 382 if (mZslStream != 0) { 383 // clear result metadata list first. 384 clearZslResultQueueLocked(); 385 return mZslStream->clearInputRingBuffer(); 386 } 387 return OK; 388} 389 390void ZslProcessor3::clearZslResultQueueLocked() { 391 mFrameList.clear(); 392 mFrameListHead = 0; 393 mFrameList.insertAt(0, mFrameListDepth); 394} 395 396void ZslProcessor3::dump(int fd, const Vector<String16>& /*args*/) const { 397 Mutex::Autolock l(mInputMutex); 398 if (!mLatestCapturedRequest.isEmpty()) { 399 String8 result(" Latest ZSL capture request:\n"); 400 write(fd, result.string(), result.size()); 401 mLatestCapturedRequest.dump(fd, 2, 6); 402 } else { 403 String8 result(" Latest ZSL capture request: none yet\n"); 404 write(fd, result.string(), result.size()); 405 } 406 dumpZslQueue(fd); 407} 408 409bool ZslProcessor3::threadLoop() { 410 // TODO: remove dependency on thread. For now, shut thread down right 411 // away. 412 return false; 413} 414 415void ZslProcessor3::dumpZslQueue(int fd) const { 416 String8 header("ZSL queue contents:"); 417 String8 indent(" "); 418 ALOGV("%s", header.string()); 419 if (fd != -1) { 420 header = indent + header + "\n"; 421 write(fd, header.string(), header.size()); 422 } 423 for (size_t i = 0; i < mZslQueue.size(); i++) { 424 const ZslPair &queueEntry = mZslQueue[i]; 425 nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp; 426 camera_metadata_ro_entry_t entry; 427 nsecs_t frameTimestamp = 0; 428 int frameAeState = -1; 429 if (!queueEntry.frame.isEmpty()) { 430 entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP); 431 if (entry.count > 0) frameTimestamp = entry.data.i64[0]; 432 entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE); 433 if (entry.count > 0) frameAeState = entry.data.u8[0]; 434 } 435 String8 result = 436 String8::format(" %zu: b: %" PRId64 "\tf: %" PRId64 ", AE state: %d", i, 437 bufferTimestamp, frameTimestamp, frameAeState); 438 ALOGV("%s", result.string()); 439 if (fd != -1) { 440 result = indent + result + "\n"; 441 write(fd, result.string(), result.size()); 442 } 443 444 } 445} 446 447nsecs_t ZslProcessor3::getCandidateTimestampLocked(size_t* metadataIdx) const { 448 /** 449 * Find the smallest timestamp we know about so far 450 * - ensure that aeState is either converged or locked 451 */ 452 453 size_t idx = 0; 454 nsecs_t minTimestamp = -1; 455 456 size_t emptyCount = mFrameList.size(); 457 458 for (size_t j = 0; j < mFrameList.size(); j++) { 459 const CameraMetadata &frame = mFrameList[j]; 460 if (!frame.isEmpty()) { 461 462 emptyCount--; 463 464 camera_metadata_ro_entry_t entry; 465 entry = frame.find(ANDROID_SENSOR_TIMESTAMP); 466 if (entry.count == 0) { 467 ALOGE("%s: Can't find timestamp in frame!", 468 __FUNCTION__); 469 continue; 470 } 471 nsecs_t frameTimestamp = entry.data.i64[0]; 472 if (minTimestamp > frameTimestamp || minTimestamp == -1) { 473 474 entry = frame.find(ANDROID_CONTROL_AE_STATE); 475 476 if (entry.count == 0) { 477 /** 478 * This is most likely a HAL bug. The aeState field is 479 * mandatory, so it should always be in a metadata packet. 480 */ 481 ALOGW("%s: ZSL queue frame has no AE state field!", 482 __FUNCTION__); 483 continue; 484 } 485 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 486 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 487 ALOGVV("%s: ZSL queue frame AE state is %d, need " 488 "full capture", __FUNCTION__, entry.data.u8[0]); 489 continue; 490 } 491 492 minTimestamp = frameTimestamp; 493 idx = j; 494 } 495 496 ALOGVV("%s: Saw timestamp %" PRId64, __FUNCTION__, frameTimestamp); 497 } 498 } 499 500 if (emptyCount == mFrameList.size()) { 501 /** 502 * This could be mildly bad and means our ZSL was triggered before 503 * there were any frames yet received by the camera framework. 504 * 505 * This is a fairly corner case which can happen under: 506 * + a user presses the shutter button real fast when the camera starts 507 * (startPreview followed immediately by takePicture). 508 * + burst capture case (hitting shutter button as fast possible) 509 * 510 * If this happens in steady case (preview running for a while, call 511 * a single takePicture) then this might be a fwk bug. 512 */ 513 ALOGW("%s: ZSL queue has no metadata frames", __FUNCTION__); 514 } 515 516 ALOGV("%s: Candidate timestamp %" PRId64 " (idx %zu), empty frames: %zu", 517 __FUNCTION__, minTimestamp, idx, emptyCount); 518 519 if (metadataIdx) { 520 *metadataIdx = idx; 521 } 522 523 return minTimestamp; 524} 525 526void ZslProcessor3::onBufferAcquired(const BufferInfo& /*bufferInfo*/) { 527 // Intentionally left empty 528 // Although theoretically we could use this to get better dump info 529} 530 531void ZslProcessor3::onBufferReleased(const BufferInfo& bufferInfo) { 532 Mutex::Autolock l(mInputMutex); 533 534 // ignore output buffers 535 if (bufferInfo.mOutput) { 536 return; 537 } 538 539 // TODO: Verify that the buffer is in our queue by looking at timestamp 540 // theoretically unnecessary unless we change the following assumptions: 541 // -- only 1 buffer reprocessed at a time (which is the case now) 542 543 // Erase entire ZSL queue since we've now completed the capture and preview 544 // is stopped. 545 // 546 // We need to guarantee that if we do two back-to-back captures, 547 // the second won't use a buffer that's older/the same as the first, which 548 // is theoretically possible if we don't clear out the queue and the 549 // selection criteria is something like 'newest'. Clearing out the result 550 // metadata queue on a completed capture ensures we'll only use new timestamp. 551 // Calling clearZslQueueLocked is a guaranteed deadlock because this callback 552 // holds the Camera3Stream internal lock (mLock), and clearZslQueueLocked requires 553 // to hold the same lock. 554 // TODO: need figure out a way to clear the Zsl buffer queue properly. Right now 555 // it is safe not to do so, as back to back ZSL capture requires stop and start 556 // preview, which will flush ZSL queue automatically. 557 ALOGV("%s: Memory optimization, clearing ZSL queue", 558 __FUNCTION__); 559 clearZslResultQueueLocked(); 560 561 // Required so we accept more ZSL requests 562 mState = RUNNING; 563} 564 565}; // namespace camera2 566}; // namespace android 567