ZslProcessor3.cpp revision 6b7a2294b9e4da784cfe4b562ee1720ad606c852
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define LOG_TAG "Camera2-ZslProcessor3" 18#define ATRACE_TAG ATRACE_TAG_CAMERA 19//#define LOG_NDEBUG 0 20//#define LOG_NNDEBUG 0 21 22#ifdef LOG_NNDEBUG 23#define ALOGVV(...) ALOGV(__VA_ARGS__) 24#else 25#define ALOGVV(...) ((void)0) 26#endif 27 28#include <inttypes.h> 29 30#include <utils/Log.h> 31#include <utils/Trace.h> 32#include <gui/Surface.h> 33 34#include "common/CameraDeviceBase.h" 35#include "api1/Camera2Client.h" 36#include "api1/client2/CaptureSequencer.h" 37#include "api1/client2/ZslProcessor3.h" 38#include "device3/Camera3Device.h" 39 40namespace android { 41namespace camera2 { 42 43ZslProcessor3::ZslProcessor3( 44 sp<Camera2Client> client, 45 wp<CaptureSequencer> sequencer): 46 Thread(false), 47 mLatestClearedBufferTimestamp(0), 48 mState(RUNNING), 49 mClient(client), 50 mSequencer(sequencer), 51 mId(client->getCameraId()), 52 mZslStreamId(NO_STREAM), 53 mFrameListHead(0), 54 mZslQueueHead(0), 55 mZslQueueTail(0), 56 mHasFocuser(false) { 57 // Initialize buffer queue and frame list based on pipeline max depth. 58 size_t pipelineMaxDepth = kDefaultMaxPipelineDepth; 59 if (client != 0) { 60 sp<Camera3Device> device = 61 static_cast<Camera3Device*>(client->getCameraDevice().get()); 62 if (device != 0) { 63 camera_metadata_ro_entry_t entry = 64 device->info().find(ANDROID_REQUEST_PIPELINE_MAX_DEPTH); 65 if (entry.count == 1) { 66 pipelineMaxDepth = entry.data.u8[0]; 67 } else { 68 ALOGW("%s: Unable to find the android.request.pipelineMaxDepth," 69 " use default pipeline max depth %zu", __FUNCTION__, 70 kDefaultMaxPipelineDepth); 71 } 72 73 entry = device->info().find(ANDROID_LENS_INFO_MINIMUM_FOCUS_DISTANCE); 74 if (entry.count > 0 && entry.data.f[0] != 0.) { 75 mHasFocuser = true; 76 } 77 } 78 } 79 80 ALOGV("%s: Initialize buffer queue and frame list depth based on max pipeline depth (%d)", 81 __FUNCTION__, pipelineMaxDepth); 82 // Need to keep buffer queue longer than metadata queue because sometimes buffer arrives 83 // earlier than metadata which causes the buffer corresponding to oldest metadata being 84 // removed. 85 mFrameListDepth = pipelineMaxDepth; 86 mBufferQueueDepth = mFrameListDepth + 1; 87 88 89 mZslQueue.insertAt(0, mBufferQueueDepth); 90 mFrameList.insertAt(0, mFrameListDepth); 91 sp<CaptureSequencer> captureSequencer = mSequencer.promote(); 92 if (captureSequencer != 0) captureSequencer->setZslProcessor(this); 93} 94 95ZslProcessor3::~ZslProcessor3() { 96 ALOGV("%s: Exit", __FUNCTION__); 97 deleteStream(); 98} 99 100void ZslProcessor3::onResultAvailable(const CaptureResult &result) { 101 ATRACE_CALL(); 102 ALOGV("%s:", __FUNCTION__); 103 Mutex::Autolock l(mInputMutex); 104 camera_metadata_ro_entry_t entry; 105 entry = result.mMetadata.find(ANDROID_SENSOR_TIMESTAMP); 106 nsecs_t timestamp = entry.data.i64[0]; 107 if (entry.count == 0) { 108 ALOGE("%s: metadata doesn't have timestamp, skip this result", __FUNCTION__); 109 return; 110 } 111 112 entry = result.mMetadata.find(ANDROID_REQUEST_FRAME_COUNT); 113 if (entry.count == 0) { 114 ALOGE("%s: metadata doesn't have frame number, skip this result", __FUNCTION__); 115 return; 116 } 117 int32_t frameNumber = entry.data.i32[0]; 118 119 ALOGVV("Got preview metadata for frame %d with timestamp %" PRId64, frameNumber, timestamp); 120 121 if (mState != RUNNING) return; 122 123 // Corresponding buffer has been cleared. No need to push into mFrameList 124 if (timestamp <= mLatestClearedBufferTimestamp) return; 125 126 mFrameList.editItemAt(mFrameListHead) = result.mMetadata; 127 mFrameListHead = (mFrameListHead + 1) % mFrameListDepth; 128} 129 130status_t ZslProcessor3::updateStream(const Parameters ¶ms) { 131 ATRACE_CALL(); 132 ALOGV("%s: Configuring ZSL streams", __FUNCTION__); 133 status_t res; 134 135 Mutex::Autolock l(mInputMutex); 136 137 sp<Camera2Client> client = mClient.promote(); 138 if (client == 0) { 139 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 140 return INVALID_OPERATION; 141 } 142 sp<Camera3Device> device = 143 static_cast<Camera3Device*>(client->getCameraDevice().get()); 144 if (device == 0) { 145 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 146 return INVALID_OPERATION; 147 } 148 149 if (mZslStreamId != NO_STREAM) { 150 // Check if stream parameters have to change 151 uint32_t currentWidth, currentHeight; 152 res = device->getStreamInfo(mZslStreamId, 153 ¤tWidth, ¤tHeight, 0); 154 if (res != OK) { 155 ALOGE("%s: Camera %d: Error querying capture output stream info: " 156 "%s (%d)", __FUNCTION__, 157 client->getCameraId(), strerror(-res), res); 158 return res; 159 } 160 if (currentWidth != (uint32_t)params.fastInfo.arrayWidth || 161 currentHeight != (uint32_t)params.fastInfo.arrayHeight) { 162 ALOGV("%s: Camera %d: Deleting stream %d since the buffer " 163 "dimensions changed", 164 __FUNCTION__, client->getCameraId(), mZslStreamId); 165 res = device->deleteStream(mZslStreamId); 166 if (res == -EBUSY) { 167 ALOGV("%s: Camera %d: Device is busy, call updateStream again " 168 " after it becomes idle", __FUNCTION__, mId); 169 return res; 170 } else if(res != OK) { 171 ALOGE("%s: Camera %d: Unable to delete old output stream " 172 "for ZSL: %s (%d)", __FUNCTION__, 173 client->getCameraId(), strerror(-res), res); 174 return res; 175 } 176 mZslStreamId = NO_STREAM; 177 } 178 } 179 180 if (mZslStreamId == NO_STREAM) { 181 // Create stream for HAL production 182 // TODO: Sort out better way to select resolution for ZSL 183 184 // Note that format specified internally in Camera3ZslStream 185 res = device->createZslStream( 186 params.fastInfo.arrayWidth, params.fastInfo.arrayHeight, 187 mBufferQueueDepth, 188 &mZslStreamId, 189 &mZslStream); 190 if (res != OK) { 191 ALOGE("%s: Camera %d: Can't create ZSL stream: " 192 "%s (%d)", __FUNCTION__, client->getCameraId(), 193 strerror(-res), res); 194 return res; 195 } 196 197 // Only add the camera3 buffer listener when the stream is created. 198 mZslStream->addBufferListener(this); 199 } 200 201 client->registerFrameListener(Camera2Client::kPreviewRequestIdStart, 202 Camera2Client::kPreviewRequestIdEnd, 203 this, 204 /*sendPartials*/false); 205 206 return OK; 207} 208 209status_t ZslProcessor3::deleteStream() { 210 ATRACE_CALL(); 211 status_t res; 212 213 Mutex::Autolock l(mInputMutex); 214 215 if (mZslStreamId != NO_STREAM) { 216 sp<Camera2Client> client = mClient.promote(); 217 if (client == 0) { 218 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 219 return INVALID_OPERATION; 220 } 221 222 sp<Camera3Device> device = 223 reinterpret_cast<Camera3Device*>(client->getCameraDevice().get()); 224 if (device == 0) { 225 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 226 return INVALID_OPERATION; 227 } 228 229 res = device->deleteStream(mZslStreamId); 230 if (res != OK) { 231 ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: " 232 "%s (%d)", __FUNCTION__, client->getCameraId(), 233 mZslStreamId, strerror(-res), res); 234 return res; 235 } 236 237 mZslStreamId = NO_STREAM; 238 } 239 return OK; 240} 241 242int ZslProcessor3::getStreamId() const { 243 Mutex::Autolock l(mInputMutex); 244 return mZslStreamId; 245} 246 247status_t ZslProcessor3::pushToReprocess(int32_t requestId) { 248 ALOGV("%s: Send in reprocess request with id %d", 249 __FUNCTION__, requestId); 250 Mutex::Autolock l(mInputMutex); 251 status_t res; 252 sp<Camera2Client> client = mClient.promote(); 253 254 if (client == 0) { 255 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 256 return INVALID_OPERATION; 257 } 258 259 IF_ALOGV() { 260 dumpZslQueue(-1); 261 } 262 263 size_t metadataIdx; 264 nsecs_t candidateTimestamp = getCandidateTimestampLocked(&metadataIdx); 265 266 if (candidateTimestamp == -1) { 267 ALOGE("%s: Could not find good candidate for ZSL reprocessing", 268 __FUNCTION__); 269 return NOT_ENOUGH_DATA; 270 } 271 272 res = mZslStream->enqueueInputBufferByTimestamp(candidateTimestamp, 273 /*actualTimestamp*/NULL); 274 275 if (res == mZslStream->NO_BUFFER_AVAILABLE) { 276 ALOGV("%s: No ZSL buffers yet", __FUNCTION__); 277 return NOT_ENOUGH_DATA; 278 } else if (res != OK) { 279 ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)", 280 __FUNCTION__, strerror(-res), res); 281 return res; 282 } 283 284 { 285 CameraMetadata request = mFrameList[metadataIdx]; 286 287 // Verify that the frame is reasonable for reprocessing 288 289 camera_metadata_entry_t entry; 290 entry = request.find(ANDROID_CONTROL_AE_STATE); 291 if (entry.count == 0) { 292 ALOGE("%s: ZSL queue frame has no AE state field!", 293 __FUNCTION__); 294 return BAD_VALUE; 295 } 296 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 297 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 298 ALOGV("%s: ZSL queue frame AE state is %d, need full capture", 299 __FUNCTION__, entry.data.u8[0]); 300 return NOT_ENOUGH_DATA; 301 } 302 303 uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS; 304 res = request.update(ANDROID_REQUEST_TYPE, 305 &requestType, 1); 306 if (res != OK) { 307 ALOGE("%s: Unable to update request type", 308 __FUNCTION__); 309 return INVALID_OPERATION; 310 } 311 312 int32_t inputStreams[1] = 313 { mZslStreamId }; 314 res = request.update(ANDROID_REQUEST_INPUT_STREAMS, 315 inputStreams, 1); 316 if (res != OK) { 317 ALOGE("%s: Unable to update request input streams", 318 __FUNCTION__); 319 return INVALID_OPERATION; 320 } 321 322 uint8_t captureIntent = 323 static_cast<uint8_t>(ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE); 324 res = request.update(ANDROID_CONTROL_CAPTURE_INTENT, 325 &captureIntent, 1); 326 if (res != OK ) { 327 ALOGE("%s: Unable to update request capture intent", 328 __FUNCTION__); 329 return INVALID_OPERATION; 330 } 331 332 // TODO: Shouldn't we also update the latest preview frame? 333 int32_t outputStreams[1] = 334 { client->getCaptureStreamId() }; 335 res = request.update(ANDROID_REQUEST_OUTPUT_STREAMS, 336 outputStreams, 1); 337 if (res != OK) { 338 ALOGE("%s: Unable to update request output streams", 339 __FUNCTION__); 340 return INVALID_OPERATION; 341 } 342 343 res = request.update(ANDROID_REQUEST_ID, 344 &requestId, 1); 345 if (res != OK ) { 346 ALOGE("%s: Unable to update frame to a reprocess request", 347 __FUNCTION__); 348 return INVALID_OPERATION; 349 } 350 351 res = client->stopStream(); 352 if (res != OK) { 353 ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: " 354 "%s (%d)", 355 __FUNCTION__, client->getCameraId(), strerror(-res), res); 356 return INVALID_OPERATION; 357 } 358 359 // Update JPEG settings 360 { 361 SharedParameters::Lock l(client->getParameters()); 362 res = l.mParameters.updateRequestJpeg(&request); 363 if (res != OK) { 364 ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL " 365 "capture request: %s (%d)", __FUNCTION__, 366 client->getCameraId(), 367 strerror(-res), res); 368 return res; 369 } 370 } 371 372 mLatestCapturedRequest = request; 373 res = client->getCameraDevice()->capture(request); 374 if (res != OK ) { 375 ALOGE("%s: Unable to send ZSL reprocess request to capture: %s" 376 " (%d)", __FUNCTION__, strerror(-res), res); 377 return res; 378 } 379 380 mState = LOCKED; 381 } 382 383 return OK; 384} 385 386status_t ZslProcessor3::clearZslQueue() { 387 Mutex::Autolock l(mInputMutex); 388 // If in middle of capture, can't clear out queue 389 if (mState == LOCKED) return OK; 390 391 return clearZslQueueLocked(); 392} 393 394status_t ZslProcessor3::clearZslQueueLocked() { 395 if (mZslStream != 0) { 396 // clear result metadata list first. 397 clearZslResultQueueLocked(); 398 return mZslStream->clearInputRingBuffer(&mLatestClearedBufferTimestamp); 399 } 400 return OK; 401} 402 403void ZslProcessor3::clearZslResultQueueLocked() { 404 mFrameList.clear(); 405 mFrameListHead = 0; 406 mFrameList.insertAt(0, mFrameListDepth); 407} 408 409void ZslProcessor3::dump(int fd, const Vector<String16>& /*args*/) const { 410 Mutex::Autolock l(mInputMutex); 411 if (!mLatestCapturedRequest.isEmpty()) { 412 String8 result(" Latest ZSL capture request:\n"); 413 write(fd, result.string(), result.size()); 414 mLatestCapturedRequest.dump(fd, 2, 6); 415 } else { 416 String8 result(" Latest ZSL capture request: none yet\n"); 417 write(fd, result.string(), result.size()); 418 } 419 dumpZslQueue(fd); 420} 421 422bool ZslProcessor3::threadLoop() { 423 // TODO: remove dependency on thread. For now, shut thread down right 424 // away. 425 return false; 426} 427 428void ZslProcessor3::dumpZslQueue(int fd) const { 429 String8 header("ZSL queue contents:"); 430 String8 indent(" "); 431 ALOGV("%s", header.string()); 432 if (fd != -1) { 433 header = indent + header + "\n"; 434 write(fd, header.string(), header.size()); 435 } 436 for (size_t i = 0; i < mZslQueue.size(); i++) { 437 const ZslPair &queueEntry = mZslQueue[i]; 438 nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp; 439 camera_metadata_ro_entry_t entry; 440 nsecs_t frameTimestamp = 0; 441 int frameAeState = -1; 442 if (!queueEntry.frame.isEmpty()) { 443 entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP); 444 if (entry.count > 0) frameTimestamp = entry.data.i64[0]; 445 entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE); 446 if (entry.count > 0) frameAeState = entry.data.u8[0]; 447 } 448 String8 result = 449 String8::format(" %zu: b: %" PRId64 "\tf: %" PRId64 ", AE state: %d", i, 450 bufferTimestamp, frameTimestamp, frameAeState); 451 ALOGV("%s", result.string()); 452 if (fd != -1) { 453 result = indent + result + "\n"; 454 write(fd, result.string(), result.size()); 455 } 456 457 } 458} 459 460nsecs_t ZslProcessor3::getCandidateTimestampLocked(size_t* metadataIdx) const { 461 /** 462 * Find the smallest timestamp we know about so far 463 * - ensure that aeState is either converged or locked 464 */ 465 466 size_t idx = 0; 467 nsecs_t minTimestamp = -1; 468 469 size_t emptyCount = mFrameList.size(); 470 471 for (size_t j = 0; j < mFrameList.size(); j++) { 472 const CameraMetadata &frame = mFrameList[j]; 473 if (!frame.isEmpty()) { 474 475 emptyCount--; 476 477 camera_metadata_ro_entry_t entry; 478 entry = frame.find(ANDROID_SENSOR_TIMESTAMP); 479 if (entry.count == 0) { 480 ALOGE("%s: Can't find timestamp in frame!", 481 __FUNCTION__); 482 continue; 483 } 484 nsecs_t frameTimestamp = entry.data.i64[0]; 485 if (minTimestamp > frameTimestamp || minTimestamp == -1) { 486 487 entry = frame.find(ANDROID_CONTROL_AE_STATE); 488 489 if (entry.count == 0) { 490 /** 491 * This is most likely a HAL bug. The aeState field is 492 * mandatory, so it should always be in a metadata packet. 493 */ 494 ALOGW("%s: ZSL queue frame has no AE state field!", 495 __FUNCTION__); 496 continue; 497 } 498 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 499 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 500 ALOGVV("%s: ZSL queue frame AE state is %d, need " 501 "full capture", __FUNCTION__, entry.data.u8[0]); 502 continue; 503 } 504 505 // Check AF state if device has focuser 506 if (mHasFocuser) { 507 // Make sure the candidate frame has good focus. 508 entry = frame.find(ANDROID_CONTROL_AF_STATE); 509 if (entry.count == 0) { 510 ALOGW("%s: ZSL queue frame has no AF state field!", 511 __FUNCTION__); 512 continue; 513 } 514 uint8_t afState = entry.data.u8[0]; 515 if (afState != ANDROID_CONTROL_AF_STATE_PASSIVE_FOCUSED && 516 afState != ANDROID_CONTROL_AF_STATE_FOCUSED_LOCKED && 517 afState != ANDROID_CONTROL_AF_STATE_NOT_FOCUSED_LOCKED) { 518 ALOGW("%s: ZSL queue frame AF state is %d is not good for capture, skip it", 519 __FUNCTION__, afState); 520 continue; 521 } 522 } 523 524 minTimestamp = frameTimestamp; 525 idx = j; 526 } 527 528 ALOGVV("%s: Saw timestamp %" PRId64, __FUNCTION__, frameTimestamp); 529 } 530 } 531 532 if (emptyCount == mFrameList.size()) { 533 /** 534 * This could be mildly bad and means our ZSL was triggered before 535 * there were any frames yet received by the camera framework. 536 * 537 * This is a fairly corner case which can happen under: 538 * + a user presses the shutter button real fast when the camera starts 539 * (startPreview followed immediately by takePicture). 540 * + burst capture case (hitting shutter button as fast possible) 541 * 542 * If this happens in steady case (preview running for a while, call 543 * a single takePicture) then this might be a fwk bug. 544 */ 545 ALOGW("%s: ZSL queue has no metadata frames", __FUNCTION__); 546 } 547 548 ALOGV("%s: Candidate timestamp %" PRId64 " (idx %zu), empty frames: %zu", 549 __FUNCTION__, minTimestamp, idx, emptyCount); 550 551 if (metadataIdx) { 552 *metadataIdx = idx; 553 } 554 555 return minTimestamp; 556} 557 558void ZslProcessor3::onBufferAcquired(const BufferInfo& /*bufferInfo*/) { 559 // Intentionally left empty 560 // Although theoretically we could use this to get better dump info 561} 562 563void ZslProcessor3::onBufferReleased(const BufferInfo& bufferInfo) { 564 565 // ignore output buffers 566 if (bufferInfo.mOutput) { 567 return; 568 } 569 570 // Lock mutex only once we know this is an input buffer returned to avoid 571 // potential deadlock 572 Mutex::Autolock l(mInputMutex); 573 // TODO: Verify that the buffer is in our queue by looking at timestamp 574 // theoretically unnecessary unless we change the following assumptions: 575 // -- only 1 buffer reprocessed at a time (which is the case now) 576 577 // Erase entire ZSL queue since we've now completed the capture and preview 578 // is stopped. 579 // 580 // We need to guarantee that if we do two back-to-back captures, 581 // the second won't use a buffer that's older/the same as the first, which 582 // is theoretically possible if we don't clear out the queue and the 583 // selection criteria is something like 'newest'. Clearing out the result 584 // metadata queue on a completed capture ensures we'll only use new timestamp. 585 // Calling clearZslQueueLocked is a guaranteed deadlock because this callback 586 // holds the Camera3Stream internal lock (mLock), and clearZslQueueLocked requires 587 // to hold the same lock. 588 // TODO: need figure out a way to clear the Zsl buffer queue properly. Right now 589 // it is safe not to do so, as back to back ZSL capture requires stop and start 590 // preview, which will flush ZSL queue automatically. 591 ALOGV("%s: Memory optimization, clearing ZSL queue", 592 __FUNCTION__); 593 clearZslResultQueueLocked(); 594 595 // Required so we accept more ZSL requests 596 mState = RUNNING; 597} 598 599}; // namespace camera2 600}; // namespace android 601