ZslProcessor3.cpp revision 690895cfe0f128ee3fc375a357ab99b0b98394b6
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define LOG_TAG "Camera2-ZslProcessor3" 18#define ATRACE_TAG ATRACE_TAG_CAMERA 19//#define LOG_NDEBUG 0 20//#define LOG_NNDEBUG 0 21 22#ifdef LOG_NNDEBUG 23#define ALOGVV(...) ALOGV(__VA_ARGS__) 24#else 25#define ALOGVV(...) ((void)0) 26#endif 27 28#include <inttypes.h> 29 30#include <utils/Log.h> 31#include <utils/Trace.h> 32#include <gui/Surface.h> 33 34#include "common/CameraDeviceBase.h" 35#include "api1/Camera2Client.h" 36#include "api1/client2/CaptureSequencer.h" 37#include "api1/client2/ZslProcessor3.h" 38#include "device3/Camera3Device.h" 39 40namespace android { 41namespace camera2 { 42 43ZslProcessor3::ZslProcessor3( 44 sp<Camera2Client> client, 45 wp<CaptureSequencer> sequencer): 46 Thread(false), 47 mState(RUNNING), 48 mClient(client), 49 mSequencer(sequencer), 50 mId(client->getCameraId()), 51 mZslStreamId(NO_STREAM), 52 mFrameListHead(0), 53 mZslQueueHead(0), 54 mZslQueueTail(0), 55 mHasFocuser(false) { 56 // Initialize buffer queue and frame list based on pipeline max depth. 57 size_t pipelineMaxDepth = kDefaultMaxPipelineDepth; 58 if (client != 0) { 59 sp<Camera3Device> device = 60 static_cast<Camera3Device*>(client->getCameraDevice().get()); 61 if (device != 0) { 62 camera_metadata_ro_entry_t entry = 63 device->info().find(ANDROID_REQUEST_PIPELINE_MAX_DEPTH); 64 if (entry.count == 1) { 65 pipelineMaxDepth = entry.data.u8[0]; 66 } else { 67 ALOGW("%s: Unable to find the android.request.pipelineMaxDepth," 68 " use default pipeline max depth %zu", __FUNCTION__, 69 kDefaultMaxPipelineDepth); 70 } 71 72 entry = device->info().find(ANDROID_LENS_INFO_MINIMUM_FOCUS_DISTANCE); 73 if (entry.count > 0 && entry.data.f[0] != 0.) { 74 mHasFocuser = true; 75 } 76 } 77 } 78 79 ALOGV("%s: Initialize buffer queue and frame list depth based on max pipeline depth (%d)", 80 __FUNCTION__, pipelineMaxDepth); 81 // Need to keep buffer queue longer than metadata queue because sometimes buffer arrives 82 // earlier than metadata which causes the buffer corresponding to oldest metadata being 83 // removed. 84 mFrameListDepth = pipelineMaxDepth; 85 mBufferQueueDepth = mFrameListDepth + 1; 86 87 88 mZslQueue.insertAt(0, mBufferQueueDepth); 89 mFrameList.insertAt(0, mFrameListDepth); 90 sp<CaptureSequencer> captureSequencer = mSequencer.promote(); 91 if (captureSequencer != 0) captureSequencer->setZslProcessor(this); 92} 93 94ZslProcessor3::~ZslProcessor3() { 95 ALOGV("%s: Exit", __FUNCTION__); 96 deleteStream(); 97} 98 99void ZslProcessor3::onResultAvailable(const CaptureResult &result) { 100 ATRACE_CALL(); 101 ALOGV("%s:", __FUNCTION__); 102 Mutex::Autolock l(mInputMutex); 103 camera_metadata_ro_entry_t entry; 104 entry = result.mMetadata.find(ANDROID_SENSOR_TIMESTAMP); 105 nsecs_t timestamp = entry.data.i64[0]; 106 if (entry.count == 0) { 107 ALOGE("%s: metadata doesn't have timestamp, skip this result", __FUNCTION__); 108 return; 109 } 110 (void)timestamp; 111 112 entry = result.mMetadata.find(ANDROID_REQUEST_FRAME_COUNT); 113 if (entry.count == 0) { 114 ALOGE("%s: metadata doesn't have frame number, skip this result", __FUNCTION__); 115 return; 116 } 117 int32_t frameNumber = entry.data.i32[0]; 118 119 ALOGVV("Got preview metadata for frame %d with timestamp %" PRId64, frameNumber, timestamp); 120 121 if (mState != RUNNING) return; 122 123 mFrameList.editItemAt(mFrameListHead) = result.mMetadata; 124 mFrameListHead = (mFrameListHead + 1) % mFrameListDepth; 125} 126 127status_t ZslProcessor3::updateStream(const Parameters ¶ms) { 128 ATRACE_CALL(); 129 ALOGV("%s: Configuring ZSL streams", __FUNCTION__); 130 status_t res; 131 132 Mutex::Autolock l(mInputMutex); 133 134 sp<Camera2Client> client = mClient.promote(); 135 if (client == 0) { 136 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 137 return INVALID_OPERATION; 138 } 139 sp<Camera3Device> device = 140 static_cast<Camera3Device*>(client->getCameraDevice().get()); 141 if (device == 0) { 142 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 143 return INVALID_OPERATION; 144 } 145 146 if (mZslStreamId != NO_STREAM) { 147 // Check if stream parameters have to change 148 uint32_t currentWidth, currentHeight; 149 res = device->getStreamInfo(mZslStreamId, 150 ¤tWidth, ¤tHeight, 0); 151 if (res != OK) { 152 ALOGE("%s: Camera %d: Error querying capture output stream info: " 153 "%s (%d)", __FUNCTION__, 154 client->getCameraId(), strerror(-res), res); 155 return res; 156 } 157 if (currentWidth != (uint32_t)params.fastInfo.arrayWidth || 158 currentHeight != (uint32_t)params.fastInfo.arrayHeight) { 159 ALOGV("%s: Camera %d: Deleting stream %d since the buffer " 160 "dimensions changed", 161 __FUNCTION__, client->getCameraId(), mZslStreamId); 162 res = device->deleteStream(mZslStreamId); 163 if (res == -EBUSY) { 164 ALOGV("%s: Camera %d: Device is busy, call updateStream again " 165 " after it becomes idle", __FUNCTION__, mId); 166 return res; 167 } else if(res != OK) { 168 ALOGE("%s: Camera %d: Unable to delete old output stream " 169 "for ZSL: %s (%d)", __FUNCTION__, 170 client->getCameraId(), strerror(-res), res); 171 return res; 172 } 173 mZslStreamId = NO_STREAM; 174 } 175 } 176 177 if (mZslStreamId == NO_STREAM) { 178 // Create stream for HAL production 179 // TODO: Sort out better way to select resolution for ZSL 180 181 // Note that format specified internally in Camera3ZslStream 182 res = device->createZslStream( 183 params.fastInfo.arrayWidth, params.fastInfo.arrayHeight, 184 mBufferQueueDepth, 185 &mZslStreamId, 186 &mZslStream); 187 if (res != OK) { 188 ALOGE("%s: Camera %d: Can't create ZSL stream: " 189 "%s (%d)", __FUNCTION__, client->getCameraId(), 190 strerror(-res), res); 191 return res; 192 } 193 194 // Only add the camera3 buffer listener when the stream is created. 195 mZslStream->addBufferListener(this); 196 } 197 198 client->registerFrameListener(Camera2Client::kPreviewRequestIdStart, 199 Camera2Client::kPreviewRequestIdEnd, 200 this, 201 /*sendPartials*/false); 202 203 return OK; 204} 205 206status_t ZslProcessor3::deleteStream() { 207 ATRACE_CALL(); 208 status_t res; 209 210 Mutex::Autolock l(mInputMutex); 211 212 if (mZslStreamId != NO_STREAM) { 213 sp<Camera2Client> client = mClient.promote(); 214 if (client == 0) { 215 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 216 return INVALID_OPERATION; 217 } 218 219 sp<Camera3Device> device = 220 reinterpret_cast<Camera3Device*>(client->getCameraDevice().get()); 221 if (device == 0) { 222 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 223 return INVALID_OPERATION; 224 } 225 226 res = device->deleteStream(mZslStreamId); 227 if (res != OK) { 228 ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: " 229 "%s (%d)", __FUNCTION__, client->getCameraId(), 230 mZslStreamId, strerror(-res), res); 231 return res; 232 } 233 234 mZslStreamId = NO_STREAM; 235 } 236 return OK; 237} 238 239int ZslProcessor3::getStreamId() const { 240 Mutex::Autolock l(mInputMutex); 241 return mZslStreamId; 242} 243 244status_t ZslProcessor3::pushToReprocess(int32_t requestId) { 245 ALOGV("%s: Send in reprocess request with id %d", 246 __FUNCTION__, requestId); 247 Mutex::Autolock l(mInputMutex); 248 status_t res; 249 sp<Camera2Client> client = mClient.promote(); 250 251 if (client == 0) { 252 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 253 return INVALID_OPERATION; 254 } 255 256 IF_ALOGV() { 257 dumpZslQueue(-1); 258 } 259 260 size_t metadataIdx; 261 nsecs_t candidateTimestamp = getCandidateTimestampLocked(&metadataIdx); 262 263 if (candidateTimestamp == -1) { 264 ALOGE("%s: Could not find good candidate for ZSL reprocessing", 265 __FUNCTION__); 266 return NOT_ENOUGH_DATA; 267 } 268 269 res = mZslStream->enqueueInputBufferByTimestamp(candidateTimestamp, 270 /*actualTimestamp*/NULL); 271 272 if (res == mZslStream->NO_BUFFER_AVAILABLE) { 273 ALOGV("%s: No ZSL buffers yet", __FUNCTION__); 274 return NOT_ENOUGH_DATA; 275 } else if (res != OK) { 276 ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)", 277 __FUNCTION__, strerror(-res), res); 278 return res; 279 } 280 281 { 282 CameraMetadata request = mFrameList[metadataIdx]; 283 284 // Verify that the frame is reasonable for reprocessing 285 286 camera_metadata_entry_t entry; 287 entry = request.find(ANDROID_CONTROL_AE_STATE); 288 if (entry.count == 0) { 289 ALOGE("%s: ZSL queue frame has no AE state field!", 290 __FUNCTION__); 291 return BAD_VALUE; 292 } 293 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 294 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 295 ALOGV("%s: ZSL queue frame AE state is %d, need full capture", 296 __FUNCTION__, entry.data.u8[0]); 297 return NOT_ENOUGH_DATA; 298 } 299 300 uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS; 301 res = request.update(ANDROID_REQUEST_TYPE, 302 &requestType, 1); 303 if (res != OK) { 304 ALOGE("%s: Unable to update request type", 305 __FUNCTION__); 306 return INVALID_OPERATION; 307 } 308 309 int32_t inputStreams[1] = 310 { mZslStreamId }; 311 res = request.update(ANDROID_REQUEST_INPUT_STREAMS, 312 inputStreams, 1); 313 if (res != OK) { 314 ALOGE("%s: Unable to update request input streams", 315 __FUNCTION__); 316 return INVALID_OPERATION; 317 } 318 319 uint8_t captureIntent = 320 static_cast<uint8_t>(ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE); 321 res = request.update(ANDROID_CONTROL_CAPTURE_INTENT, 322 &captureIntent, 1); 323 if (res != OK ) { 324 ALOGE("%s: Unable to update request capture intent", 325 __FUNCTION__); 326 return INVALID_OPERATION; 327 } 328 329 // TODO: Shouldn't we also update the latest preview frame? 330 int32_t outputStreams[1] = 331 { client->getCaptureStreamId() }; 332 res = request.update(ANDROID_REQUEST_OUTPUT_STREAMS, 333 outputStreams, 1); 334 if (res != OK) { 335 ALOGE("%s: Unable to update request output streams", 336 __FUNCTION__); 337 return INVALID_OPERATION; 338 } 339 340 res = request.update(ANDROID_REQUEST_ID, 341 &requestId, 1); 342 if (res != OK ) { 343 ALOGE("%s: Unable to update frame to a reprocess request", 344 __FUNCTION__); 345 return INVALID_OPERATION; 346 } 347 348 res = client->stopStream(); 349 if (res != OK) { 350 ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: " 351 "%s (%d)", 352 __FUNCTION__, client->getCameraId(), strerror(-res), res); 353 return INVALID_OPERATION; 354 } 355 356 // Update JPEG settings 357 { 358 SharedParameters::Lock l(client->getParameters()); 359 res = l.mParameters.updateRequestJpeg(&request); 360 if (res != OK) { 361 ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL " 362 "capture request: %s (%d)", __FUNCTION__, 363 client->getCameraId(), 364 strerror(-res), res); 365 return res; 366 } 367 } 368 369 mLatestCapturedRequest = request; 370 res = client->getCameraDevice()->capture(request); 371 if (res != OK ) { 372 ALOGE("%s: Unable to send ZSL reprocess request to capture: %s" 373 " (%d)", __FUNCTION__, strerror(-res), res); 374 return res; 375 } 376 377 mState = LOCKED; 378 } 379 380 return OK; 381} 382 383status_t ZslProcessor3::clearZslQueue() { 384 Mutex::Autolock l(mInputMutex); 385 // If in middle of capture, can't clear out queue 386 if (mState == LOCKED) return OK; 387 388 return clearZslQueueLocked(); 389} 390 391status_t ZslProcessor3::clearZslQueueLocked() { 392 if (mZslStream != 0) { 393 // clear result metadata list first. 394 clearZslResultQueueLocked(); 395 return mZslStream->clearInputRingBuffer(); 396 } 397 return OK; 398} 399 400void ZslProcessor3::clearZslResultQueueLocked() { 401 mFrameList.clear(); 402 mFrameListHead = 0; 403 mFrameList.insertAt(0, mFrameListDepth); 404} 405 406void ZslProcessor3::dump(int fd, const Vector<String16>& /*args*/) const { 407 Mutex::Autolock l(mInputMutex); 408 if (!mLatestCapturedRequest.isEmpty()) { 409 String8 result(" Latest ZSL capture request:\n"); 410 write(fd, result.string(), result.size()); 411 mLatestCapturedRequest.dump(fd, 2, 6); 412 } else { 413 String8 result(" Latest ZSL capture request: none yet\n"); 414 write(fd, result.string(), result.size()); 415 } 416 dumpZslQueue(fd); 417} 418 419bool ZslProcessor3::threadLoop() { 420 // TODO: remove dependency on thread. For now, shut thread down right 421 // away. 422 return false; 423} 424 425void ZslProcessor3::dumpZslQueue(int fd) const { 426 String8 header("ZSL queue contents:"); 427 String8 indent(" "); 428 ALOGV("%s", header.string()); 429 if (fd != -1) { 430 header = indent + header + "\n"; 431 write(fd, header.string(), header.size()); 432 } 433 for (size_t i = 0; i < mZslQueue.size(); i++) { 434 const ZslPair &queueEntry = mZslQueue[i]; 435 nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp; 436 camera_metadata_ro_entry_t entry; 437 nsecs_t frameTimestamp = 0; 438 int frameAeState = -1; 439 if (!queueEntry.frame.isEmpty()) { 440 entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP); 441 if (entry.count > 0) frameTimestamp = entry.data.i64[0]; 442 entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE); 443 if (entry.count > 0) frameAeState = entry.data.u8[0]; 444 } 445 String8 result = 446 String8::format(" %zu: b: %" PRId64 "\tf: %" PRId64 ", AE state: %d", i, 447 bufferTimestamp, frameTimestamp, frameAeState); 448 ALOGV("%s", result.string()); 449 if (fd != -1) { 450 result = indent + result + "\n"; 451 write(fd, result.string(), result.size()); 452 } 453 454 } 455} 456 457nsecs_t ZslProcessor3::getCandidateTimestampLocked(size_t* metadataIdx) const { 458 /** 459 * Find the smallest timestamp we know about so far 460 * - ensure that aeState is either converged or locked 461 */ 462 463 size_t idx = 0; 464 nsecs_t minTimestamp = -1; 465 466 size_t emptyCount = mFrameList.size(); 467 468 for (size_t j = 0; j < mFrameList.size(); j++) { 469 const CameraMetadata &frame = mFrameList[j]; 470 if (!frame.isEmpty()) { 471 472 emptyCount--; 473 474 camera_metadata_ro_entry_t entry; 475 entry = frame.find(ANDROID_SENSOR_TIMESTAMP); 476 if (entry.count == 0) { 477 ALOGE("%s: Can't find timestamp in frame!", 478 __FUNCTION__); 479 continue; 480 } 481 nsecs_t frameTimestamp = entry.data.i64[0]; 482 if (minTimestamp > frameTimestamp || minTimestamp == -1) { 483 484 entry = frame.find(ANDROID_CONTROL_AE_STATE); 485 486 if (entry.count == 0) { 487 /** 488 * This is most likely a HAL bug. The aeState field is 489 * mandatory, so it should always be in a metadata packet. 490 */ 491 ALOGW("%s: ZSL queue frame has no AE state field!", 492 __FUNCTION__); 493 continue; 494 } 495 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 496 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 497 ALOGVV("%s: ZSL queue frame AE state is %d, need " 498 "full capture", __FUNCTION__, entry.data.u8[0]); 499 continue; 500 } 501 502 // Check AF state if device has focuser 503 if (mHasFocuser) { 504 // Make sure the candidate frame has good focus. 505 entry = frame.find(ANDROID_CONTROL_AF_STATE); 506 if (entry.count == 0) { 507 ALOGW("%s: ZSL queue frame has no AF state field!", 508 __FUNCTION__); 509 continue; 510 } 511 uint8_t afState = entry.data.u8[0]; 512 if (afState != ANDROID_CONTROL_AF_STATE_PASSIVE_FOCUSED && 513 afState != ANDROID_CONTROL_AF_STATE_FOCUSED_LOCKED && 514 afState != ANDROID_CONTROL_AF_STATE_NOT_FOCUSED_LOCKED) { 515 ALOGW("%s: ZSL queue frame AF state is %d is not good for capture, skip it", 516 __FUNCTION__, afState); 517 continue; 518 } 519 } 520 521 minTimestamp = frameTimestamp; 522 idx = j; 523 } 524 525 ALOGVV("%s: Saw timestamp %" PRId64, __FUNCTION__, frameTimestamp); 526 } 527 } 528 529 if (emptyCount == mFrameList.size()) { 530 /** 531 * This could be mildly bad and means our ZSL was triggered before 532 * there were any frames yet received by the camera framework. 533 * 534 * This is a fairly corner case which can happen under: 535 * + a user presses the shutter button real fast when the camera starts 536 * (startPreview followed immediately by takePicture). 537 * + burst capture case (hitting shutter button as fast possible) 538 * 539 * If this happens in steady case (preview running for a while, call 540 * a single takePicture) then this might be a fwk bug. 541 */ 542 ALOGW("%s: ZSL queue has no metadata frames", __FUNCTION__); 543 } 544 545 ALOGV("%s: Candidate timestamp %" PRId64 " (idx %zu), empty frames: %zu", 546 __FUNCTION__, minTimestamp, idx, emptyCount); 547 548 if (metadataIdx) { 549 *metadataIdx = idx; 550 } 551 552 return minTimestamp; 553} 554 555void ZslProcessor3::onBufferAcquired(const BufferInfo& /*bufferInfo*/) { 556 // Intentionally left empty 557 // Although theoretically we could use this to get better dump info 558} 559 560void ZslProcessor3::onBufferReleased(const BufferInfo& bufferInfo) { 561 Mutex::Autolock l(mInputMutex); 562 563 // ignore output buffers 564 if (bufferInfo.mOutput) { 565 return; 566 } 567 568 // TODO: Verify that the buffer is in our queue by looking at timestamp 569 // theoretically unnecessary unless we change the following assumptions: 570 // -- only 1 buffer reprocessed at a time (which is the case now) 571 572 // Erase entire ZSL queue since we've now completed the capture and preview 573 // is stopped. 574 // 575 // We need to guarantee that if we do two back-to-back captures, 576 // the second won't use a buffer that's older/the same as the first, which 577 // is theoretically possible if we don't clear out the queue and the 578 // selection criteria is something like 'newest'. Clearing out the result 579 // metadata queue on a completed capture ensures we'll only use new timestamp. 580 // Calling clearZslQueueLocked is a guaranteed deadlock because this callback 581 // holds the Camera3Stream internal lock (mLock), and clearZslQueueLocked requires 582 // to hold the same lock. 583 // TODO: need figure out a way to clear the Zsl buffer queue properly. Right now 584 // it is safe not to do so, as back to back ZSL capture requires stop and start 585 // preview, which will flush ZSL queue automatically. 586 ALOGV("%s: Memory optimization, clearing ZSL queue", 587 __FUNCTION__); 588 clearZslResultQueueLocked(); 589 590 // Required so we accept more ZSL requests 591 mState = RUNNING; 592} 593 594}; // namespace camera2 595}; // namespace android 596