ZslProcessor3.cpp revision 638794db70e692a0f5a6094809be0deedc0f94f4
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define LOG_TAG "Camera2-ZslProcessor3" 18#define ATRACE_TAG ATRACE_TAG_CAMERA 19//#define LOG_NDEBUG 0 20//#define LOG_NNDEBUG 0 21 22#ifdef LOG_NNDEBUG 23#define ALOGVV(...) ALOGV(__VA_ARGS__) 24#else 25#define ALOGVV(...) ((void)0) 26#endif 27 28#include <inttypes.h> 29 30#include <utils/Log.h> 31#include <utils/Trace.h> 32#include <gui/Surface.h> 33 34#include "common/CameraDeviceBase.h" 35#include "api1/Camera2Client.h" 36#include "api1/client2/CaptureSequencer.h" 37#include "api1/client2/ZslProcessor3.h" 38#include "device3/Camera3Device.h" 39 40namespace android { 41namespace camera2 { 42 43ZslProcessor3::ZslProcessor3( 44 sp<Camera2Client> client, 45 wp<CaptureSequencer> sequencer): 46 Thread(false), 47 mState(RUNNING), 48 mClient(client), 49 mSequencer(sequencer), 50 mId(client->getCameraId()), 51 mZslStreamId(NO_STREAM), 52 mFrameListHead(0), 53 mZslQueueHead(0), 54 mZslQueueTail(0), 55 mHasFocuser(false) { 56 // Initialize buffer queue and frame list based on pipeline max depth. 57 size_t pipelineMaxDepth = kDefaultMaxPipelineDepth; 58 if (client != 0) { 59 sp<Camera3Device> device = 60 static_cast<Camera3Device*>(client->getCameraDevice().get()); 61 if (device != 0) { 62 camera_metadata_ro_entry_t entry = 63 device->info().find(ANDROID_REQUEST_PIPELINE_MAX_DEPTH); 64 if (entry.count == 1) { 65 pipelineMaxDepth = entry.data.u8[0]; 66 } else { 67 ALOGW("%s: Unable to find the android.request.pipelineMaxDepth," 68 " use default pipeline max depth %zu", __FUNCTION__, 69 kDefaultMaxPipelineDepth); 70 } 71 72 entry = device->info().find(ANDROID_LENS_INFO_MINIMUM_FOCUS_DISTANCE); 73 if (entry.count > 0 && entry.data.f[0] != 0.) { 74 mHasFocuser = true; 75 } 76 } 77 } 78 79 ALOGV("%s: Initialize buffer queue and frame list depth based on max pipeline depth (%d)", 80 __FUNCTION__, pipelineMaxDepth); 81 mBufferQueueDepth = pipelineMaxDepth + 1; 82 mFrameListDepth = pipelineMaxDepth + 1; 83 84 mZslQueue.insertAt(0, mBufferQueueDepth); 85 mFrameList.insertAt(0, mFrameListDepth); 86 sp<CaptureSequencer> captureSequencer = mSequencer.promote(); 87 if (captureSequencer != 0) captureSequencer->setZslProcessor(this); 88} 89 90ZslProcessor3::~ZslProcessor3() { 91 ALOGV("%s: Exit", __FUNCTION__); 92 deleteStream(); 93} 94 95void ZslProcessor3::onResultAvailable(const CaptureResult &result) { 96 ATRACE_CALL(); 97 ALOGV("%s:", __FUNCTION__); 98 Mutex::Autolock l(mInputMutex); 99 camera_metadata_ro_entry_t entry; 100 entry = result.mMetadata.find(ANDROID_SENSOR_TIMESTAMP); 101 nsecs_t timestamp = entry.data.i64[0]; 102 if (entry.count == 0) { 103 ALOGE("%s: metadata doesn't have timestamp, skip this result", __FUNCTION__); 104 return; 105 } 106 (void)timestamp; 107 108 entry = result.mMetadata.find(ANDROID_REQUEST_FRAME_COUNT); 109 if (entry.count == 0) { 110 ALOGE("%s: metadata doesn't have frame number, skip this result", __FUNCTION__); 111 return; 112 } 113 int32_t frameNumber = entry.data.i32[0]; 114 115 ALOGVV("Got preview metadata for frame %d with timestamp %" PRId64, frameNumber, timestamp); 116 117 if (mState != RUNNING) return; 118 119 mFrameList.editItemAt(mFrameListHead) = result.mMetadata; 120 mFrameListHead = (mFrameListHead + 1) % mFrameListDepth; 121} 122 123status_t ZslProcessor3::updateStream(const Parameters ¶ms) { 124 ATRACE_CALL(); 125 ALOGV("%s: Configuring ZSL streams", __FUNCTION__); 126 status_t res; 127 128 Mutex::Autolock l(mInputMutex); 129 130 sp<Camera2Client> client = mClient.promote(); 131 if (client == 0) { 132 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 133 return INVALID_OPERATION; 134 } 135 sp<Camera3Device> device = 136 static_cast<Camera3Device*>(client->getCameraDevice().get()); 137 if (device == 0) { 138 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 139 return INVALID_OPERATION; 140 } 141 142 if (mZslStreamId != NO_STREAM) { 143 // Check if stream parameters have to change 144 uint32_t currentWidth, currentHeight; 145 res = device->getStreamInfo(mZslStreamId, 146 ¤tWidth, ¤tHeight, 0); 147 if (res != OK) { 148 ALOGE("%s: Camera %d: Error querying capture output stream info: " 149 "%s (%d)", __FUNCTION__, 150 client->getCameraId(), strerror(-res), res); 151 return res; 152 } 153 if (currentWidth != (uint32_t)params.fastInfo.arrayWidth || 154 currentHeight != (uint32_t)params.fastInfo.arrayHeight) { 155 ALOGV("%s: Camera %d: Deleting stream %d since the buffer " 156 "dimensions changed", 157 __FUNCTION__, client->getCameraId(), mZslStreamId); 158 res = device->deleteStream(mZslStreamId); 159 if (res == -EBUSY) { 160 ALOGV("%s: Camera %d: Device is busy, call updateStream again " 161 " after it becomes idle", __FUNCTION__, mId); 162 return res; 163 } else if(res != OK) { 164 ALOGE("%s: Camera %d: Unable to delete old output stream " 165 "for ZSL: %s (%d)", __FUNCTION__, 166 client->getCameraId(), strerror(-res), res); 167 return res; 168 } 169 mZslStreamId = NO_STREAM; 170 } 171 } 172 173 if (mZslStreamId == NO_STREAM) { 174 // Create stream for HAL production 175 // TODO: Sort out better way to select resolution for ZSL 176 177 // Note that format specified internally in Camera3ZslStream 178 res = device->createZslStream( 179 params.fastInfo.arrayWidth, params.fastInfo.arrayHeight, 180 mBufferQueueDepth, 181 &mZslStreamId, 182 &mZslStream); 183 if (res != OK) { 184 ALOGE("%s: Camera %d: Can't create ZSL stream: " 185 "%s (%d)", __FUNCTION__, client->getCameraId(), 186 strerror(-res), res); 187 return res; 188 } 189 190 // Only add the camera3 buffer listener when the stream is created. 191 mZslStream->addBufferListener(this); 192 } 193 194 client->registerFrameListener(Camera2Client::kPreviewRequestIdStart, 195 Camera2Client::kPreviewRequestIdEnd, 196 this, 197 /*sendPartials*/false); 198 199 return OK; 200} 201 202status_t ZslProcessor3::deleteStream() { 203 ATRACE_CALL(); 204 status_t res; 205 206 Mutex::Autolock l(mInputMutex); 207 208 if (mZslStreamId != NO_STREAM) { 209 sp<Camera2Client> client = mClient.promote(); 210 if (client == 0) { 211 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 212 return INVALID_OPERATION; 213 } 214 215 sp<Camera3Device> device = 216 reinterpret_cast<Camera3Device*>(client->getCameraDevice().get()); 217 if (device == 0) { 218 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 219 return INVALID_OPERATION; 220 } 221 222 res = device->deleteStream(mZslStreamId); 223 if (res != OK) { 224 ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: " 225 "%s (%d)", __FUNCTION__, client->getCameraId(), 226 mZslStreamId, strerror(-res), res); 227 return res; 228 } 229 230 mZslStreamId = NO_STREAM; 231 } 232 return OK; 233} 234 235int ZslProcessor3::getStreamId() const { 236 Mutex::Autolock l(mInputMutex); 237 return mZslStreamId; 238} 239 240status_t ZslProcessor3::pushToReprocess(int32_t requestId) { 241 ALOGV("%s: Send in reprocess request with id %d", 242 __FUNCTION__, requestId); 243 Mutex::Autolock l(mInputMutex); 244 status_t res; 245 sp<Camera2Client> client = mClient.promote(); 246 247 if (client == 0) { 248 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 249 return INVALID_OPERATION; 250 } 251 252 IF_ALOGV() { 253 dumpZslQueue(-1); 254 } 255 256 size_t metadataIdx; 257 nsecs_t candidateTimestamp = getCandidateTimestampLocked(&metadataIdx); 258 259 if (candidateTimestamp == -1) { 260 ALOGE("%s: Could not find good candidate for ZSL reprocessing", 261 __FUNCTION__); 262 return NOT_ENOUGH_DATA; 263 } 264 265 res = mZslStream->enqueueInputBufferByTimestamp(candidateTimestamp, 266 /*actualTimestamp*/NULL); 267 268 if (res == mZslStream->NO_BUFFER_AVAILABLE) { 269 ALOGV("%s: No ZSL buffers yet", __FUNCTION__); 270 return NOT_ENOUGH_DATA; 271 } else if (res != OK) { 272 ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)", 273 __FUNCTION__, strerror(-res), res); 274 return res; 275 } 276 277 { 278 CameraMetadata request = mFrameList[metadataIdx]; 279 280 // Verify that the frame is reasonable for reprocessing 281 282 camera_metadata_entry_t entry; 283 entry = request.find(ANDROID_CONTROL_AE_STATE); 284 if (entry.count == 0) { 285 ALOGE("%s: ZSL queue frame has no AE state field!", 286 __FUNCTION__); 287 return BAD_VALUE; 288 } 289 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 290 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 291 ALOGV("%s: ZSL queue frame AE state is %d, need full capture", 292 __FUNCTION__, entry.data.u8[0]); 293 return NOT_ENOUGH_DATA; 294 } 295 296 uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS; 297 res = request.update(ANDROID_REQUEST_TYPE, 298 &requestType, 1); 299 if (res != OK) { 300 ALOGE("%s: Unable to update request type", 301 __FUNCTION__); 302 return INVALID_OPERATION; 303 } 304 305 int32_t inputStreams[1] = 306 { mZslStreamId }; 307 res = request.update(ANDROID_REQUEST_INPUT_STREAMS, 308 inputStreams, 1); 309 if (res != OK) { 310 ALOGE("%s: Unable to update request input streams", 311 __FUNCTION__); 312 return INVALID_OPERATION; 313 } 314 315 uint8_t captureIntent = 316 static_cast<uint8_t>(ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE); 317 res = request.update(ANDROID_CONTROL_CAPTURE_INTENT, 318 &captureIntent, 1); 319 if (res != OK ) { 320 ALOGE("%s: Unable to update request capture intent", 321 __FUNCTION__); 322 return INVALID_OPERATION; 323 } 324 325 // TODO: Shouldn't we also update the latest preview frame? 326 int32_t outputStreams[1] = 327 { client->getCaptureStreamId() }; 328 res = request.update(ANDROID_REQUEST_OUTPUT_STREAMS, 329 outputStreams, 1); 330 if (res != OK) { 331 ALOGE("%s: Unable to update request output streams", 332 __FUNCTION__); 333 return INVALID_OPERATION; 334 } 335 336 res = request.update(ANDROID_REQUEST_ID, 337 &requestId, 1); 338 if (res != OK ) { 339 ALOGE("%s: Unable to update frame to a reprocess request", 340 __FUNCTION__); 341 return INVALID_OPERATION; 342 } 343 344 res = client->stopStream(); 345 if (res != OK) { 346 ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: " 347 "%s (%d)", 348 __FUNCTION__, client->getCameraId(), strerror(-res), res); 349 return INVALID_OPERATION; 350 } 351 352 // Update JPEG settings 353 { 354 SharedParameters::Lock l(client->getParameters()); 355 res = l.mParameters.updateRequestJpeg(&request); 356 if (res != OK) { 357 ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL " 358 "capture request: %s (%d)", __FUNCTION__, 359 client->getCameraId(), 360 strerror(-res), res); 361 return res; 362 } 363 } 364 365 mLatestCapturedRequest = request; 366 res = client->getCameraDevice()->capture(request); 367 if (res != OK ) { 368 ALOGE("%s: Unable to send ZSL reprocess request to capture: %s" 369 " (%d)", __FUNCTION__, strerror(-res), res); 370 return res; 371 } 372 373 mState = LOCKED; 374 } 375 376 return OK; 377} 378 379status_t ZslProcessor3::clearZslQueue() { 380 Mutex::Autolock l(mInputMutex); 381 // If in middle of capture, can't clear out queue 382 if (mState == LOCKED) return OK; 383 384 return clearZslQueueLocked(); 385} 386 387status_t ZslProcessor3::clearZslQueueLocked() { 388 if (mZslStream != 0) { 389 // clear result metadata list first. 390 clearZslResultQueueLocked(); 391 return mZslStream->clearInputRingBuffer(); 392 } 393 return OK; 394} 395 396void ZslProcessor3::clearZslResultQueueLocked() { 397 mFrameList.clear(); 398 mFrameListHead = 0; 399 mFrameList.insertAt(0, mFrameListDepth); 400} 401 402void ZslProcessor3::dump(int fd, const Vector<String16>& /*args*/) const { 403 Mutex::Autolock l(mInputMutex); 404 if (!mLatestCapturedRequest.isEmpty()) { 405 String8 result(" Latest ZSL capture request:\n"); 406 write(fd, result.string(), result.size()); 407 mLatestCapturedRequest.dump(fd, 2, 6); 408 } else { 409 String8 result(" Latest ZSL capture request: none yet\n"); 410 write(fd, result.string(), result.size()); 411 } 412 dumpZslQueue(fd); 413} 414 415bool ZslProcessor3::threadLoop() { 416 // TODO: remove dependency on thread. For now, shut thread down right 417 // away. 418 return false; 419} 420 421void ZslProcessor3::dumpZslQueue(int fd) const { 422 String8 header("ZSL queue contents:"); 423 String8 indent(" "); 424 ALOGV("%s", header.string()); 425 if (fd != -1) { 426 header = indent + header + "\n"; 427 write(fd, header.string(), header.size()); 428 } 429 for (size_t i = 0; i < mZslQueue.size(); i++) { 430 const ZslPair &queueEntry = mZslQueue[i]; 431 nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp; 432 camera_metadata_ro_entry_t entry; 433 nsecs_t frameTimestamp = 0; 434 int frameAeState = -1; 435 if (!queueEntry.frame.isEmpty()) { 436 entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP); 437 if (entry.count > 0) frameTimestamp = entry.data.i64[0]; 438 entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE); 439 if (entry.count > 0) frameAeState = entry.data.u8[0]; 440 } 441 String8 result = 442 String8::format(" %zu: b: %" PRId64 "\tf: %" PRId64 ", AE state: %d", i, 443 bufferTimestamp, frameTimestamp, frameAeState); 444 ALOGV("%s", result.string()); 445 if (fd != -1) { 446 result = indent + result + "\n"; 447 write(fd, result.string(), result.size()); 448 } 449 450 } 451} 452 453nsecs_t ZslProcessor3::getCandidateTimestampLocked(size_t* metadataIdx) const { 454 /** 455 * Find the smallest timestamp we know about so far 456 * - ensure that aeState is either converged or locked 457 */ 458 459 size_t idx = 0; 460 nsecs_t minTimestamp = -1; 461 462 size_t emptyCount = mFrameList.size(); 463 464 for (size_t j = 0; j < mFrameList.size(); j++) { 465 const CameraMetadata &frame = mFrameList[j]; 466 if (!frame.isEmpty()) { 467 468 emptyCount--; 469 470 camera_metadata_ro_entry_t entry; 471 entry = frame.find(ANDROID_SENSOR_TIMESTAMP); 472 if (entry.count == 0) { 473 ALOGE("%s: Can't find timestamp in frame!", 474 __FUNCTION__); 475 continue; 476 } 477 nsecs_t frameTimestamp = entry.data.i64[0]; 478 if (minTimestamp > frameTimestamp || minTimestamp == -1) { 479 480 entry = frame.find(ANDROID_CONTROL_AE_STATE); 481 482 if (entry.count == 0) { 483 /** 484 * This is most likely a HAL bug. The aeState field is 485 * mandatory, so it should always be in a metadata packet. 486 */ 487 ALOGW("%s: ZSL queue frame has no AE state field!", 488 __FUNCTION__); 489 continue; 490 } 491 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 492 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 493 ALOGVV("%s: ZSL queue frame AE state is %d, need " 494 "full capture", __FUNCTION__, entry.data.u8[0]); 495 continue; 496 } 497 498 // Check AF state if device has focuser 499 if (mHasFocuser) { 500 // Make sure the candidate frame has good focus. 501 entry = frame.find(ANDROID_CONTROL_AF_STATE); 502 if (entry.count == 0) { 503 ALOGW("%s: ZSL queue frame has no AF state field!", 504 __FUNCTION__); 505 continue; 506 } 507 uint8_t afState = entry.data.u8[0]; 508 if (afState != ANDROID_CONTROL_AF_STATE_PASSIVE_FOCUSED && 509 afState != ANDROID_CONTROL_AF_STATE_FOCUSED_LOCKED && 510 afState != ANDROID_CONTROL_AF_STATE_NOT_FOCUSED_LOCKED) { 511 ALOGW("%s: ZSL queue frame AF state is %d is not good for capture, skip it", 512 __FUNCTION__, afState); 513 continue; 514 } 515 } 516 517 minTimestamp = frameTimestamp; 518 idx = j; 519 } 520 521 ALOGVV("%s: Saw timestamp %" PRId64, __FUNCTION__, frameTimestamp); 522 } 523 } 524 525 if (emptyCount == mFrameList.size()) { 526 /** 527 * This could be mildly bad and means our ZSL was triggered before 528 * there were any frames yet received by the camera framework. 529 * 530 * This is a fairly corner case which can happen under: 531 * + a user presses the shutter button real fast when the camera starts 532 * (startPreview followed immediately by takePicture). 533 * + burst capture case (hitting shutter button as fast possible) 534 * 535 * If this happens in steady case (preview running for a while, call 536 * a single takePicture) then this might be a fwk bug. 537 */ 538 ALOGW("%s: ZSL queue has no metadata frames", __FUNCTION__); 539 } 540 541 ALOGV("%s: Candidate timestamp %" PRId64 " (idx %zu), empty frames: %zu", 542 __FUNCTION__, minTimestamp, idx, emptyCount); 543 544 if (metadataIdx) { 545 *metadataIdx = idx; 546 } 547 548 return minTimestamp; 549} 550 551void ZslProcessor3::onBufferAcquired(const BufferInfo& /*bufferInfo*/) { 552 // Intentionally left empty 553 // Although theoretically we could use this to get better dump info 554} 555 556void ZslProcessor3::onBufferReleased(const BufferInfo& bufferInfo) { 557 558 // ignore output buffers 559 if (bufferInfo.mOutput) { 560 return; 561 } 562 563 // Lock mutex only once we know this is an input buffer returned to avoid 564 // potential deadlock 565 Mutex::Autolock l(mInputMutex); 566 // TODO: Verify that the buffer is in our queue by looking at timestamp 567 // theoretically unnecessary unless we change the following assumptions: 568 // -- only 1 buffer reprocessed at a time (which is the case now) 569 570 // Erase entire ZSL queue since we've now completed the capture and preview 571 // is stopped. 572 // 573 // We need to guarantee that if we do two back-to-back captures, 574 // the second won't use a buffer that's older/the same as the first, which 575 // is theoretically possible if we don't clear out the queue and the 576 // selection criteria is something like 'newest'. Clearing out the result 577 // metadata queue on a completed capture ensures we'll only use new timestamp. 578 // Calling clearZslQueueLocked is a guaranteed deadlock because this callback 579 // holds the Camera3Stream internal lock (mLock), and clearZslQueueLocked requires 580 // to hold the same lock. 581 // TODO: need figure out a way to clear the Zsl buffer queue properly. Right now 582 // it is safe not to do so, as back to back ZSL capture requires stop and start 583 // preview, which will flush ZSL queue automatically. 584 ALOGV("%s: Memory optimization, clearing ZSL queue", 585 __FUNCTION__); 586 clearZslResultQueueLocked(); 587 588 // Required so we accept more ZSL requests 589 mState = RUNNING; 590} 591 592}; // namespace camera2 593}; // namespace android 594