ZslProcessor3.cpp revision 25a0aef19e170d2695f64b4c48296e7914155a88
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define LOG_TAG "Camera2-ZslProcessor3" 18#define ATRACE_TAG ATRACE_TAG_CAMERA 19//#define LOG_NDEBUG 0 20//#define LOG_NNDEBUG 0 21 22#ifdef LOG_NNDEBUG 23#define ALOGVV(...) ALOGV(__VA_ARGS__) 24#else 25#define ALOGVV(...) ((void)0) 26#endif 27 28#include <inttypes.h> 29 30#include <utils/Log.h> 31#include <utils/Trace.h> 32#include <gui/Surface.h> 33 34#include "common/CameraDeviceBase.h" 35#include "api1/Camera2Client.h" 36#include "api1/client2/CaptureSequencer.h" 37#include "api1/client2/ZslProcessor3.h" 38#include "device3/Camera3Device.h" 39 40namespace android { 41namespace camera2 { 42 43ZslProcessor3::ZslProcessor3( 44 sp<Camera2Client> client, 45 wp<CaptureSequencer> sequencer): 46 Thread(false), 47 mState(RUNNING), 48 mClient(client), 49 mSequencer(sequencer), 50 mId(client->getCameraId()), 51 mZslStreamId(NO_STREAM), 52 mFrameListHead(0), 53 mZslQueueHead(0), 54 mZslQueueTail(0) { 55 mZslQueue.insertAt(0, kZslBufferDepth); 56 mFrameList.insertAt(0, kFrameListDepth); 57 sp<CaptureSequencer> captureSequencer = mSequencer.promote(); 58 if (captureSequencer != 0) captureSequencer->setZslProcessor(this); 59} 60 61ZslProcessor3::~ZslProcessor3() { 62 ALOGV("%s: Exit", __FUNCTION__); 63 deleteStream(); 64} 65 66void ZslProcessor3::onResultAvailable(const CaptureResult &result) { 67 ATRACE_CALL(); 68 ALOGV("%s:", __FUNCTION__); 69 Mutex::Autolock l(mInputMutex); 70 camera_metadata_ro_entry_t entry; 71 entry = result.mMetadata.find(ANDROID_SENSOR_TIMESTAMP); 72 nsecs_t timestamp = entry.data.i64[0]; 73 (void)timestamp; 74 ALOGVV("Got preview metadata for timestamp %" PRId64, timestamp); 75 76 if (mState != RUNNING) return; 77 78 mFrameList.editItemAt(mFrameListHead) = result.mMetadata; 79 mFrameListHead = (mFrameListHead + 1) % kFrameListDepth; 80} 81 82status_t ZslProcessor3::updateStream(const Parameters ¶ms) { 83 ATRACE_CALL(); 84 ALOGV("%s: Configuring ZSL streams", __FUNCTION__); 85 status_t res; 86 87 Mutex::Autolock l(mInputMutex); 88 89 sp<Camera2Client> client = mClient.promote(); 90 if (client == 0) { 91 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 92 return INVALID_OPERATION; 93 } 94 sp<Camera3Device> device = 95 static_cast<Camera3Device*>(client->getCameraDevice().get()); 96 if (device == 0) { 97 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 98 return INVALID_OPERATION; 99 } 100 101 if (mZslStreamId != NO_STREAM) { 102 // Check if stream parameters have to change 103 uint32_t currentWidth, currentHeight; 104 res = device->getStreamInfo(mZslStreamId, 105 ¤tWidth, ¤tHeight, 0); 106 if (res != OK) { 107 ALOGE("%s: Camera %d: Error querying capture output stream info: " 108 "%s (%d)", __FUNCTION__, 109 client->getCameraId(), strerror(-res), res); 110 return res; 111 } 112 if (currentWidth != (uint32_t)params.fastInfo.arrayWidth || 113 currentHeight != (uint32_t)params.fastInfo.arrayHeight) { 114 ALOGV("%s: Camera %d: Deleting stream %d since the buffer " 115 "dimensions changed", 116 __FUNCTION__, client->getCameraId(), mZslStreamId); 117 res = device->deleteStream(mZslStreamId); 118 if (res == -EBUSY) { 119 ALOGV("%s: Camera %d: Device is busy, call updateStream again " 120 " after it becomes idle", __FUNCTION__, mId); 121 return res; 122 } else if(res != OK) { 123 ALOGE("%s: Camera %d: Unable to delete old output stream " 124 "for ZSL: %s (%d)", __FUNCTION__, 125 client->getCameraId(), strerror(-res), res); 126 return res; 127 } 128 mZslStreamId = NO_STREAM; 129 } 130 } 131 132 if (mZslStreamId == NO_STREAM) { 133 // Create stream for HAL production 134 // TODO: Sort out better way to select resolution for ZSL 135 136 // Note that format specified internally in Camera3ZslStream 137 res = device->createZslStream( 138 params.fastInfo.arrayWidth, params.fastInfo.arrayHeight, 139 kZslBufferDepth, 140 &mZslStreamId, 141 &mZslStream); 142 if (res != OK) { 143 ALOGE("%s: Camera %d: Can't create ZSL stream: " 144 "%s (%d)", __FUNCTION__, client->getCameraId(), 145 strerror(-res), res); 146 return res; 147 } 148 } 149 client->registerFrameListener(Camera2Client::kPreviewRequestIdStart, 150 Camera2Client::kPreviewRequestIdEnd, 151 this, 152 /*sendPartials*/false); 153 154 return OK; 155} 156 157status_t ZslProcessor3::deleteStream() { 158 ATRACE_CALL(); 159 status_t res; 160 161 Mutex::Autolock l(mInputMutex); 162 163 if (mZslStreamId != NO_STREAM) { 164 sp<Camera2Client> client = mClient.promote(); 165 if (client == 0) { 166 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 167 return INVALID_OPERATION; 168 } 169 170 sp<Camera3Device> device = 171 reinterpret_cast<Camera3Device*>(client->getCameraDevice().get()); 172 if (device == 0) { 173 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 174 return INVALID_OPERATION; 175 } 176 177 res = device->deleteStream(mZslStreamId); 178 if (res != OK) { 179 ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: " 180 "%s (%d)", __FUNCTION__, client->getCameraId(), 181 mZslStreamId, strerror(-res), res); 182 return res; 183 } 184 185 mZslStreamId = NO_STREAM; 186 } 187 return OK; 188} 189 190int ZslProcessor3::getStreamId() const { 191 Mutex::Autolock l(mInputMutex); 192 return mZslStreamId; 193} 194 195status_t ZslProcessor3::pushToReprocess(int32_t requestId) { 196 ALOGV("%s: Send in reprocess request with id %d", 197 __FUNCTION__, requestId); 198 Mutex::Autolock l(mInputMutex); 199 status_t res; 200 sp<Camera2Client> client = mClient.promote(); 201 202 if (client == 0) { 203 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 204 return INVALID_OPERATION; 205 } 206 207 IF_ALOGV() { 208 dumpZslQueue(-1); 209 } 210 211 size_t metadataIdx; 212 nsecs_t candidateTimestamp = getCandidateTimestampLocked(&metadataIdx); 213 214 if (candidateTimestamp == -1) { 215 ALOGE("%s: Could not find good candidate for ZSL reprocessing", 216 __FUNCTION__); 217 return NOT_ENOUGH_DATA; 218 } 219 220 res = mZslStream->enqueueInputBufferByTimestamp(candidateTimestamp, 221 /*actualTimestamp*/NULL); 222 223 if (res == mZslStream->NO_BUFFER_AVAILABLE) { 224 ALOGV("%s: No ZSL buffers yet", __FUNCTION__); 225 return NOT_ENOUGH_DATA; 226 } else if (res != OK) { 227 ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)", 228 __FUNCTION__, strerror(-res), res); 229 return res; 230 } 231 232 { 233 CameraMetadata request = mFrameList[metadataIdx]; 234 235 // Verify that the frame is reasonable for reprocessing 236 237 camera_metadata_entry_t entry; 238 entry = request.find(ANDROID_CONTROL_AE_STATE); 239 if (entry.count == 0) { 240 ALOGE("%s: ZSL queue frame has no AE state field!", 241 __FUNCTION__); 242 return BAD_VALUE; 243 } 244 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 245 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 246 ALOGV("%s: ZSL queue frame AE state is %d, need full capture", 247 __FUNCTION__, entry.data.u8[0]); 248 return NOT_ENOUGH_DATA; 249 } 250 251 uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS; 252 res = request.update(ANDROID_REQUEST_TYPE, 253 &requestType, 1); 254 int32_t inputStreams[1] = 255 { mZslStreamId }; 256 if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS, 257 inputStreams, 1); 258 // TODO: Shouldn't we also update the latest preview frame? 259 int32_t outputStreams[1] = 260 { client->getCaptureStreamId() }; 261 if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS, 262 outputStreams, 1); 263 res = request.update(ANDROID_REQUEST_ID, 264 &requestId, 1); 265 266 if (res != OK ) { 267 ALOGE("%s: Unable to update frame to a reprocess request", 268 __FUNCTION__); 269 return INVALID_OPERATION; 270 } 271 272 res = client->stopStream(); 273 if (res != OK) { 274 ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: " 275 "%s (%d)", 276 __FUNCTION__, client->getCameraId(), strerror(-res), res); 277 return INVALID_OPERATION; 278 } 279 280 // Flush device to clear out all in-flight requests pending in HAL. 281 res = client->getCameraDevice()->flush(); 282 if (res != OK) { 283 ALOGE("%s: Camera %d: Failed to flush device: " 284 "%s (%d)", 285 __FUNCTION__, client->getCameraId(), strerror(-res), res); 286 return res; 287 } 288 289 // Update JPEG settings 290 { 291 SharedParameters::Lock l(client->getParameters()); 292 res = l.mParameters.updateRequestJpeg(&request); 293 if (res != OK) { 294 ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL " 295 "capture request: %s (%d)", __FUNCTION__, 296 client->getCameraId(), 297 strerror(-res), res); 298 return res; 299 } 300 } 301 302 mLatestCapturedRequest = request; 303 res = client->getCameraDevice()->capture(request); 304 if (res != OK ) { 305 ALOGE("%s: Unable to send ZSL reprocess request to capture: %s" 306 " (%d)", __FUNCTION__, strerror(-res), res); 307 return res; 308 } 309 310 mState = LOCKED; 311 } 312 313 return OK; 314} 315 316status_t ZslProcessor3::clearZslQueue() { 317 Mutex::Autolock l(mInputMutex); 318 // If in middle of capture, can't clear out queue 319 if (mState == LOCKED) return OK; 320 321 return clearZslQueueLocked(); 322} 323 324status_t ZslProcessor3::clearZslQueueLocked() { 325 if (mZslStream != 0) { 326 return mZslStream->clearInputRingBuffer(); 327 } 328 return OK; 329} 330 331void ZslProcessor3::dump(int fd, const Vector<String16>& /*args*/) const { 332 Mutex::Autolock l(mInputMutex); 333 if (!mLatestCapturedRequest.isEmpty()) { 334 String8 result(" Latest ZSL capture request:\n"); 335 write(fd, result.string(), result.size()); 336 mLatestCapturedRequest.dump(fd, 2, 6); 337 } else { 338 String8 result(" Latest ZSL capture request: none yet\n"); 339 write(fd, result.string(), result.size()); 340 } 341 dumpZslQueue(fd); 342} 343 344bool ZslProcessor3::threadLoop() { 345 // TODO: remove dependency on thread. For now, shut thread down right 346 // away. 347 return false; 348} 349 350void ZslProcessor3::dumpZslQueue(int fd) const { 351 String8 header("ZSL queue contents:"); 352 String8 indent(" "); 353 ALOGV("%s", header.string()); 354 if (fd != -1) { 355 header = indent + header + "\n"; 356 write(fd, header.string(), header.size()); 357 } 358 for (size_t i = 0; i < mZslQueue.size(); i++) { 359 const ZslPair &queueEntry = mZslQueue[i]; 360 nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp; 361 camera_metadata_ro_entry_t entry; 362 nsecs_t frameTimestamp = 0; 363 int frameAeState = -1; 364 if (!queueEntry.frame.isEmpty()) { 365 entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP); 366 if (entry.count > 0) frameTimestamp = entry.data.i64[0]; 367 entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE); 368 if (entry.count > 0) frameAeState = entry.data.u8[0]; 369 } 370 String8 result = 371 String8::format(" %zu: b: %" PRId64 "\tf: %" PRId64 ", AE state: %d", i, 372 bufferTimestamp, frameTimestamp, frameAeState); 373 ALOGV("%s", result.string()); 374 if (fd != -1) { 375 result = indent + result + "\n"; 376 write(fd, result.string(), result.size()); 377 } 378 379 } 380} 381 382nsecs_t ZslProcessor3::getCandidateTimestampLocked(size_t* metadataIdx) const { 383 /** 384 * Find the smallest timestamp we know about so far 385 * - ensure that aeState is either converged or locked 386 */ 387 388 size_t idx = 0; 389 nsecs_t minTimestamp = -1; 390 391 size_t emptyCount = mFrameList.size(); 392 393 for (size_t j = 0; j < mFrameList.size(); j++) { 394 const CameraMetadata &frame = mFrameList[j]; 395 if (!frame.isEmpty()) { 396 397 emptyCount--; 398 399 camera_metadata_ro_entry_t entry; 400 entry = frame.find(ANDROID_SENSOR_TIMESTAMP); 401 if (entry.count == 0) { 402 ALOGE("%s: Can't find timestamp in frame!", 403 __FUNCTION__); 404 continue; 405 } 406 nsecs_t frameTimestamp = entry.data.i64[0]; 407 if (minTimestamp > frameTimestamp || minTimestamp == -1) { 408 409 entry = frame.find(ANDROID_CONTROL_AE_STATE); 410 411 if (entry.count == 0) { 412 /** 413 * This is most likely a HAL bug. The aeState field is 414 * mandatory, so it should always be in a metadata packet. 415 */ 416 ALOGW("%s: ZSL queue frame has no AE state field!", 417 __FUNCTION__); 418 continue; 419 } 420 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 421 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 422 ALOGVV("%s: ZSL queue frame AE state is %d, need " 423 "full capture", __FUNCTION__, entry.data.u8[0]); 424 continue; 425 } 426 427 minTimestamp = frameTimestamp; 428 idx = j; 429 } 430 431 ALOGVV("%s: Saw timestamp %" PRId64, __FUNCTION__, frameTimestamp); 432 } 433 } 434 435 if (emptyCount == mFrameList.size()) { 436 /** 437 * This could be mildly bad and means our ZSL was triggered before 438 * there were any frames yet received by the camera framework. 439 * 440 * This is a fairly corner case which can happen under: 441 * + a user presses the shutter button real fast when the camera starts 442 * (startPreview followed immediately by takePicture). 443 * + burst capture case (hitting shutter button as fast possible) 444 * 445 * If this happens in steady case (preview running for a while, call 446 * a single takePicture) then this might be a fwk bug. 447 */ 448 ALOGW("%s: ZSL queue has no metadata frames", __FUNCTION__); 449 } 450 451 ALOGV("%s: Candidate timestamp %" PRId64 " (idx %zu), empty frames: %zu", 452 __FUNCTION__, minTimestamp, idx, emptyCount); 453 454 if (metadataIdx) { 455 *metadataIdx = idx; 456 } 457 458 return minTimestamp; 459} 460 461void ZslProcessor3::onBufferAcquired(const BufferInfo& /*bufferInfo*/) { 462 // Intentionally left empty 463 // Although theoretically we could use this to get better dump info 464} 465 466void ZslProcessor3::onBufferReleased(const BufferInfo& bufferInfo) { 467 Mutex::Autolock l(mInputMutex); 468 469 // ignore output buffers 470 if (bufferInfo.mOutput) { 471 return; 472 } 473 474 // TODO: Verify that the buffer is in our queue by looking at timestamp 475 // theoretically unnecessary unless we change the following assumptions: 476 // -- only 1 buffer reprocessed at a time (which is the case now) 477 478 // Erase entire ZSL queue since we've now completed the capture and preview 479 // is stopped. 480 // 481 // We need to guarantee that if we do two back-to-back captures, 482 // the second won't use a buffer that's older/the same as the first, which 483 // is theoretically possible if we don't clear out the queue and the 484 // selection criteria is something like 'newest'. Clearing out the queue 485 // on a completed capture ensures we'll only use new data. 486 ALOGV("%s: Memory optimization, clearing ZSL queue", 487 __FUNCTION__); 488 clearZslQueueLocked(); 489 490 // Required so we accept more ZSL requests 491 mState = RUNNING; 492} 493 494}; // namespace camera2 495}; // namespace android 496