ZslProcessor3.cpp revision 491e341211b4772c75f719158f6b397e1c40497d
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define LOG_TAG "Camera2-ZslProcessor3" 18#define ATRACE_TAG ATRACE_TAG_CAMERA 19//#define LOG_NDEBUG 0 20//#define LOG_NNDEBUG 0 21 22#ifdef LOG_NNDEBUG 23#define ALOGVV(...) ALOGV(__VA_ARGS__) 24#else 25#define ALOGVV(...) ((void)0) 26#endif 27 28#include <utils/Log.h> 29#include <utils/Trace.h> 30#include <gui/Surface.h> 31 32#include "common/CameraDeviceBase.h" 33#include "api1/Camera2Client.h" 34#include "api1/client2/CaptureSequencer.h" 35#include "api1/client2/ZslProcessor3.h" 36#include "device3/Camera3Device.h" 37 38namespace android { 39namespace camera2 { 40 41ZslProcessor3::ZslProcessor3( 42 sp<Camera2Client> client, 43 wp<CaptureSequencer> sequencer): 44 Thread(false), 45 mState(RUNNING), 46 mClient(client), 47 mSequencer(sequencer), 48 mId(client->getCameraId()), 49 mZslStreamId(NO_STREAM), 50 mFrameListHead(0), 51 mZslQueueHead(0), 52 mZslQueueTail(0) { 53 mZslQueue.insertAt(0, kZslBufferDepth); 54 mFrameList.insertAt(0, kFrameListDepth); 55 sp<CaptureSequencer> captureSequencer = mSequencer.promote(); 56 if (captureSequencer != 0) captureSequencer->setZslProcessor(this); 57} 58 59ZslProcessor3::~ZslProcessor3() { 60 ALOGV("%s: Exit", __FUNCTION__); 61 deleteStream(); 62} 63 64void ZslProcessor3::onFrameAvailable(int32_t /*requestId*/, 65 const CameraMetadata &frame) { 66 Mutex::Autolock l(mInputMutex); 67 camera_metadata_ro_entry_t entry; 68 entry = frame.find(ANDROID_SENSOR_TIMESTAMP); 69 nsecs_t timestamp = entry.data.i64[0]; 70 (void)timestamp; 71 ALOGVV("Got preview metadata for timestamp %lld", timestamp); 72 73 if (mState != RUNNING) return; 74 75 mFrameList.editItemAt(mFrameListHead) = frame; 76 mFrameListHead = (mFrameListHead + 1) % kFrameListDepth; 77} 78 79status_t ZslProcessor3::updateStream(const Parameters ¶ms) { 80 ATRACE_CALL(); 81 ALOGV("%s: Configuring ZSL streams", __FUNCTION__); 82 status_t res; 83 84 Mutex::Autolock l(mInputMutex); 85 86 sp<Camera2Client> client = mClient.promote(); 87 if (client == 0) { 88 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 89 return INVALID_OPERATION; 90 } 91 sp<Camera3Device> device = 92 static_cast<Camera3Device*>(client->getCameraDevice().get()); 93 if (device == 0) { 94 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 95 return INVALID_OPERATION; 96 } 97 98 if (mZslStreamId != NO_STREAM) { 99 // Check if stream parameters have to change 100 uint32_t currentWidth, currentHeight; 101 res = device->getStreamInfo(mZslStreamId, 102 ¤tWidth, ¤tHeight, 0); 103 if (res != OK) { 104 ALOGE("%s: Camera %d: Error querying capture output stream info: " 105 "%s (%d)", __FUNCTION__, 106 client->getCameraId(), strerror(-res), res); 107 return res; 108 } 109 if (currentWidth != (uint32_t)params.fastInfo.arrayWidth || 110 currentHeight != (uint32_t)params.fastInfo.arrayHeight) { 111 ALOGV("%s: Camera %d: Deleting stream %d since the buffer " 112 "dimensions changed", 113 __FUNCTION__, client->getCameraId(), mZslStreamId); 114 res = device->deleteStream(mZslStreamId); 115 if (res == -EBUSY) { 116 ALOGV("%s: Camera %d: Device is busy, call updateStream again " 117 " after it becomes idle", __FUNCTION__, mId); 118 return res; 119 } else if(res != OK) { 120 ALOGE("%s: Camera %d: Unable to delete old output stream " 121 "for ZSL: %s (%d)", __FUNCTION__, 122 client->getCameraId(), strerror(-res), res); 123 return res; 124 } 125 mZslStreamId = NO_STREAM; 126 } 127 } 128 129 if (mZslStreamId == NO_STREAM) { 130 // Create stream for HAL production 131 // TODO: Sort out better way to select resolution for ZSL 132 133 // Note that format specified internally in Camera3ZslStream 134 res = device->createZslStream( 135 params.fastInfo.arrayWidth, params.fastInfo.arrayHeight, 136 kZslBufferDepth, 137 &mZslStreamId, 138 &mZslStream); 139 if (res != OK) { 140 ALOGE("%s: Camera %d: Can't create ZSL stream: " 141 "%s (%d)", __FUNCTION__, client->getCameraId(), 142 strerror(-res), res); 143 return res; 144 } 145 } 146 client->registerFrameListener(Camera2Client::kPreviewRequestIdStart, 147 Camera2Client::kPreviewRequestIdEnd, 148 this); 149 150 return OK; 151} 152 153status_t ZslProcessor3::deleteStream() { 154 ATRACE_CALL(); 155 status_t res; 156 157 Mutex::Autolock l(mInputMutex); 158 159 if (mZslStreamId != NO_STREAM) { 160 sp<Camera2Client> client = mClient.promote(); 161 if (client == 0) { 162 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 163 return INVALID_OPERATION; 164 } 165 166 sp<Camera3Device> device = 167 reinterpret_cast<Camera3Device*>(client->getCameraDevice().get()); 168 if (device == 0) { 169 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 170 return INVALID_OPERATION; 171 } 172 173 res = device->deleteStream(mZslStreamId); 174 if (res != OK) { 175 ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: " 176 "%s (%d)", __FUNCTION__, client->getCameraId(), 177 mZslStreamId, strerror(-res), res); 178 return res; 179 } 180 181 mZslStreamId = NO_STREAM; 182 } 183 return OK; 184} 185 186int ZslProcessor3::getStreamId() const { 187 Mutex::Autolock l(mInputMutex); 188 return mZslStreamId; 189} 190 191status_t ZslProcessor3::pushToReprocess(int32_t requestId) { 192 ALOGV("%s: Send in reprocess request with id %d", 193 __FUNCTION__, requestId); 194 Mutex::Autolock l(mInputMutex); 195 status_t res; 196 sp<Camera2Client> client = mClient.promote(); 197 198 if (client == 0) { 199 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 200 return INVALID_OPERATION; 201 } 202 203 IF_ALOGV() { 204 dumpZslQueue(-1); 205 } 206 207 size_t metadataIdx; 208 nsecs_t candidateTimestamp = getCandidateTimestampLocked(&metadataIdx); 209 210 if (candidateTimestamp == -1) { 211 ALOGE("%s: Could not find good candidate for ZSL reprocessing", 212 __FUNCTION__); 213 return NOT_ENOUGH_DATA; 214 } 215 216 res = mZslStream->enqueueInputBufferByTimestamp(candidateTimestamp, 217 /*actualTimestamp*/NULL); 218 219 if (res == mZslStream->NO_BUFFER_AVAILABLE) { 220 ALOGV("%s: No ZSL buffers yet", __FUNCTION__); 221 return NOT_ENOUGH_DATA; 222 } else if (res != OK) { 223 ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)", 224 __FUNCTION__, strerror(-res), res); 225 return res; 226 } 227 228 { 229 CameraMetadata request = mFrameList[metadataIdx]; 230 231 // Verify that the frame is reasonable for reprocessing 232 233 camera_metadata_entry_t entry; 234 entry = request.find(ANDROID_CONTROL_AE_STATE); 235 if (entry.count == 0) { 236 ALOGE("%s: ZSL queue frame has no AE state field!", 237 __FUNCTION__); 238 return BAD_VALUE; 239 } 240 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 241 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 242 ALOGV("%s: ZSL queue frame AE state is %d, need full capture", 243 __FUNCTION__, entry.data.u8[0]); 244 return NOT_ENOUGH_DATA; 245 } 246 247 uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS; 248 res = request.update(ANDROID_REQUEST_TYPE, 249 &requestType, 1); 250 int32_t inputStreams[1] = 251 { mZslStreamId }; 252 if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS, 253 inputStreams, 1); 254 // TODO: Shouldn't we also update the latest preview frame? 255 int32_t outputStreams[1] = 256 { client->getCaptureStreamId() }; 257 if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS, 258 outputStreams, 1); 259 res = request.update(ANDROID_REQUEST_ID, 260 &requestId, 1); 261 262 if (res != OK ) { 263 ALOGE("%s: Unable to update frame to a reprocess request", 264 __FUNCTION__); 265 return INVALID_OPERATION; 266 } 267 268 res = client->stopStream(); 269 if (res != OK) { 270 ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: " 271 "%s (%d)", 272 __FUNCTION__, client->getCameraId(), strerror(-res), res); 273 return INVALID_OPERATION; 274 } 275 276 // Flush device to clear out all in-flight requests pending in HAL. 277 res = client->getCameraDevice()->flush(); 278 if (res != OK) { 279 ALOGE("%s: Camera %d: Failed to flush device: " 280 "%s (%d)", 281 __FUNCTION__, client->getCameraId(), strerror(-res), res); 282 return res; 283 } 284 285 // Update JPEG settings 286 { 287 SharedParameters::Lock l(client->getParameters()); 288 res = l.mParameters.updateRequestJpeg(&request); 289 if (res != OK) { 290 ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL " 291 "capture request: %s (%d)", __FUNCTION__, 292 client->getCameraId(), 293 strerror(-res), res); 294 return res; 295 } 296 } 297 298 mLatestCapturedRequest = request; 299 res = client->getCameraDevice()->capture(request); 300 if (res != OK ) { 301 ALOGE("%s: Unable to send ZSL reprocess request to capture: %s" 302 " (%d)", __FUNCTION__, strerror(-res), res); 303 return res; 304 } 305 306 mState = LOCKED; 307 } 308 309 return OK; 310} 311 312status_t ZslProcessor3::clearZslQueue() { 313 Mutex::Autolock l(mInputMutex); 314 // If in middle of capture, can't clear out queue 315 if (mState == LOCKED) return OK; 316 317 return clearZslQueueLocked(); 318} 319 320status_t ZslProcessor3::clearZslQueueLocked() { 321 if (mZslStream != 0) { 322 return mZslStream->clearInputRingBuffer(); 323 } 324 return OK; 325} 326 327void ZslProcessor3::dump(int fd, const Vector<String16>& /*args*/) const { 328 Mutex::Autolock l(mInputMutex); 329 if (!mLatestCapturedRequest.isEmpty()) { 330 String8 result(" Latest ZSL capture request:\n"); 331 write(fd, result.string(), result.size()); 332 mLatestCapturedRequest.dump(fd, 2, 6); 333 } else { 334 String8 result(" Latest ZSL capture request: none yet\n"); 335 write(fd, result.string(), result.size()); 336 } 337 dumpZslQueue(fd); 338} 339 340bool ZslProcessor3::threadLoop() { 341 // TODO: remove dependency on thread. For now, shut thread down right 342 // away. 343 return false; 344} 345 346void ZslProcessor3::dumpZslQueue(int fd) const { 347 String8 header("ZSL queue contents:"); 348 String8 indent(" "); 349 ALOGV("%s", header.string()); 350 if (fd != -1) { 351 header = indent + header + "\n"; 352 write(fd, header.string(), header.size()); 353 } 354 for (size_t i = 0; i < mZslQueue.size(); i++) { 355 const ZslPair &queueEntry = mZslQueue[i]; 356 nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp; 357 camera_metadata_ro_entry_t entry; 358 nsecs_t frameTimestamp = 0; 359 int frameAeState = -1; 360 if (!queueEntry.frame.isEmpty()) { 361 entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP); 362 if (entry.count > 0) frameTimestamp = entry.data.i64[0]; 363 entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE); 364 if (entry.count > 0) frameAeState = entry.data.u8[0]; 365 } 366 String8 result = 367 String8::format(" %d: b: %lld\tf: %lld, AE state: %d", i, 368 bufferTimestamp, frameTimestamp, frameAeState); 369 ALOGV("%s", result.string()); 370 if (fd != -1) { 371 result = indent + result + "\n"; 372 write(fd, result.string(), result.size()); 373 } 374 375 } 376} 377 378nsecs_t ZslProcessor3::getCandidateTimestampLocked(size_t* metadataIdx) const { 379 /** 380 * Find the smallest timestamp we know about so far 381 * - ensure that aeState is either converged or locked 382 */ 383 384 size_t idx = 0; 385 nsecs_t minTimestamp = -1; 386 387 size_t emptyCount = mFrameList.size(); 388 389 for (size_t j = 0; j < mFrameList.size(); j++) { 390 const CameraMetadata &frame = mFrameList[j]; 391 if (!frame.isEmpty()) { 392 393 emptyCount--; 394 395 camera_metadata_ro_entry_t entry; 396 entry = frame.find(ANDROID_SENSOR_TIMESTAMP); 397 if (entry.count == 0) { 398 ALOGE("%s: Can't find timestamp in frame!", 399 __FUNCTION__); 400 continue; 401 } 402 nsecs_t frameTimestamp = entry.data.i64[0]; 403 if (minTimestamp > frameTimestamp || minTimestamp == -1) { 404 405 entry = frame.find(ANDROID_CONTROL_AE_STATE); 406 407 if (entry.count == 0) { 408 /** 409 * This is most likely a HAL bug. The aeState field is 410 * mandatory, so it should always be in a metadata packet. 411 */ 412 ALOGW("%s: ZSL queue frame has no AE state field!", 413 __FUNCTION__); 414 continue; 415 } 416 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 417 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 418 ALOGVV("%s: ZSL queue frame AE state is %d, need " 419 "full capture", __FUNCTION__, entry.data.u8[0]); 420 continue; 421 } 422 423 minTimestamp = frameTimestamp; 424 idx = j; 425 } 426 427 ALOGVV("%s: Saw timestamp %lld", __FUNCTION__, frameTimestamp); 428 } 429 } 430 431 if (emptyCount == mFrameList.size()) { 432 /** 433 * This could be mildly bad and means our ZSL was triggered before 434 * there were any frames yet received by the camera framework. 435 * 436 * This is a fairly corner case which can happen under: 437 * + a user presses the shutter button real fast when the camera starts 438 * (startPreview followed immediately by takePicture). 439 * + burst capture case (hitting shutter button as fast possible) 440 * 441 * If this happens in steady case (preview running for a while, call 442 * a single takePicture) then this might be a fwk bug. 443 */ 444 ALOGW("%s: ZSL queue has no metadata frames", __FUNCTION__); 445 } 446 447 ALOGV("%s: Candidate timestamp %lld (idx %d), empty frames: %d", 448 __FUNCTION__, minTimestamp, idx, emptyCount); 449 450 if (metadataIdx) { 451 *metadataIdx = idx; 452 } 453 454 return minTimestamp; 455} 456 457void ZslProcessor3::onBufferAcquired(const BufferInfo& /*bufferInfo*/) { 458 // Intentionally left empty 459 // Although theoretically we could use this to get better dump info 460} 461 462void ZslProcessor3::onBufferReleased(const BufferInfo& bufferInfo) { 463 Mutex::Autolock l(mInputMutex); 464 465 // ignore output buffers 466 if (bufferInfo.mOutput) { 467 return; 468 } 469 470 // TODO: Verify that the buffer is in our queue by looking at timestamp 471 // theoretically unnecessary unless we change the following assumptions: 472 // -- only 1 buffer reprocessed at a time (which is the case now) 473 474 // Erase entire ZSL queue since we've now completed the capture and preview 475 // is stopped. 476 // 477 // We need to guarantee that if we do two back-to-back captures, 478 // the second won't use a buffer that's older/the same as the first, which 479 // is theoretically possible if we don't clear out the queue and the 480 // selection criteria is something like 'newest'. Clearing out the queue 481 // on a completed capture ensures we'll only use new data. 482 ALOGV("%s: Memory optimization, clearing ZSL queue", 483 __FUNCTION__); 484 clearZslQueueLocked(); 485 486 // Required so we accept more ZSL requests 487 mState = RUNNING; 488} 489 490}; // namespace camera2 491}; // namespace android 492