ExternalCameraDeviceSession.cpp revision 72eb5eea6b5c2a35dbb574f7381ff07d86e3063a
1/* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16#define LOG_TAG "ExtCamDevSsn@3.4" 17//#define LOG_NDEBUG 0 18#include <log/log.h> 19 20#include <inttypes.h> 21#include "ExternalCameraDeviceSession.h" 22 23#include "android-base/macros.h" 24#include <utils/Timers.h> 25#include <linux/videodev2.h> 26#include <sync/sync.h> 27 28#define HAVE_JPEG // required for libyuv.h to export MJPEG decode APIs 29#include <libyuv.h> 30 31#include <jpeglib.h> 32 33 34namespace android { 35namespace hardware { 36namespace camera { 37namespace device { 38namespace V3_4 { 39namespace implementation { 40 41namespace { 42// Size of request/result metadata fast message queue. Change to 0 to always use hwbinder buffer. 43static constexpr size_t kMetadataMsgQueueSize = 1 << 18 /* 256kB */; 44 45const int kBadFramesAfterStreamOn = 1; // drop x frames after streamOn to get rid of some initial 46 // bad frames. TODO: develop a better bad frame detection 47 // method 48 49} // Anonymous namespace 50 51// Static instances 52const int ExternalCameraDeviceSession::kMaxProcessedStream; 53const int ExternalCameraDeviceSession::kMaxStallStream; 54HandleImporter ExternalCameraDeviceSession::sHandleImporter; 55 56ExternalCameraDeviceSession::ExternalCameraDeviceSession( 57 const sp<ICameraDeviceCallback>& callback, 58 const ExternalCameraConfig& cfg, 59 const std::vector<SupportedV4L2Format>& sortedFormats, 60 const CroppingType& croppingType, 61 const common::V1_0::helper::CameraMetadata& chars, 62 unique_fd v4l2Fd) : 63 mCallback(callback), 64 mCfg(cfg), 65 mCameraCharacteristics(chars), 66 mSupportedFormats(sortedFormats), 67 mCroppingType(croppingType), 68 mV4l2Fd(std::move(v4l2Fd)), 69 mOutputThread(new OutputThread(this, mCroppingType)), 70 mMaxThumbResolution(getMaxThumbResolution()), 71 mMaxJpegResolution(getMaxJpegResolution()) { 72 mInitFail = initialize(); 73} 74 75bool ExternalCameraDeviceSession::initialize() { 76 if (mV4l2Fd.get() < 0) { 77 ALOGE("%s: invalid v4l2 device fd %d!", __FUNCTION__, mV4l2Fd.get()); 78 return true; 79 } 80 81 status_t status = initDefaultRequests(); 82 if (status != OK) { 83 ALOGE("%s: init default requests failed!", __FUNCTION__); 84 return true; 85 } 86 87 mRequestMetadataQueue = std::make_unique<RequestMetadataQueue>( 88 kMetadataMsgQueueSize, false /* non blocking */); 89 if (!mRequestMetadataQueue->isValid()) { 90 ALOGE("%s: invalid request fmq", __FUNCTION__); 91 return true; 92 } 93 mResultMetadataQueue = std::make_shared<RequestMetadataQueue>( 94 kMetadataMsgQueueSize, false /* non blocking */); 95 if (!mResultMetadataQueue->isValid()) { 96 ALOGE("%s: invalid result fmq", __FUNCTION__); 97 return true; 98 } 99 100 // TODO: check is PRIORITY_DISPLAY enough? 101 mOutputThread->run("ExtCamOut", PRIORITY_DISPLAY); 102 return false; 103} 104 105Status ExternalCameraDeviceSession::initStatus() const { 106 Mutex::Autolock _l(mLock); 107 Status status = Status::OK; 108 if (mInitFail || mClosed) { 109 ALOGI("%s: sesssion initFailed %d closed %d", __FUNCTION__, mInitFail, mClosed); 110 status = Status::INTERNAL_ERROR; 111 } 112 return status; 113} 114 115ExternalCameraDeviceSession::~ExternalCameraDeviceSession() { 116 if (!isClosed()) { 117 ALOGE("ExternalCameraDeviceSession deleted before close!"); 118 close(); 119 } 120} 121 122void ExternalCameraDeviceSession::dumpState(const native_handle_t*) { 123 // TODO: b/72261676 dump more runtime information 124} 125 126Return<void> ExternalCameraDeviceSession::constructDefaultRequestSettings( 127 V3_2::RequestTemplate type, 128 V3_2::ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb) { 129 V3_2::CameraMetadata outMetadata; 130 Status status = constructDefaultRequestSettingsRaw( 131 static_cast<RequestTemplate>(type), &outMetadata); 132 _hidl_cb(status, outMetadata); 133 return Void(); 134} 135 136Status ExternalCameraDeviceSession::constructDefaultRequestSettingsRaw(RequestTemplate type, 137 V3_2::CameraMetadata *outMetadata) { 138 CameraMetadata emptyMd; 139 Status status = initStatus(); 140 if (status != Status::OK) { 141 return status; 142 } 143 144 switch (type) { 145 case RequestTemplate::PREVIEW: 146 case RequestTemplate::STILL_CAPTURE: 147 case RequestTemplate::VIDEO_RECORD: 148 case RequestTemplate::VIDEO_SNAPSHOT: { 149 *outMetadata = mDefaultRequests[type]; 150 break; 151 } 152 case RequestTemplate::MANUAL: 153 case RequestTemplate::ZERO_SHUTTER_LAG: 154 // Don't support MANUAL, ZSL templates 155 status = Status::ILLEGAL_ARGUMENT; 156 break; 157 default: 158 ALOGE("%s: unknown request template type %d", __FUNCTION__, static_cast<int>(type)); 159 status = Status::ILLEGAL_ARGUMENT; 160 break; 161 } 162 return status; 163} 164 165Return<void> ExternalCameraDeviceSession::configureStreams( 166 const V3_2::StreamConfiguration& streams, 167 ICameraDeviceSession::configureStreams_cb _hidl_cb) { 168 V3_2::HalStreamConfiguration outStreams; 169 V3_3::HalStreamConfiguration outStreams_v33; 170 Mutex::Autolock _il(mInterfaceLock); 171 172 Status status = configureStreams(streams, &outStreams_v33); 173 size_t size = outStreams_v33.streams.size(); 174 outStreams.streams.resize(size); 175 for (size_t i = 0; i < size; i++) { 176 outStreams.streams[i] = outStreams_v33.streams[i].v3_2; 177 } 178 _hidl_cb(status, outStreams); 179 return Void(); 180} 181 182Return<void> ExternalCameraDeviceSession::configureStreams_3_3( 183 const V3_2::StreamConfiguration& streams, 184 ICameraDeviceSession::configureStreams_3_3_cb _hidl_cb) { 185 V3_3::HalStreamConfiguration outStreams; 186 Mutex::Autolock _il(mInterfaceLock); 187 188 Status status = configureStreams(streams, &outStreams); 189 _hidl_cb(status, outStreams); 190 return Void(); 191} 192 193Return<void> ExternalCameraDeviceSession::configureStreams_3_4( 194 const V3_4::StreamConfiguration& requestedConfiguration, 195 ICameraDeviceSession::configureStreams_3_4_cb _hidl_cb) { 196 V3_2::StreamConfiguration config_v32; 197 V3_3::HalStreamConfiguration outStreams_v33; 198 Mutex::Autolock _il(mInterfaceLock); 199 200 config_v32.operationMode = requestedConfiguration.operationMode; 201 config_v32.streams.resize(requestedConfiguration.streams.size()); 202 for (size_t i = 0; i < config_v32.streams.size(); i++) { 203 config_v32.streams[i] = requestedConfiguration.streams[i].v3_2; 204 } 205 206 // Ignore requestedConfiguration.sessionParams. External camera does not support it 207 Status status = configureStreams(config_v32, &outStreams_v33); 208 209 V3_4::HalStreamConfiguration outStreams; 210 outStreams.streams.resize(outStreams_v33.streams.size()); 211 for (size_t i = 0; i < outStreams.streams.size(); i++) { 212 outStreams.streams[i].v3_3 = outStreams_v33.streams[i]; 213 } 214 _hidl_cb(status, outStreams); 215 return Void(); 216} 217 218Return<void> ExternalCameraDeviceSession::getCaptureRequestMetadataQueue( 219 ICameraDeviceSession::getCaptureRequestMetadataQueue_cb _hidl_cb) { 220 Mutex::Autolock _il(mInterfaceLock); 221 _hidl_cb(*mRequestMetadataQueue->getDesc()); 222 return Void(); 223} 224 225Return<void> ExternalCameraDeviceSession::getCaptureResultMetadataQueue( 226 ICameraDeviceSession::getCaptureResultMetadataQueue_cb _hidl_cb) { 227 Mutex::Autolock _il(mInterfaceLock); 228 _hidl_cb(*mResultMetadataQueue->getDesc()); 229 return Void(); 230} 231 232Return<void> ExternalCameraDeviceSession::processCaptureRequest( 233 const hidl_vec<CaptureRequest>& requests, 234 const hidl_vec<BufferCache>& cachesToRemove, 235 ICameraDeviceSession::processCaptureRequest_cb _hidl_cb) { 236 Mutex::Autolock _il(mInterfaceLock); 237 updateBufferCaches(cachesToRemove); 238 239 uint32_t numRequestProcessed = 0; 240 Status s = Status::OK; 241 for (size_t i = 0; i < requests.size(); i++, numRequestProcessed++) { 242 s = processOneCaptureRequest(requests[i]); 243 if (s != Status::OK) { 244 break; 245 } 246 } 247 248 _hidl_cb(s, numRequestProcessed); 249 return Void(); 250} 251 252Return<void> ExternalCameraDeviceSession::processCaptureRequest_3_4( 253 const hidl_vec<V3_4::CaptureRequest>& requests, 254 const hidl_vec<V3_2::BufferCache>& cachesToRemove, 255 ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb) { 256 Mutex::Autolock _il(mInterfaceLock); 257 updateBufferCaches(cachesToRemove); 258 259 uint32_t numRequestProcessed = 0; 260 Status s = Status::OK; 261 for (size_t i = 0; i < requests.size(); i++, numRequestProcessed++) { 262 s = processOneCaptureRequest(requests[i].v3_2); 263 if (s != Status::OK) { 264 break; 265 } 266 } 267 268 _hidl_cb(s, numRequestProcessed); 269 return Void(); 270} 271 272Return<Status> ExternalCameraDeviceSession::flush() { 273 return Status::OK; 274} 275 276Return<void> ExternalCameraDeviceSession::close() { 277 Mutex::Autolock _il(mInterfaceLock); 278 Mutex::Autolock _l(mLock); 279 if (!mClosed) { 280 // TODO: b/72261676 Cleanup inflight buffers/V4L2 buffer queue 281 ALOGV("%s: closing V4L2 camera FD %d", __FUNCTION__, mV4l2Fd.get()); 282 mV4l2Fd.reset(); 283 mOutputThread->requestExit(); // TODO: join? 284 285 // free all imported buffers 286 for(auto& pair : mCirculatingBuffers) { 287 CirculatingBuffers& buffers = pair.second; 288 for (auto& p2 : buffers) { 289 sHandleImporter.freeBuffer(p2.second); 290 } 291 } 292 293 mClosed = true; 294 } 295 return Void(); 296} 297 298Status ExternalCameraDeviceSession::importRequest( 299 const CaptureRequest& request, 300 hidl_vec<buffer_handle_t*>& allBufPtrs, 301 hidl_vec<int>& allFences) { 302 size_t numOutputBufs = request.outputBuffers.size(); 303 size_t numBufs = numOutputBufs; 304 // Validate all I/O buffers 305 hidl_vec<buffer_handle_t> allBufs; 306 hidl_vec<uint64_t> allBufIds; 307 allBufs.resize(numBufs); 308 allBufIds.resize(numBufs); 309 allBufPtrs.resize(numBufs); 310 allFences.resize(numBufs); 311 std::vector<int32_t> streamIds(numBufs); 312 313 for (size_t i = 0; i < numOutputBufs; i++) { 314 allBufs[i] = request.outputBuffers[i].buffer.getNativeHandle(); 315 allBufIds[i] = request.outputBuffers[i].bufferId; 316 allBufPtrs[i] = &allBufs[i]; 317 streamIds[i] = request.outputBuffers[i].streamId; 318 } 319 320 for (size_t i = 0; i < numBufs; i++) { 321 buffer_handle_t buf = allBufs[i]; 322 uint64_t bufId = allBufIds[i]; 323 CirculatingBuffers& cbs = mCirculatingBuffers[streamIds[i]]; 324 if (cbs.count(bufId) == 0) { 325 if (buf == nullptr) { 326 ALOGE("%s: bufferId %" PRIu64 " has null buffer handle!", __FUNCTION__, bufId); 327 return Status::ILLEGAL_ARGUMENT; 328 } 329 // Register a newly seen buffer 330 buffer_handle_t importedBuf = buf; 331 sHandleImporter.importBuffer(importedBuf); 332 if (importedBuf == nullptr) { 333 ALOGE("%s: output buffer %zu is invalid!", __FUNCTION__, i); 334 return Status::INTERNAL_ERROR; 335 } else { 336 cbs[bufId] = importedBuf; 337 } 338 } 339 allBufPtrs[i] = &cbs[bufId]; 340 } 341 342 // All buffers are imported. Now validate output buffer acquire fences 343 for (size_t i = 0; i < numOutputBufs; i++) { 344 if (!sHandleImporter.importFence( 345 request.outputBuffers[i].acquireFence, allFences[i])) { 346 ALOGE("%s: output buffer %zu acquire fence is invalid", __FUNCTION__, i); 347 cleanupInflightFences(allFences, i); 348 return Status::INTERNAL_ERROR; 349 } 350 } 351 return Status::OK; 352} 353 354void ExternalCameraDeviceSession::cleanupInflightFences( 355 hidl_vec<int>& allFences, size_t numFences) { 356 for (size_t j = 0; j < numFences; j++) { 357 sHandleImporter.closeFence(allFences[j]); 358 } 359} 360 361Status ExternalCameraDeviceSession::processOneCaptureRequest(const CaptureRequest& request) { 362 Status status = initStatus(); 363 if (status != Status::OK) { 364 return status; 365 } 366 367 if (request.inputBuffer.streamId != -1) { 368 ALOGE("%s: external camera does not support reprocessing!", __FUNCTION__); 369 return Status::ILLEGAL_ARGUMENT; 370 } 371 372 Mutex::Autolock _l(mLock); 373 if (!mV4l2Streaming) { 374 ALOGE("%s: cannot process request in streamOff state!", __FUNCTION__); 375 return Status::INTERNAL_ERROR; 376 } 377 378 const camera_metadata_t *rawSettings = nullptr; 379 bool converted = true; 380 CameraMetadata settingsFmq; // settings from FMQ 381 if (request.fmqSettingsSize > 0) { 382 // non-blocking read; client must write metadata before calling 383 // processOneCaptureRequest 384 settingsFmq.resize(request.fmqSettingsSize); 385 bool read = mRequestMetadataQueue->read(settingsFmq.data(), request.fmqSettingsSize); 386 if (read) { 387 converted = V3_2::implementation::convertFromHidl(settingsFmq, &rawSettings); 388 } else { 389 ALOGE("%s: capture request settings metadata couldn't be read from fmq!", __FUNCTION__); 390 converted = false; 391 } 392 } else { 393 converted = V3_2::implementation::convertFromHidl(request.settings, &rawSettings); 394 } 395 396 if (converted && rawSettings != nullptr) { 397 mLatestReqSetting = rawSettings; 398 } 399 400 if (!converted) { 401 ALOGE("%s: capture request settings metadata is corrupt!", __FUNCTION__); 402 return Status::ILLEGAL_ARGUMENT; 403 } 404 405 if (mFirstRequest && rawSettings == nullptr) { 406 ALOGE("%s: capture request settings must not be null for first request!", 407 __FUNCTION__); 408 return Status::ILLEGAL_ARGUMENT; 409 } 410 411 hidl_vec<buffer_handle_t*> allBufPtrs; 412 hidl_vec<int> allFences; 413 size_t numOutputBufs = request.outputBuffers.size(); 414 415 if (numOutputBufs == 0) { 416 ALOGE("%s: capture request must have at least one output buffer!", __FUNCTION__); 417 return Status::ILLEGAL_ARGUMENT; 418 } 419 420 status = importRequest(request, allBufPtrs, allFences); 421 if (status != Status::OK) { 422 return status; 423 } 424 425 // TODO: program fps range per capture request here 426 // or limit the set of availableFpsRange 427 428 sp<V4L2Frame> frameIn = dequeueV4l2FrameLocked(); 429 if ( frameIn == nullptr) { 430 ALOGE("%s: V4L2 deque frame failed!", __FUNCTION__); 431 return Status::INTERNAL_ERROR; 432 } 433 // TODO: This can probably be replaced by use v4lbuffer timestamp 434 // if the device supports it 435 nsecs_t shutterTs = systemTime(SYSTEM_TIME_MONOTONIC); 436 437 438 // TODO: reduce object copy in this path 439 HalRequest halReq = { 440 .frameNumber = request.frameNumber, 441 .setting = mLatestReqSetting, 442 .frameIn = frameIn, 443 .shutterTs = shutterTs}; 444 halReq.buffers.resize(numOutputBufs); 445 for (size_t i = 0; i < numOutputBufs; i++) { 446 HalStreamBuffer& halBuf = halReq.buffers[i]; 447 int streamId = halBuf.streamId = request.outputBuffers[i].streamId; 448 halBuf.bufferId = request.outputBuffers[i].bufferId; 449 const Stream& stream = mStreamMap[streamId]; 450 halBuf.width = stream.width; 451 halBuf.height = stream.height; 452 halBuf.format = stream.format; 453 halBuf.usage = stream.usage; 454 halBuf.bufPtr = allBufPtrs[i]; 455 halBuf.acquireFence = allFences[i]; 456 halBuf.fenceTimeout = false; 457 } 458 mInflightFrames.insert(halReq.frameNumber); 459 // Send request to OutputThread for the rest of processing 460 mOutputThread->submitRequest(halReq); 461 mFirstRequest = false; 462 return Status::OK; 463} 464 465void ExternalCameraDeviceSession::notifyShutter(uint32_t frameNumber, nsecs_t shutterTs) { 466 NotifyMsg msg; 467 msg.type = MsgType::SHUTTER; 468 msg.msg.shutter.frameNumber = frameNumber; 469 msg.msg.shutter.timestamp = shutterTs; 470 mCallback->notify({msg}); 471} 472 473void ExternalCameraDeviceSession::notifyError( 474 uint32_t frameNumber, int32_t streamId, ErrorCode ec) { 475 NotifyMsg msg; 476 msg.type = MsgType::ERROR; 477 msg.msg.error.frameNumber = frameNumber; 478 msg.msg.error.errorStreamId = streamId; 479 msg.msg.error.errorCode = ec; 480 mCallback->notify({msg}); 481} 482 483//TODO: refactor with processCaptureResult 484Status ExternalCameraDeviceSession::processCaptureRequestError(HalRequest& req) { 485 // Return V4L2 buffer to V4L2 buffer queue 486 enqueueV4l2Frame(req.frameIn); 487 488 // NotifyShutter 489 notifyShutter(req.frameNumber, req.shutterTs); 490 491 notifyError(/*frameNum*/req.frameNumber, /*stream*/-1, ErrorCode::ERROR_REQUEST); 492 493 // Fill output buffers 494 hidl_vec<CaptureResult> results; 495 results.resize(1); 496 CaptureResult& result = results[0]; 497 result.frameNumber = req.frameNumber; 498 result.partialResult = 1; 499 result.inputBuffer.streamId = -1; 500 result.outputBuffers.resize(req.buffers.size()); 501 for (size_t i = 0; i < req.buffers.size(); i++) { 502 result.outputBuffers[i].streamId = req.buffers[i].streamId; 503 result.outputBuffers[i].bufferId = req.buffers[i].bufferId; 504 result.outputBuffers[i].status = BufferStatus::ERROR; 505 if (req.buffers[i].acquireFence >= 0) { 506 native_handle_t* handle = native_handle_create(/*numFds*/1, /*numInts*/0); 507 handle->data[0] = req.buffers[i].acquireFence; 508 result.outputBuffers[i].releaseFence.setTo(handle, /*shouldOwn*/false); 509 } 510 } 511 512 // update inflight records 513 { 514 Mutex::Autolock _l(mLock); 515 mInflightFrames.erase(req.frameNumber); 516 } 517 518 // Callback into framework 519 invokeProcessCaptureResultCallback(results, /* tryWriteFmq */true); 520 freeReleaseFences(results); 521 return Status::OK; 522} 523 524Status ExternalCameraDeviceSession::processCaptureResult(HalRequest& req) { 525 // Return V4L2 buffer to V4L2 buffer queue 526 enqueueV4l2Frame(req.frameIn); 527 528 // NotifyShutter 529 notifyShutter(req.frameNumber, req.shutterTs); 530 531 // Fill output buffers 532 hidl_vec<CaptureResult> results; 533 results.resize(1); 534 CaptureResult& result = results[0]; 535 result.frameNumber = req.frameNumber; 536 result.partialResult = 1; 537 result.inputBuffer.streamId = -1; 538 result.outputBuffers.resize(req.buffers.size()); 539 for (size_t i = 0; i < req.buffers.size(); i++) { 540 result.outputBuffers[i].streamId = req.buffers[i].streamId; 541 result.outputBuffers[i].bufferId = req.buffers[i].bufferId; 542 if (req.buffers[i].fenceTimeout) { 543 result.outputBuffers[i].status = BufferStatus::ERROR; 544 native_handle_t* handle = native_handle_create(/*numFds*/1, /*numInts*/0); 545 handle->data[0] = req.buffers[i].acquireFence; 546 result.outputBuffers[i].releaseFence.setTo(handle, /*shouldOwn*/false); 547 notifyError(req.frameNumber, req.buffers[i].streamId, ErrorCode::ERROR_BUFFER); 548 } else { 549 result.outputBuffers[i].status = BufferStatus::OK; 550 // TODO: refactor 551 if (req.buffers[i].acquireFence > 0) { 552 native_handle_t* handle = native_handle_create(/*numFds*/1, /*numInts*/0); 553 handle->data[0] = req.buffers[i].acquireFence; 554 result.outputBuffers[i].releaseFence.setTo(handle, /*shouldOwn*/false); 555 } 556 } 557 } 558 559 // Fill capture result metadata 560 fillCaptureResult(req.setting, req.shutterTs); 561 const camera_metadata_t *rawResult = req.setting.getAndLock(); 562 V3_2::implementation::convertToHidl(rawResult, &result.result); 563 req.setting.unlock(rawResult); 564 565 // update inflight records 566 { 567 Mutex::Autolock _l(mLock); 568 mInflightFrames.erase(req.frameNumber); 569 } 570 571 // Callback into framework 572 invokeProcessCaptureResultCallback(results, /* tryWriteFmq */true); 573 freeReleaseFences(results); 574 return Status::OK; 575} 576 577void ExternalCameraDeviceSession::invokeProcessCaptureResultCallback( 578 hidl_vec<CaptureResult> &results, bool tryWriteFmq) { 579 if (mProcessCaptureResultLock.tryLock() != OK) { 580 const nsecs_t NS_TO_SECOND = 1000000000; 581 ALOGV("%s: previous call is not finished! waiting 1s...", __FUNCTION__); 582 if (mProcessCaptureResultLock.timedLock(/* 1s */NS_TO_SECOND) != OK) { 583 ALOGE("%s: cannot acquire lock in 1s, cannot proceed", 584 __FUNCTION__); 585 return; 586 } 587 } 588 if (tryWriteFmq && mResultMetadataQueue->availableToWrite() > 0) { 589 for (CaptureResult &result : results) { 590 if (result.result.size() > 0) { 591 if (mResultMetadataQueue->write(result.result.data(), result.result.size())) { 592 result.fmqResultSize = result.result.size(); 593 result.result.resize(0); 594 } else { 595 ALOGW("%s: couldn't utilize fmq, fall back to hwbinder", __FUNCTION__); 596 result.fmqResultSize = 0; 597 } 598 } else { 599 result.fmqResultSize = 0; 600 } 601 } 602 } 603 auto status = mCallback->processCaptureResult(results); 604 if (!status.isOk()) { 605 ALOGE("%s: processCaptureResult ERROR : %s", __FUNCTION__, 606 status.description().c_str()); 607 } 608 609 mProcessCaptureResultLock.unlock(); 610} 611 612void ExternalCameraDeviceSession::freeReleaseFences(hidl_vec<CaptureResult>& results) { 613 for (auto& result : results) { 614 if (result.inputBuffer.releaseFence.getNativeHandle() != nullptr) { 615 native_handle_t* handle = const_cast<native_handle_t*>( 616 result.inputBuffer.releaseFence.getNativeHandle()); 617 native_handle_close(handle); 618 native_handle_delete(handle); 619 } 620 for (auto& buf : result.outputBuffers) { 621 if (buf.releaseFence.getNativeHandle() != nullptr) { 622 native_handle_t* handle = const_cast<native_handle_t*>( 623 buf.releaseFence.getNativeHandle()); 624 native_handle_close(handle); 625 native_handle_delete(handle); 626 } 627 } 628 } 629 return; 630} 631 632ExternalCameraDeviceSession::OutputThread::OutputThread( 633 wp<ExternalCameraDeviceSession> parent, 634 CroppingType ct) : mParent(parent), mCroppingType(ct) {} 635 636ExternalCameraDeviceSession::OutputThread::~OutputThread() {} 637 638uint32_t ExternalCameraDeviceSession::OutputThread::getFourCcFromLayout( 639 const YCbCrLayout& layout) { 640 intptr_t cb = reinterpret_cast<intptr_t>(layout.cb); 641 intptr_t cr = reinterpret_cast<intptr_t>(layout.cr); 642 if (std::abs(cb - cr) == 1 && layout.chromaStep == 2) { 643 // Interleaved format 644 if (layout.cb > layout.cr) { 645 return V4L2_PIX_FMT_NV21; 646 } else { 647 return V4L2_PIX_FMT_NV12; 648 } 649 } else if (layout.chromaStep == 1) { 650 // Planar format 651 if (layout.cb > layout.cr) { 652 return V4L2_PIX_FMT_YVU420; // YV12 653 } else { 654 return V4L2_PIX_FMT_YUV420; // YU12 655 } 656 } else { 657 return FLEX_YUV_GENERIC; 658 } 659} 660 661int ExternalCameraDeviceSession::OutputThread::getCropRect( 662 CroppingType ct, const Size& inSize, const Size& outSize, IMapper::Rect* out) { 663 if (out == nullptr) { 664 ALOGE("%s: out is null", __FUNCTION__); 665 return -1; 666 } 667 668 uint32_t inW = inSize.width; 669 uint32_t inH = inSize.height; 670 uint32_t outW = outSize.width; 671 uint32_t outH = outSize.height; 672 673 // Handle special case where aspect ratio is close to input but scaled 674 // dimension is slightly larger than input 675 float arIn = ASPECT_RATIO(inSize); 676 float arOut = ASPECT_RATIO(outSize); 677 if (isAspectRatioClose(arIn, arOut)) { 678 out->left = 0; 679 out->top = 0; 680 out->width = inW; 681 out->height = inH; 682 return 0; 683 } 684 685 if (ct == VERTICAL) { 686 uint64_t scaledOutH = static_cast<uint64_t>(outH) * inW / outW; 687 if (scaledOutH > inH) { 688 ALOGE("%s: Output size %dx%d cannot be vertically cropped from input size %dx%d", 689 __FUNCTION__, outW, outH, inW, inH); 690 return -1; 691 } 692 scaledOutH = scaledOutH & ~0x1; // make it multiple of 2 693 694 out->left = 0; 695 out->top = ((inH - scaledOutH) / 2) & ~0x1; 696 out->width = inW; 697 out->height = static_cast<int32_t>(scaledOutH); 698 ALOGV("%s: crop %dx%d to %dx%d: top %d, scaledH %d", 699 __FUNCTION__, inW, inH, outW, outH, out->top, static_cast<int32_t>(scaledOutH)); 700 } else { 701 uint64_t scaledOutW = static_cast<uint64_t>(outW) * inH / outH; 702 if (scaledOutW > inW) { 703 ALOGE("%s: Output size %dx%d cannot be horizontally cropped from input size %dx%d", 704 __FUNCTION__, outW, outH, inW, inH); 705 return -1; 706 } 707 scaledOutW = scaledOutW & ~0x1; // make it multiple of 2 708 709 out->left = ((inW - scaledOutW) / 2) & ~0x1; 710 out->top = 0; 711 out->width = static_cast<int32_t>(scaledOutW); 712 out->height = inH; 713 ALOGV("%s: crop %dx%d to %dx%d: top %d, scaledW %d", 714 __FUNCTION__, inW, inH, outW, outH, out->top, static_cast<int32_t>(scaledOutW)); 715 } 716 717 return 0; 718} 719 720int ExternalCameraDeviceSession::OutputThread::cropAndScaleLocked( 721 sp<AllocatedFrame>& in, const Size& outSz, YCbCrLayout* out) { 722 Size inSz = {in->mWidth, in->mHeight}; 723 724 int ret; 725 if (inSz == outSz) { 726 ret = in->getLayout(out); 727 if (ret != 0) { 728 ALOGE("%s: failed to get input image layout", __FUNCTION__); 729 return ret; 730 } 731 return ret; 732 } 733 734 // Cropping to output aspect ratio 735 IMapper::Rect inputCrop; 736 ret = getCropRect(mCroppingType, inSz, outSz, &inputCrop); 737 if (ret != 0) { 738 ALOGE("%s: failed to compute crop rect for output size %dx%d", 739 __FUNCTION__, outSz.width, outSz.height); 740 return ret; 741 } 742 743 YCbCrLayout croppedLayout; 744 ret = in->getCroppedLayout(inputCrop, &croppedLayout); 745 if (ret != 0) { 746 ALOGE("%s: failed to crop input image %dx%d to output size %dx%d", 747 __FUNCTION__, inSz.width, inSz.height, outSz.width, outSz.height); 748 return ret; 749 } 750 751 if ((mCroppingType == VERTICAL && inSz.width == outSz.width) || 752 (mCroppingType == HORIZONTAL && inSz.height == outSz.height)) { 753 // No scale is needed 754 *out = croppedLayout; 755 return 0; 756 } 757 758 auto it = mScaledYu12Frames.find(outSz); 759 sp<AllocatedFrame> scaledYu12Buf; 760 if (it != mScaledYu12Frames.end()) { 761 scaledYu12Buf = it->second; 762 } else { 763 it = mIntermediateBuffers.find(outSz); 764 if (it == mIntermediateBuffers.end()) { 765 ALOGE("%s: failed to find intermediate buffer size %dx%d", 766 __FUNCTION__, outSz.width, outSz.height); 767 return -1; 768 } 769 scaledYu12Buf = it->second; 770 } 771 // Scale 772 YCbCrLayout outLayout; 773 ret = scaledYu12Buf->getLayout(&outLayout); 774 if (ret != 0) { 775 ALOGE("%s: failed to get output buffer layout", __FUNCTION__); 776 return ret; 777 } 778 779 ret = libyuv::I420Scale( 780 static_cast<uint8_t*>(croppedLayout.y), 781 croppedLayout.yStride, 782 static_cast<uint8_t*>(croppedLayout.cb), 783 croppedLayout.cStride, 784 static_cast<uint8_t*>(croppedLayout.cr), 785 croppedLayout.cStride, 786 inputCrop.width, 787 inputCrop.height, 788 static_cast<uint8_t*>(outLayout.y), 789 outLayout.yStride, 790 static_cast<uint8_t*>(outLayout.cb), 791 outLayout.cStride, 792 static_cast<uint8_t*>(outLayout.cr), 793 outLayout.cStride, 794 outSz.width, 795 outSz.height, 796 // TODO: b/72261744 see if we can use better filter without losing too much perf 797 libyuv::FilterMode::kFilterNone); 798 799 if (ret != 0) { 800 ALOGE("%s: failed to scale buffer from %dx%d to %dx%d. Ret %d", 801 __FUNCTION__, inputCrop.width, inputCrop.height, 802 outSz.width, outSz.height, ret); 803 return ret; 804 } 805 806 *out = outLayout; 807 mScaledYu12Frames.insert({outSz, scaledYu12Buf}); 808 return 0; 809} 810 811 812int ExternalCameraDeviceSession::OutputThread::cropAndScaleThumbLocked( 813 sp<AllocatedFrame>& in, const Size &outSz, YCbCrLayout* out) { 814 Size inSz {in->mWidth, in->mHeight}; 815 816 if ((outSz.width * outSz.height) > 817 (mYu12ThumbFrame->mWidth * mYu12ThumbFrame->mHeight)) { 818 ALOGE("%s: Requested thumbnail size too big (%d,%d) > (%d,%d)", 819 __FUNCTION__, outSz.width, outSz.height, 820 mYu12ThumbFrame->mWidth, mYu12ThumbFrame->mHeight); 821 return -1; 822 } 823 824 int ret; 825 826 /* This will crop-and-zoom the input YUV frame to the thumbnail size 827 * Based on the following logic: 828 * 1) Square pixels come in, square pixels come out, therefore single 829 * scale factor is computed to either make input bigger or smaller 830 * depending on if we are upscaling or downscaling 831 * 2) That single scale factor would either make height too tall or width 832 * too wide so we need to crop the input either horizontally or vertically 833 * but not both 834 */ 835 836 /* Convert the input and output dimensions into floats for ease of math */ 837 float fWin = static_cast<float>(inSz.width); 838 float fHin = static_cast<float>(inSz.height); 839 float fWout = static_cast<float>(outSz.width); 840 float fHout = static_cast<float>(outSz.height); 841 842 /* Compute the one scale factor from (1) above, it will be the smaller of 843 * the two possibilities. */ 844 float scaleFactor = std::min( fHin / fHout, fWin / fWout ); 845 846 /* Since we are crop-and-zooming (as opposed to letter/pillar boxing) we can 847 * simply multiply the output by our scaleFactor to get the cropped input 848 * size. Note that at least one of {fWcrop, fHcrop} is going to wind up 849 * being {fWin, fHin} respectively because fHout or fWout cancels out the 850 * scaleFactor calculation above. 851 * 852 * Specifically: 853 * if ( fHin / fHout ) < ( fWin / fWout ) we crop the sides off 854 * input, in which case 855 * scaleFactor = fHin / fHout 856 * fWcrop = fHin / fHout * fWout 857 * fHcrop = fHin 858 * 859 * Note that fWcrop <= fWin ( because ( fHin / fHout ) * fWout < fWin, which 860 * is just the inequality above with both sides multiplied by fWout 861 * 862 * on the other hand if ( fWin / fWout ) < ( fHin / fHout) we crop the top 863 * and the bottom off of input, and 864 * scaleFactor = fWin / fWout 865 * fWcrop = fWin 866 * fHCrop = fWin / fWout * fHout 867 */ 868 float fWcrop = scaleFactor * fWout; 869 float fHcrop = scaleFactor * fHout; 870 871 /* Convert to integer and truncate to an even number */ 872 Size cropSz = { 2*static_cast<uint32_t>(fWcrop/2.0f), 873 2*static_cast<uint32_t>(fHcrop/2.0f) }; 874 875 /* Convert to a centered rectange with even top/left */ 876 IMapper::Rect inputCrop { 877 2*static_cast<int32_t>((inSz.width - cropSz.width)/4), 878 2*static_cast<int32_t>((inSz.height - cropSz.height)/4), 879 static_cast<int32_t>(cropSz.width), 880 static_cast<int32_t>(cropSz.height) }; 881 882 if ((inputCrop.top < 0) || 883 (inputCrop.top >= static_cast<int32_t>(inSz.height)) || 884 (inputCrop.left < 0) || 885 (inputCrop.left >= static_cast<int32_t>(inSz.width)) || 886 (inputCrop.width <= 0) || 887 (inputCrop.width + inputCrop.left > static_cast<int32_t>(inSz.width)) || 888 (inputCrop.height <= 0) || 889 (inputCrop.height + inputCrop.top > static_cast<int32_t>(inSz.height))) 890 { 891 ALOGE("%s: came up with really wrong crop rectangle",__FUNCTION__); 892 ALOGE("%s: input layout %dx%d to for output size %dx%d", 893 __FUNCTION__, inSz.width, inSz.height, outSz.width, outSz.height); 894 ALOGE("%s: computed input crop +%d,+%d %dx%d", 895 __FUNCTION__, inputCrop.left, inputCrop.top, 896 inputCrop.width, inputCrop.height); 897 return -1; 898 } 899 900 YCbCrLayout inputLayout; 901 ret = in->getCroppedLayout(inputCrop, &inputLayout); 902 if (ret != 0) { 903 ALOGE("%s: failed to crop input layout %dx%d to for output size %dx%d", 904 __FUNCTION__, inSz.width, inSz.height, outSz.width, outSz.height); 905 ALOGE("%s: computed input crop +%d,+%d %dx%d", 906 __FUNCTION__, inputCrop.left, inputCrop.top, 907 inputCrop.width, inputCrop.height); 908 return ret; 909 } 910 ALOGV("%s: crop input layout %dx%d to for output size %dx%d", 911 __FUNCTION__, inSz.width, inSz.height, outSz.width, outSz.height); 912 ALOGV("%s: computed input crop +%d,+%d %dx%d", 913 __FUNCTION__, inputCrop.left, inputCrop.top, 914 inputCrop.width, inputCrop.height); 915 916 917 // Scale 918 YCbCrLayout outFullLayout; 919 920 ret = mYu12ThumbFrame->getLayout(&outFullLayout); 921 if (ret != 0) { 922 ALOGE("%s: failed to get output buffer layout", __FUNCTION__); 923 return ret; 924 } 925 926 927 ret = libyuv::I420Scale( 928 static_cast<uint8_t*>(inputLayout.y), 929 inputLayout.yStride, 930 static_cast<uint8_t*>(inputLayout.cb), 931 inputLayout.cStride, 932 static_cast<uint8_t*>(inputLayout.cr), 933 inputLayout.cStride, 934 inputCrop.width, 935 inputCrop.height, 936 static_cast<uint8_t*>(outFullLayout.y), 937 outFullLayout.yStride, 938 static_cast<uint8_t*>(outFullLayout.cb), 939 outFullLayout.cStride, 940 static_cast<uint8_t*>(outFullLayout.cr), 941 outFullLayout.cStride, 942 outSz.width, 943 outSz.height, 944 libyuv::FilterMode::kFilterNone); 945 946 if (ret != 0) { 947 ALOGE("%s: failed to scale buffer from %dx%d to %dx%d. Ret %d", 948 __FUNCTION__, inputCrop.width, inputCrop.height, 949 outSz.width, outSz.height, ret); 950 return ret; 951 } 952 953 *out = outFullLayout; 954 return 0; 955} 956 957int ExternalCameraDeviceSession::OutputThread::formatConvertLocked( 958 const YCbCrLayout& in, const YCbCrLayout& out, Size sz, uint32_t format) { 959 int ret = 0; 960 switch (format) { 961 case V4L2_PIX_FMT_NV21: 962 ret = libyuv::I420ToNV21( 963 static_cast<uint8_t*>(in.y), 964 in.yStride, 965 static_cast<uint8_t*>(in.cb), 966 in.cStride, 967 static_cast<uint8_t*>(in.cr), 968 in.cStride, 969 static_cast<uint8_t*>(out.y), 970 out.yStride, 971 static_cast<uint8_t*>(out.cr), 972 out.cStride, 973 sz.width, 974 sz.height); 975 if (ret != 0) { 976 ALOGE("%s: convert to NV21 buffer failed! ret %d", 977 __FUNCTION__, ret); 978 return ret; 979 } 980 break; 981 case V4L2_PIX_FMT_NV12: 982 ret = libyuv::I420ToNV12( 983 static_cast<uint8_t*>(in.y), 984 in.yStride, 985 static_cast<uint8_t*>(in.cb), 986 in.cStride, 987 static_cast<uint8_t*>(in.cr), 988 in.cStride, 989 static_cast<uint8_t*>(out.y), 990 out.yStride, 991 static_cast<uint8_t*>(out.cb), 992 out.cStride, 993 sz.width, 994 sz.height); 995 if (ret != 0) { 996 ALOGE("%s: convert to NV12 buffer failed! ret %d", 997 __FUNCTION__, ret); 998 return ret; 999 } 1000 break; 1001 case V4L2_PIX_FMT_YVU420: // YV12 1002 case V4L2_PIX_FMT_YUV420: // YU12 1003 // TODO: maybe we can speed up here by somehow save this copy? 1004 ret = libyuv::I420Copy( 1005 static_cast<uint8_t*>(in.y), 1006 in.yStride, 1007 static_cast<uint8_t*>(in.cb), 1008 in.cStride, 1009 static_cast<uint8_t*>(in.cr), 1010 in.cStride, 1011 static_cast<uint8_t*>(out.y), 1012 out.yStride, 1013 static_cast<uint8_t*>(out.cb), 1014 out.cStride, 1015 static_cast<uint8_t*>(out.cr), 1016 out.cStride, 1017 sz.width, 1018 sz.height); 1019 if (ret != 0) { 1020 ALOGE("%s: copy to YV12 or YU12 buffer failed! ret %d", 1021 __FUNCTION__, ret); 1022 return ret; 1023 } 1024 break; 1025 case FLEX_YUV_GENERIC: 1026 // TODO: b/72261744 write to arbitrary flexible YUV layout. Slow. 1027 ALOGE("%s: unsupported flexible yuv layout" 1028 " y %p cb %p cr %p y_str %d c_str %d c_step %d", 1029 __FUNCTION__, out.y, out.cb, out.cr, 1030 out.yStride, out.cStride, out.chromaStep); 1031 return -1; 1032 default: 1033 ALOGE("%s: unknown YUV format 0x%x!", __FUNCTION__, format); 1034 return -1; 1035 } 1036 return 0; 1037} 1038 1039int ExternalCameraDeviceSession::OutputThread::encodeJpegYU12( 1040 const Size & inSz, const YCbCrLayout& inLayout, 1041 int jpegQuality, const void *app1Buffer, size_t app1Size, 1042 void *out, const size_t maxOutSize, size_t &actualCodeSize) 1043{ 1044 /* libjpeg is a C library so we use C-style "inheritance" by 1045 * putting libjpeg's jpeg_destination_mgr first in our custom 1046 * struct. This allows us to cast jpeg_destination_mgr* to 1047 * CustomJpegDestMgr* when we get it passed to us in a callback */ 1048 struct CustomJpegDestMgr { 1049 struct jpeg_destination_mgr mgr; 1050 JOCTET *mBuffer; 1051 size_t mBufferSize; 1052 size_t mEncodedSize; 1053 bool mSuccess; 1054 } dmgr; 1055 1056 jpeg_compress_struct cinfo = {}; 1057 jpeg_error_mgr jerr; 1058 1059 /* Initialize error handling with standard callbacks, but 1060 * then override output_message (to print to ALOG) and 1061 * error_exit to set a flag and print a message instead 1062 * of killing the whole process */ 1063 cinfo.err = jpeg_std_error(&jerr); 1064 1065 cinfo.err->output_message = [](j_common_ptr cinfo) { 1066 char buffer[JMSG_LENGTH_MAX]; 1067 1068 /* Create the message */ 1069 (*cinfo->err->format_message)(cinfo, buffer); 1070 ALOGE("libjpeg error: %s", buffer); 1071 }; 1072 cinfo.err->error_exit = [](j_common_ptr cinfo) { 1073 (*cinfo->err->output_message)(cinfo); 1074 if(cinfo->client_data) { 1075 auto & dmgr = 1076 *reinterpret_cast<CustomJpegDestMgr*>(cinfo->client_data); 1077 dmgr.mSuccess = false; 1078 } 1079 }; 1080 /* Now that we initialized some callbacks, let's create our compressor */ 1081 jpeg_create_compress(&cinfo); 1082 1083 /* Initialize our destination manager */ 1084 dmgr.mBuffer = static_cast<JOCTET*>(out); 1085 dmgr.mBufferSize = maxOutSize; 1086 dmgr.mEncodedSize = 0; 1087 dmgr.mSuccess = true; 1088 cinfo.client_data = static_cast<void*>(&dmgr); 1089 1090 /* These lambdas become C-style function pointers and as per C++11 spec 1091 * may not capture anything */ 1092 dmgr.mgr.init_destination = [](j_compress_ptr cinfo) { 1093 auto & dmgr = reinterpret_cast<CustomJpegDestMgr&>(*cinfo->dest); 1094 dmgr.mgr.next_output_byte = dmgr.mBuffer; 1095 dmgr.mgr.free_in_buffer = dmgr.mBufferSize; 1096 ALOGV("%s:%d jpeg start: %p [%zu]", 1097 __FUNCTION__, __LINE__, dmgr.mBuffer, dmgr.mBufferSize); 1098 }; 1099 1100 dmgr.mgr.empty_output_buffer = [](j_compress_ptr cinfo __unused) { 1101 ALOGV("%s:%d Out of buffer", __FUNCTION__, __LINE__); 1102 return 0; 1103 }; 1104 1105 dmgr.mgr.term_destination = [](j_compress_ptr cinfo) { 1106 auto & dmgr = reinterpret_cast<CustomJpegDestMgr&>(*cinfo->dest); 1107 dmgr.mEncodedSize = dmgr.mBufferSize - dmgr.mgr.free_in_buffer; 1108 ALOGV("%s:%d Done with jpeg: %zu", __FUNCTION__, __LINE__, dmgr.mEncodedSize); 1109 }; 1110 cinfo.dest = reinterpret_cast<struct jpeg_destination_mgr*>(&dmgr); 1111 1112 /* We are going to be using JPEG in raw data mode, so we are passing 1113 * straight subsampled planar YCbCr and it will not touch our pixel 1114 * data or do any scaling or anything */ 1115 cinfo.image_width = inSz.width; 1116 cinfo.image_height = inSz.height; 1117 cinfo.input_components = 3; 1118 cinfo.in_color_space = JCS_YCbCr; 1119 1120 /* Initialize defaults and then override what we want */ 1121 jpeg_set_defaults(&cinfo); 1122 1123 jpeg_set_quality(&cinfo, jpegQuality, 1); 1124 jpeg_set_colorspace(&cinfo, JCS_YCbCr); 1125 cinfo.raw_data_in = 1; 1126 cinfo.dct_method = JDCT_IFAST; 1127 1128 /* Configure sampling factors. The sampling factor is JPEG subsampling 420 1129 * because the source format is YUV420. Note that libjpeg sampling factors 1130 * are... a little weird. Sampling of Y=2,U=1,V=1 means there is 1 U and 1131 * 1 V value for each 2 Y values */ 1132 cinfo.comp_info[0].h_samp_factor = 2; 1133 cinfo.comp_info[0].v_samp_factor = 2; 1134 cinfo.comp_info[1].h_samp_factor = 1; 1135 cinfo.comp_info[1].v_samp_factor = 1; 1136 cinfo.comp_info[2].h_samp_factor = 1; 1137 cinfo.comp_info[2].v_samp_factor = 1; 1138 1139 /* Let's not hardcode YUV420 in 6 places... 5 was enough */ 1140 int maxVSampFactor = std::max( { 1141 cinfo.comp_info[0].v_samp_factor, 1142 cinfo.comp_info[1].v_samp_factor, 1143 cinfo.comp_info[2].v_samp_factor 1144 }); 1145 int cVSubSampling = cinfo.comp_info[0].v_samp_factor / 1146 cinfo.comp_info[1].v_samp_factor; 1147 1148 /* Start the compressor */ 1149 jpeg_start_compress(&cinfo, TRUE); 1150 1151 /* Compute our macroblock height, so we can pad our input to be vertically 1152 * macroblock aligned. 1153 * TODO: Does it need to be horizontally MCU aligned too? */ 1154 1155 size_t mcuV = DCTSIZE*maxVSampFactor; 1156 size_t paddedHeight = mcuV * ((inSz.height + mcuV - 1) / mcuV); 1157 1158 /* libjpeg uses arrays of row pointers, which makes it really easy to pad 1159 * data vertically (unfortunately doesn't help horizontally) */ 1160 std::vector<JSAMPROW> yLines (paddedHeight); 1161 std::vector<JSAMPROW> cbLines(paddedHeight/cVSubSampling); 1162 std::vector<JSAMPROW> crLines(paddedHeight/cVSubSampling); 1163 1164 uint8_t *py = static_cast<uint8_t*>(inLayout.y); 1165 uint8_t *pcr = static_cast<uint8_t*>(inLayout.cr); 1166 uint8_t *pcb = static_cast<uint8_t*>(inLayout.cb); 1167 1168 for(uint32_t i = 0; i < paddedHeight; i++) 1169 { 1170 /* Once we are in the padding territory we still point to the last line 1171 * effectively replicating it several times ~ CLAMP_TO_EDGE */ 1172 int li = std::min(i, inSz.height - 1); 1173 yLines[i] = static_cast<JSAMPROW>(py + li * inLayout.yStride); 1174 if(i < paddedHeight / cVSubSampling) 1175 { 1176 crLines[i] = static_cast<JSAMPROW>(pcr + li * inLayout.cStride); 1177 cbLines[i] = static_cast<JSAMPROW>(pcb + li * inLayout.cStride); 1178 } 1179 } 1180 1181 /* If APP1 data was passed in, use it */ 1182 if(app1Buffer && app1Size) 1183 { 1184 jpeg_write_marker(&cinfo, JPEG_APP0 + 1, 1185 static_cast<const JOCTET*>(app1Buffer), app1Size); 1186 } 1187 1188 /* While we still have padded height left to go, keep giving it one 1189 * macroblock at a time. */ 1190 while (cinfo.next_scanline < cinfo.image_height) { 1191 const uint32_t batchSize = DCTSIZE * maxVSampFactor; 1192 const uint32_t nl = cinfo.next_scanline; 1193 JSAMPARRAY planes[3]{ &yLines[nl], 1194 &cbLines[nl/cVSubSampling], 1195 &crLines[nl/cVSubSampling] }; 1196 1197 uint32_t done = jpeg_write_raw_data(&cinfo, planes, batchSize); 1198 1199 if (done != batchSize) { 1200 ALOGE("%s: compressed %u lines, expected %u (total %u/%u)", 1201 __FUNCTION__, done, batchSize, cinfo.next_scanline, 1202 cinfo.image_height); 1203 return -1; 1204 } 1205 } 1206 1207 /* This will flush everything */ 1208 jpeg_finish_compress(&cinfo); 1209 1210 /* Grab the actual code size and set it */ 1211 actualCodeSize = dmgr.mEncodedSize; 1212 1213 return 0; 1214} 1215 1216/* 1217 * TODO: There needs to be a mechanism to discover allocated buffer size 1218 * in the HAL. 1219 * 1220 * This is very fragile because it is duplicated computation from: 1221 * frameworks/av/services/camera/libcameraservice/device3/Camera3Device.cpp 1222 * 1223 */ 1224 1225/* This assumes mSupportedFormats have all been declared as supporting 1226 * HAL_PIXEL_FORMAT_BLOB to the framework */ 1227Size ExternalCameraDeviceSession::getMaxJpegResolution() const { 1228 Size ret { 0, 0 }; 1229 for(auto & fmt : mSupportedFormats) { 1230 if(fmt.width * fmt.height > ret.width * ret.height) { 1231 ret = Size { fmt.width, fmt.height }; 1232 } 1233 } 1234 return ret; 1235} 1236 1237Size ExternalCameraDeviceSession::getMaxThumbResolution() const { 1238 Size thumbSize { 0, 0 }; 1239 camera_metadata_ro_entry entry = 1240 mCameraCharacteristics.find(ANDROID_JPEG_AVAILABLE_THUMBNAIL_SIZES); 1241 for(uint32_t i = 0; i < entry.count; i += 2) { 1242 Size sz { static_cast<uint32_t>(entry.data.i32[i]), 1243 static_cast<uint32_t>(entry.data.i32[i+1]) }; 1244 if(sz.width * sz.height > thumbSize.width * thumbSize.height) { 1245 thumbSize = sz; 1246 } 1247 } 1248 1249 if (thumbSize.width * thumbSize.height == 0) { 1250 ALOGW("%s: non-zero thumbnail size not available", __FUNCTION__); 1251 } 1252 1253 return thumbSize; 1254} 1255 1256 1257ssize_t ExternalCameraDeviceSession::getJpegBufferSize( 1258 uint32_t width, uint32_t height) const { 1259 // Constant from camera3.h 1260 const ssize_t kMinJpegBufferSize = 256 * 1024 + sizeof(CameraBlob); 1261 // Get max jpeg size (area-wise). 1262 if (mMaxJpegResolution.width == 0) { 1263 ALOGE("%s: Do not have a single supported JPEG stream", 1264 __FUNCTION__); 1265 return BAD_VALUE; 1266 } 1267 1268 // Get max jpeg buffer size 1269 ssize_t maxJpegBufferSize = 0; 1270 camera_metadata_ro_entry jpegBufMaxSize = 1271 mCameraCharacteristics.find(ANDROID_JPEG_MAX_SIZE); 1272 if (jpegBufMaxSize.count == 0) { 1273 ALOGE("%s: Can't find maximum JPEG size in static metadata!", 1274 __FUNCTION__); 1275 return BAD_VALUE; 1276 } 1277 maxJpegBufferSize = jpegBufMaxSize.data.i32[0]; 1278 1279 if (maxJpegBufferSize <= kMinJpegBufferSize) { 1280 ALOGE("%s: ANDROID_JPEG_MAX_SIZE (%zd) <= kMinJpegBufferSize (%zd)", 1281 __FUNCTION__, maxJpegBufferSize, kMinJpegBufferSize); 1282 return BAD_VALUE; 1283 } 1284 1285 // Calculate final jpeg buffer size for the given resolution. 1286 float scaleFactor = ((float) (width * height)) / 1287 (mMaxJpegResolution.width * mMaxJpegResolution.height); 1288 ssize_t jpegBufferSize = scaleFactor * (maxJpegBufferSize - kMinJpegBufferSize) + 1289 kMinJpegBufferSize; 1290 if (jpegBufferSize > maxJpegBufferSize) { 1291 jpegBufferSize = maxJpegBufferSize; 1292 } 1293 1294 return jpegBufferSize; 1295} 1296 1297int ExternalCameraDeviceSession::OutputThread::createJpegLocked( 1298 HalStreamBuffer &halBuf, 1299 HalRequest &req) 1300{ 1301 int ret; 1302 auto lfail = [&](auto... args) { 1303 ALOGE(args...); 1304 1305 return 1; 1306 }; 1307 auto parent = mParent.promote(); 1308 if (parent == nullptr) { 1309 ALOGE("%s: session has been disconnected!", __FUNCTION__); 1310 return 1; 1311 } 1312 1313 ALOGV("%s: HAL buffer sid: %d bid: %" PRIu64 " w: %u h: %u", 1314 __FUNCTION__, halBuf.streamId, static_cast<uint64_t>(halBuf.bufferId), 1315 halBuf.width, halBuf.height); 1316 ALOGV("%s: HAL buffer fmt: %x usage: %" PRIx64 " ptr: %p", 1317 __FUNCTION__, halBuf.format, static_cast<uint64_t>(halBuf.usage), 1318 halBuf.bufPtr); 1319 ALOGV("%s: YV12 buffer %d x %d", 1320 __FUNCTION__, 1321 mYu12Frame->mWidth, mYu12Frame->mHeight); 1322 1323 int jpegQuality, thumbQuality; 1324 Size thumbSize; 1325 1326 if (req.setting.exists(ANDROID_JPEG_QUALITY)) { 1327 camera_metadata_entry entry = 1328 req.setting.find(ANDROID_JPEG_QUALITY); 1329 jpegQuality = entry.data.u8[0]; 1330 } else { 1331 return lfail("%s: ANDROID_JPEG_QUALITY not set",__FUNCTION__); 1332 } 1333 1334 if (req.setting.exists(ANDROID_JPEG_THUMBNAIL_QUALITY)) { 1335 camera_metadata_entry entry = 1336 req.setting.find(ANDROID_JPEG_THUMBNAIL_QUALITY); 1337 thumbQuality = entry.data.u8[0]; 1338 } else { 1339 return lfail( 1340 "%s: ANDROID_JPEG_THUMBNAIL_QUALITY not set", 1341 __FUNCTION__); 1342 } 1343 1344 if (req.setting.exists(ANDROID_JPEG_THUMBNAIL_SIZE)) { 1345 camera_metadata_entry entry = 1346 req.setting.find(ANDROID_JPEG_THUMBNAIL_SIZE); 1347 thumbSize = Size { static_cast<uint32_t>(entry.data.i32[0]), 1348 static_cast<uint32_t>(entry.data.i32[1]) 1349 }; 1350 } else { 1351 return lfail( 1352 "%s: ANDROID_JPEG_THUMBNAIL_SIZE not set", __FUNCTION__); 1353 } 1354 1355 /* Cropped and scaled YU12 buffer for main and thumbnail */ 1356 YCbCrLayout yu12Main; 1357 Size jpegSize { halBuf.width, halBuf.height }; 1358 1359 /* Compute temporary buffer sizes accounting for the following: 1360 * thumbnail can't exceed APP1 size of 64K 1361 * main image needs to hold APP1, headers, and at most a poorly 1362 * compressed image */ 1363 const ssize_t maxThumbCodeSize = 64 * 1024; 1364 const ssize_t maxJpegCodeSize = parent->getJpegBufferSize(jpegSize.width, 1365 jpegSize.height); 1366 1367 /* Check that getJpegBufferSize did not return an error */ 1368 if (maxJpegCodeSize < 0) { 1369 return lfail( 1370 "%s: getJpegBufferSize returned %zd",__FUNCTION__,maxJpegCodeSize); 1371 } 1372 1373 1374 /* Hold actual thumbnail and main image code sizes */ 1375 size_t thumbCodeSize = 0, jpegCodeSize = 0; 1376 /* Temporary thumbnail code buffer */ 1377 std::vector<uint8_t> thumbCode(maxThumbCodeSize); 1378 1379 YCbCrLayout yu12Thumb; 1380 ret = cropAndScaleThumbLocked(mYu12Frame, thumbSize, &yu12Thumb); 1381 1382 if (ret != 0) { 1383 return lfail( 1384 "%s: crop and scale thumbnail failed!", __FUNCTION__); 1385 } 1386 1387 /* Scale and crop main jpeg */ 1388 ret = cropAndScaleLocked(mYu12Frame, jpegSize, &yu12Main); 1389 1390 if (ret != 0) { 1391 return lfail("%s: crop and scale main failed!", __FUNCTION__); 1392 } 1393 1394 /* Encode the thumbnail image */ 1395 ret = encodeJpegYU12(thumbSize, yu12Thumb, 1396 thumbQuality, 0, 0, 1397 &thumbCode[0], maxThumbCodeSize, thumbCodeSize); 1398 1399 if (ret != 0) { 1400 return lfail("%s: encodeJpegYU12 failed with %d",__FUNCTION__, ret); 1401 } 1402 1403 /* Combine camera characteristics with request settings to form EXIF 1404 * metadata */ 1405 common::V1_0::helper::CameraMetadata meta(parent->mCameraCharacteristics); 1406 meta.append(req.setting); 1407 1408 /* Generate EXIF object */ 1409 std::unique_ptr<ExifUtils> utils(ExifUtils::create()); 1410 /* Make sure it's initialized */ 1411 utils->initialize(); 1412 1413 utils->setFromMetadata(meta, jpegSize.width, jpegSize.height); 1414 1415 /* Check if we made a non-zero-sized thumbnail. Currently not possible 1416 * that we got this far and the code is size 0, but if this code moves 1417 * around it might become relevant again */ 1418 1419 ret = utils->generateApp1(thumbCodeSize ? &thumbCode[0] : 0, thumbCodeSize); 1420 1421 if (!ret) { 1422 return lfail("%s: generating APP1 failed", __FUNCTION__); 1423 } 1424 1425 /* Get internal buffer */ 1426 size_t exifDataSize = utils->getApp1Length(); 1427 const uint8_t* exifData = utils->getApp1Buffer(); 1428 1429 /* Lock the HAL jpeg code buffer */ 1430 void *bufPtr = sHandleImporter.lock( 1431 *(halBuf.bufPtr), halBuf.usage, maxJpegCodeSize); 1432 1433 if (!bufPtr) { 1434 return lfail("%s: could not lock %zu bytes", __FUNCTION__, maxJpegCodeSize); 1435 } 1436 1437 /* Encode the main jpeg image */ 1438 ret = encodeJpegYU12(jpegSize, yu12Main, 1439 jpegQuality, exifData, exifDataSize, 1440 bufPtr, maxJpegCodeSize, jpegCodeSize); 1441 1442 /* TODO: Not sure this belongs here, maybe better to pass jpegCodeSize out 1443 * and do this when returning buffer to parent */ 1444 CameraBlob blob { CameraBlobId::JPEG, static_cast<uint32_t>(jpegCodeSize) }; 1445 void *blobDst = 1446 reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(bufPtr) + 1447 maxJpegCodeSize - 1448 sizeof(CameraBlob)); 1449 memcpy(blobDst, &blob, sizeof(CameraBlob)); 1450 1451 /* Unlock the HAL jpeg code buffer */ 1452 int relFence = sHandleImporter.unlock(*(halBuf.bufPtr)); 1453 if (relFence > 0) { 1454 halBuf.acquireFence = relFence; 1455 } 1456 1457 /* Check if our JPEG actually succeeded */ 1458 if (ret != 0) { 1459 return lfail( 1460 "%s: encodeJpegYU12 failed with %d",__FUNCTION__, ret); 1461 } 1462 1463 ALOGV("%s: encoded JPEG (ret:%d) with Q:%d max size: %zu", 1464 __FUNCTION__, ret, jpegQuality, maxJpegCodeSize); 1465 1466 return 0; 1467} 1468 1469bool ExternalCameraDeviceSession::OutputThread::threadLoop() { 1470 HalRequest req; 1471 auto parent = mParent.promote(); 1472 if (parent == nullptr) { 1473 ALOGE("%s: session has been disconnected!", __FUNCTION__); 1474 return false; 1475 } 1476 1477 // TODO: maybe we need to setup a sensor thread to dq/enq v4l frames 1478 // regularly to prevent v4l buffer queue filled with stale buffers 1479 // when app doesn't program a preveiw request 1480 waitForNextRequest(&req); 1481 if (req.frameIn == nullptr) { 1482 // No new request, wait again 1483 return true; 1484 } 1485 1486 if (req.frameIn->mFourcc != V4L2_PIX_FMT_MJPEG) { 1487 ALOGE("%s: do not support V4L2 format %c%c%c%c", __FUNCTION__, 1488 req.frameIn->mFourcc & 0xFF, 1489 (req.frameIn->mFourcc >> 8) & 0xFF, 1490 (req.frameIn->mFourcc >> 16) & 0xFF, 1491 (req.frameIn->mFourcc >> 24) & 0xFF); 1492 parent->notifyError( 1493 /*frameNum*/req.frameNumber, /*stream*/-1, ErrorCode::ERROR_DEVICE); 1494 return false; 1495 } 1496 1497 std::unique_lock<std::mutex> lk(mLock); 1498 1499 // Convert input V4L2 frame to YU12 of the same size 1500 // TODO: see if we can save some computation by converting to YV12 here 1501 uint8_t* inData; 1502 size_t inDataSize; 1503 req.frameIn->map(&inData, &inDataSize); 1504 // TODO: profile 1505 // TODO: in some special case maybe we can decode jpg directly to gralloc output? 1506 int res = libyuv::MJPGToI420( 1507 inData, inDataSize, 1508 static_cast<uint8_t*>(mYu12FrameLayout.y), 1509 mYu12FrameLayout.yStride, 1510 static_cast<uint8_t*>(mYu12FrameLayout.cb), 1511 mYu12FrameLayout.cStride, 1512 static_cast<uint8_t*>(mYu12FrameLayout.cr), 1513 mYu12FrameLayout.cStride, 1514 mYu12Frame->mWidth, mYu12Frame->mHeight, 1515 mYu12Frame->mWidth, mYu12Frame->mHeight); 1516 1517 if (res != 0) { 1518 // For some webcam, the first few V4L2 frames might be malformed... 1519 ALOGE("%s: Convert V4L2 frame to YU12 failed! res %d", __FUNCTION__, res); 1520 lk.unlock(); 1521 Status st = parent->processCaptureRequestError(req); 1522 if (st != Status::OK) { 1523 ALOGE("%s: failed to process capture request error!", __FUNCTION__); 1524 parent->notifyError( 1525 /*frameNum*/req.frameNumber, /*stream*/-1, ErrorCode::ERROR_DEVICE); 1526 return false; 1527 } 1528 return true; 1529 } 1530 1531 ALOGV("%s processing new request", __FUNCTION__); 1532 const int kSyncWaitTimeoutMs = 500; 1533 for (auto& halBuf : req.buffers) { 1534 if (halBuf.acquireFence != -1) { 1535 int ret = sync_wait(halBuf.acquireFence, kSyncWaitTimeoutMs); 1536 if (ret) { 1537 halBuf.fenceTimeout = true; 1538 } else { 1539 ::close(halBuf.acquireFence); 1540 halBuf.acquireFence = -1; 1541 } 1542 } 1543 1544 if (halBuf.fenceTimeout) { 1545 continue; 1546 } 1547 1548 // Gralloc lockYCbCr the buffer 1549 switch (halBuf.format) { 1550 case PixelFormat::BLOB: { 1551 int ret = createJpegLocked(halBuf, req); 1552 1553 if(ret != 0) { 1554 ALOGE("%s: createJpegLocked failed with %d", 1555 __FUNCTION__, ret); 1556 lk.unlock(); 1557 parent->notifyError( 1558 /*frameNum*/req.frameNumber, 1559 /*stream*/-1, 1560 ErrorCode::ERROR_DEVICE); 1561 1562 return false; 1563 } 1564 } break; 1565 case PixelFormat::YCBCR_420_888: 1566 case PixelFormat::YV12: { 1567 IMapper::Rect outRect {0, 0, 1568 static_cast<int32_t>(halBuf.width), 1569 static_cast<int32_t>(halBuf.height)}; 1570 YCbCrLayout outLayout = sHandleImporter.lockYCbCr( 1571 *(halBuf.bufPtr), halBuf.usage, outRect); 1572 ALOGV("%s: outLayout y %p cb %p cr %p y_str %d c_str %d c_step %d", 1573 __FUNCTION__, outLayout.y, outLayout.cb, outLayout.cr, 1574 outLayout.yStride, outLayout.cStride, outLayout.chromaStep); 1575 1576 // Convert to output buffer size/format 1577 uint32_t outputFourcc = getFourCcFromLayout(outLayout); 1578 ALOGV("%s: converting to format %c%c%c%c", __FUNCTION__, 1579 outputFourcc & 0xFF, 1580 (outputFourcc >> 8) & 0xFF, 1581 (outputFourcc >> 16) & 0xFF, 1582 (outputFourcc >> 24) & 0xFF); 1583 1584 YCbCrLayout cropAndScaled; 1585 int ret = cropAndScaleLocked( 1586 mYu12Frame, 1587 Size { halBuf.width, halBuf.height }, 1588 &cropAndScaled); 1589 if (ret != 0) { 1590 ALOGE("%s: crop and scale failed!", __FUNCTION__); 1591 lk.unlock(); 1592 parent->notifyError( 1593 /*frameNum*/req.frameNumber, /*stream*/-1, ErrorCode::ERROR_DEVICE); 1594 return false; 1595 } 1596 1597 Size sz {halBuf.width, halBuf.height}; 1598 ret = formatConvertLocked(cropAndScaled, outLayout, sz, outputFourcc); 1599 if (ret != 0) { 1600 ALOGE("%s: format coversion failed!", __FUNCTION__); 1601 lk.unlock(); 1602 parent->notifyError( 1603 /*frameNum*/req.frameNumber, /*stream*/-1, ErrorCode::ERROR_DEVICE); 1604 return false; 1605 } 1606 int relFence = sHandleImporter.unlock(*(halBuf.bufPtr)); 1607 if (relFence > 0) { 1608 halBuf.acquireFence = relFence; 1609 } 1610 } break; 1611 default: 1612 ALOGE("%s: unknown output format %x", __FUNCTION__, halBuf.format); 1613 lk.unlock(); 1614 parent->notifyError( 1615 /*frameNum*/req.frameNumber, /*stream*/-1, ErrorCode::ERROR_DEVICE); 1616 return false; 1617 } 1618 } // for each buffer 1619 mScaledYu12Frames.clear(); 1620 1621 // Don't hold the lock while calling back to parent 1622 lk.unlock(); 1623 Status st = parent->processCaptureResult(req); 1624 if (st != Status::OK) { 1625 ALOGE("%s: failed to process capture result!", __FUNCTION__); 1626 parent->notifyError( 1627 /*frameNum*/req.frameNumber, /*stream*/-1, ErrorCode::ERROR_DEVICE); 1628 return false; 1629 } 1630 return true; 1631} 1632 1633Status ExternalCameraDeviceSession::OutputThread::allocateIntermediateBuffers( 1634 const Size& v4lSize, const Size& thumbSize, 1635 const hidl_vec<Stream>& streams) { 1636 std::lock_guard<std::mutex> lk(mLock); 1637 if (mScaledYu12Frames.size() != 0) { 1638 ALOGE("%s: intermediate buffer pool has %zu inflight buffers! (expect 0)", 1639 __FUNCTION__, mScaledYu12Frames.size()); 1640 return Status::INTERNAL_ERROR; 1641 } 1642 1643 // Allocating intermediate YU12 frame 1644 if (mYu12Frame == nullptr || mYu12Frame->mWidth != v4lSize.width || 1645 mYu12Frame->mHeight != v4lSize.height) { 1646 mYu12Frame.clear(); 1647 mYu12Frame = new AllocatedFrame(v4lSize.width, v4lSize.height); 1648 int ret = mYu12Frame->allocate(&mYu12FrameLayout); 1649 if (ret != 0) { 1650 ALOGE("%s: allocating YU12 frame failed!", __FUNCTION__); 1651 return Status::INTERNAL_ERROR; 1652 } 1653 } 1654 1655 // Allocating intermediate YU12 thumbnail frame 1656 if (mYu12ThumbFrame == nullptr || 1657 mYu12ThumbFrame->mWidth != thumbSize.width || 1658 mYu12ThumbFrame->mHeight != thumbSize.height) { 1659 mYu12ThumbFrame.clear(); 1660 mYu12ThumbFrame = new AllocatedFrame(thumbSize.width, thumbSize.height); 1661 int ret = mYu12ThumbFrame->allocate(&mYu12ThumbFrameLayout); 1662 if (ret != 0) { 1663 ALOGE("%s: allocating YU12 thumb frame failed!", __FUNCTION__); 1664 return Status::INTERNAL_ERROR; 1665 } 1666 } 1667 1668 // Allocating scaled buffers 1669 for (const auto& stream : streams) { 1670 Size sz = {stream.width, stream.height}; 1671 if (sz == v4lSize) { 1672 continue; // Don't need an intermediate buffer same size as v4lBuffer 1673 } 1674 if (mIntermediateBuffers.count(sz) == 0) { 1675 // Create new intermediate buffer 1676 sp<AllocatedFrame> buf = new AllocatedFrame(stream.width, stream.height); 1677 int ret = buf->allocate(); 1678 if (ret != 0) { 1679 ALOGE("%s: allocating intermediate YU12 frame %dx%d failed!", 1680 __FUNCTION__, stream.width, stream.height); 1681 return Status::INTERNAL_ERROR; 1682 } 1683 mIntermediateBuffers[sz] = buf; 1684 } 1685 } 1686 1687 // Remove unconfigured buffers 1688 auto it = mIntermediateBuffers.begin(); 1689 while (it != mIntermediateBuffers.end()) { 1690 bool configured = false; 1691 auto sz = it->first; 1692 for (const auto& stream : streams) { 1693 if (stream.width == sz.width && stream.height == sz.height) { 1694 configured = true; 1695 break; 1696 } 1697 } 1698 if (configured) { 1699 it++; 1700 } else { 1701 it = mIntermediateBuffers.erase(it); 1702 } 1703 } 1704 return Status::OK; 1705} 1706 1707Status ExternalCameraDeviceSession::OutputThread::submitRequest(const HalRequest& req) { 1708 std::lock_guard<std::mutex> lk(mLock); 1709 // TODO: reduce object copy in this path 1710 mRequestList.push_back(req); 1711 mRequestCond.notify_one(); 1712 return Status::OK; 1713} 1714 1715void ExternalCameraDeviceSession::OutputThread::flush() { 1716 std::lock_guard<std::mutex> lk(mLock); 1717 // TODO: send buffer/request errors back to framework 1718 mRequestList.clear(); 1719} 1720 1721void ExternalCameraDeviceSession::OutputThread::waitForNextRequest(HalRequest* out) { 1722 if (out == nullptr) { 1723 ALOGE("%s: out is null", __FUNCTION__); 1724 return; 1725 } 1726 1727 std::unique_lock<std::mutex> lk(mLock); 1728 while (mRequestList.empty()) { 1729 std::chrono::seconds timeout = std::chrono::seconds(kReqWaitTimeoutSec); 1730 auto st = mRequestCond.wait_for(lk, timeout); 1731 if (st == std::cv_status::timeout) { 1732 // no new request, return 1733 return; 1734 } 1735 } 1736 *out = mRequestList.front(); 1737 mRequestList.pop_front(); 1738} 1739 1740void ExternalCameraDeviceSession::cleanupBuffersLocked(int id) { 1741 for (auto& pair : mCirculatingBuffers.at(id)) { 1742 sHandleImporter.freeBuffer(pair.second); 1743 } 1744 mCirculatingBuffers[id].clear(); 1745 mCirculatingBuffers.erase(id); 1746} 1747 1748void ExternalCameraDeviceSession::updateBufferCaches(const hidl_vec<BufferCache>& cachesToRemove) { 1749 Mutex::Autolock _l(mLock); 1750 for (auto& cache : cachesToRemove) { 1751 auto cbsIt = mCirculatingBuffers.find(cache.streamId); 1752 if (cbsIt == mCirculatingBuffers.end()) { 1753 // The stream could have been removed 1754 continue; 1755 } 1756 CirculatingBuffers& cbs = cbsIt->second; 1757 auto it = cbs.find(cache.bufferId); 1758 if (it != cbs.end()) { 1759 sHandleImporter.freeBuffer(it->second); 1760 cbs.erase(it); 1761 } else { 1762 ALOGE("%s: stream %d buffer %" PRIu64 " is not cached", 1763 __FUNCTION__, cache.streamId, cache.bufferId); 1764 } 1765 } 1766} 1767 1768bool ExternalCameraDeviceSession::isSupported(const Stream& stream) { 1769 int32_t ds = static_cast<int32_t>(stream.dataSpace); 1770 PixelFormat fmt = stream.format; 1771 uint32_t width = stream.width; 1772 uint32_t height = stream.height; 1773 // TODO: check usage flags 1774 1775 if (stream.streamType != StreamType::OUTPUT) { 1776 ALOGE("%s: does not support non-output stream type", __FUNCTION__); 1777 return false; 1778 } 1779 1780 if (stream.rotation != StreamRotation::ROTATION_0) { 1781 ALOGE("%s: does not support stream rotation", __FUNCTION__); 1782 return false; 1783 } 1784 1785 if (ds & Dataspace::DEPTH) { 1786 ALOGI("%s: does not support depth output", __FUNCTION__); 1787 return false; 1788 } 1789 1790 switch (fmt) { 1791 case PixelFormat::BLOB: 1792 if (ds != static_cast<int32_t>(Dataspace::V0_JFIF)) { 1793 ALOGI("%s: BLOB format does not support dataSpace %x", __FUNCTION__, ds); 1794 return false; 1795 } 1796 case PixelFormat::IMPLEMENTATION_DEFINED: 1797 case PixelFormat::YCBCR_420_888: 1798 case PixelFormat::YV12: 1799 // TODO: check what dataspace we can support here. 1800 // intentional no-ops. 1801 break; 1802 default: 1803 ALOGI("%s: does not support format %x", __FUNCTION__, fmt); 1804 return false; 1805 } 1806 1807 // Assume we can convert any V4L2 format to any of supported output format for now, i.e, 1808 // ignoring v4l2Fmt.fourcc for now. Might need more subtle check if we support more v4l format 1809 // in the futrue. 1810 for (const auto& v4l2Fmt : mSupportedFormats) { 1811 if (width == v4l2Fmt.width && height == v4l2Fmt.height) { 1812 return true; 1813 } 1814 } 1815 ALOGI("%s: resolution %dx%d is not supported", __FUNCTION__, width, height); 1816 return false; 1817} 1818 1819int ExternalCameraDeviceSession::v4l2StreamOffLocked() { 1820 if (!mV4l2Streaming) { 1821 return OK; 1822 } 1823 1824 { 1825 std::lock_guard<std::mutex> lk(mV4l2BufferLock); 1826 if (mNumDequeuedV4l2Buffers != 0) { 1827 ALOGE("%s: there are %zu inflight V4L buffers", 1828 __FUNCTION__, mNumDequeuedV4l2Buffers); 1829 return -1; 1830 } 1831 } 1832 mV4L2BufferCount = 0; 1833 1834 // VIDIOC_STREAMOFF 1835 v4l2_buf_type capture_type = V4L2_BUF_TYPE_VIDEO_CAPTURE; 1836 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_STREAMOFF, &capture_type)) < 0) { 1837 ALOGE("%s: STREAMOFF failed: %s", __FUNCTION__, strerror(errno)); 1838 return -errno; 1839 } 1840 1841 // VIDIOC_REQBUFS: clear buffers 1842 v4l2_requestbuffers req_buffers{}; 1843 req_buffers.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; 1844 req_buffers.memory = V4L2_MEMORY_MMAP; 1845 req_buffers.count = 0; 1846 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_REQBUFS, &req_buffers)) < 0) { 1847 ALOGE("%s: REQBUFS failed: %s", __FUNCTION__, strerror(errno)); 1848 return -errno; 1849 } 1850 1851 mV4l2Streaming = false; 1852 return OK; 1853} 1854 1855int ExternalCameraDeviceSession::configureV4l2StreamLocked(const SupportedV4L2Format& v4l2Fmt) { 1856 int ret = v4l2StreamOffLocked(); 1857 if (ret != OK) { 1858 ALOGE("%s: stop v4l2 streaming failed: ret %d", __FUNCTION__, ret); 1859 return ret; 1860 } 1861 1862 // VIDIOC_S_FMT w/h/fmt 1863 v4l2_format fmt; 1864 fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; 1865 fmt.fmt.pix.width = v4l2Fmt.width; 1866 fmt.fmt.pix.height = v4l2Fmt.height; 1867 fmt.fmt.pix.pixelformat = v4l2Fmt.fourcc; 1868 ret = TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_S_FMT, &fmt)); 1869 if (ret < 0) { 1870 ALOGE("%s: S_FMT ioctl failed: %s", __FUNCTION__, strerror(errno)); 1871 return -errno; 1872 } 1873 1874 if (v4l2Fmt.width != fmt.fmt.pix.width || v4l2Fmt.height != fmt.fmt.pix.height || 1875 v4l2Fmt.fourcc != fmt.fmt.pix.pixelformat) { 1876 ALOGE("%s: S_FMT expect %c%c%c%c %dx%d, got %c%c%c%c %dx%d instead!", __FUNCTION__, 1877 v4l2Fmt.fourcc & 0xFF, 1878 (v4l2Fmt.fourcc >> 8) & 0xFF, 1879 (v4l2Fmt.fourcc >> 16) & 0xFF, 1880 (v4l2Fmt.fourcc >> 24) & 0xFF, 1881 v4l2Fmt.width, v4l2Fmt.height, 1882 fmt.fmt.pix.pixelformat & 0xFF, 1883 (fmt.fmt.pix.pixelformat >> 8) & 0xFF, 1884 (fmt.fmt.pix.pixelformat >> 16) & 0xFF, 1885 (fmt.fmt.pix.pixelformat >> 24) & 0xFF, 1886 fmt.fmt.pix.width, fmt.fmt.pix.height); 1887 return -EINVAL; 1888 } 1889 uint32_t bufferSize = fmt.fmt.pix.sizeimage; 1890 ALOGI("%s: V4L2 buffer size is %d", __FUNCTION__, bufferSize); 1891 1892 float maxFps = -1.f; 1893 float fps = 1000.f; 1894 const float kDefaultFps = 30.f; 1895 // Try to pick the slowest fps that is at least 30 1896 for (const auto& fr : v4l2Fmt.frameRates) { 1897 double f = fr.getDouble(); 1898 if (maxFps < f) { 1899 maxFps = f; 1900 } 1901 if (f >= kDefaultFps && f < fps) { 1902 fps = f; 1903 } 1904 } 1905 if (fps == 1000.f) { 1906 fps = maxFps; 1907 } 1908 1909 // VIDIOC_G_PARM/VIDIOC_S_PARM: set fps 1910 v4l2_streamparm streamparm = { .type = V4L2_BUF_TYPE_VIDEO_CAPTURE }; 1911 // The following line checks that the driver knows about framerate get/set. 1912 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_G_PARM, &streamparm)) >= 0) { 1913 // Now check if the device is able to accept a capture framerate set. 1914 if (streamparm.parm.capture.capability & V4L2_CAP_TIMEPERFRAME) { 1915 // |frame_rate| is float, approximate by a fraction. 1916 const int kFrameRatePrecision = 10000; 1917 streamparm.parm.capture.timeperframe.numerator = kFrameRatePrecision; 1918 streamparm.parm.capture.timeperframe.denominator = 1919 (fps * kFrameRatePrecision); 1920 1921 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_S_PARM, &streamparm)) < 0) { 1922 ALOGE("%s: failed to set framerate to %f", __FUNCTION__, fps); 1923 return UNKNOWN_ERROR; 1924 } 1925 } 1926 } 1927 float retFps = streamparm.parm.capture.timeperframe.denominator / 1928 streamparm.parm.capture.timeperframe.numerator; 1929 if (std::fabs(fps - retFps) > std::numeric_limits<float>::epsilon()) { 1930 ALOGE("%s: expect fps %f, got %f instead", __FUNCTION__, fps, retFps); 1931 return BAD_VALUE; 1932 } 1933 1934 uint32_t v4lBufferCount = (fps >= kDefaultFps) ? 1935 mCfg.numVideoBuffers : mCfg.numStillBuffers; 1936 // VIDIOC_REQBUFS: create buffers 1937 v4l2_requestbuffers req_buffers{}; 1938 req_buffers.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; 1939 req_buffers.memory = V4L2_MEMORY_MMAP; 1940 req_buffers.count = v4lBufferCount; 1941 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_REQBUFS, &req_buffers)) < 0) { 1942 ALOGE("%s: VIDIOC_REQBUFS failed: %s", __FUNCTION__, strerror(errno)); 1943 return -errno; 1944 } 1945 1946 // Driver can indeed return more buffer if it needs more to operate 1947 if (req_buffers.count < v4lBufferCount) { 1948 ALOGE("%s: VIDIOC_REQBUFS expected %d buffers, got %d instead", 1949 __FUNCTION__, v4lBufferCount, req_buffers.count); 1950 return NO_MEMORY; 1951 } 1952 1953 // VIDIOC_QUERYBUF: get buffer offset in the V4L2 fd 1954 // VIDIOC_QBUF: send buffer to driver 1955 mV4L2BufferCount = req_buffers.count; 1956 for (uint32_t i = 0; i < req_buffers.count; i++) { 1957 v4l2_buffer buffer = { 1958 .type = V4L2_BUF_TYPE_VIDEO_CAPTURE, 1959 .index = i, 1960 .memory = V4L2_MEMORY_MMAP}; 1961 1962 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QUERYBUF, &buffer)) < 0) { 1963 ALOGE("%s: QUERYBUF %d failed: %s", __FUNCTION__, i, strerror(errno)); 1964 return -errno; 1965 } 1966 1967 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QBUF, &buffer)) < 0) { 1968 ALOGE("%s: QBUF %d failed: %s", __FUNCTION__, i, strerror(errno)); 1969 return -errno; 1970 } 1971 } 1972 1973 // VIDIOC_STREAMON: start streaming 1974 v4l2_buf_type capture_type = V4L2_BUF_TYPE_VIDEO_CAPTURE; 1975 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_STREAMON, &capture_type)) < 0) { 1976 ALOGE("%s: VIDIOC_STREAMON failed: %s", __FUNCTION__, strerror(errno)); 1977 return -errno; 1978 } 1979 1980 // Swallow first few frames after streamOn to account for bad frames from some devices 1981 for (int i = 0; i < kBadFramesAfterStreamOn; i++) { 1982 v4l2_buffer buffer{}; 1983 buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; 1984 buffer.memory = V4L2_MEMORY_MMAP; 1985 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_DQBUF, &buffer)) < 0) { 1986 ALOGE("%s: DQBUF fails: %s", __FUNCTION__, strerror(errno)); 1987 return -errno; 1988 } 1989 1990 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QBUF, &buffer)) < 0) { 1991 ALOGE("%s: QBUF index %d fails: %s", __FUNCTION__, buffer.index, strerror(errno)); 1992 return -errno; 1993 } 1994 } 1995 1996 mV4l2StreamingFmt = v4l2Fmt; 1997 mV4l2Streaming = true; 1998 return OK; 1999} 2000 2001sp<V4L2Frame> ExternalCameraDeviceSession::dequeueV4l2FrameLocked() { 2002 sp<V4L2Frame> ret = nullptr; 2003 2004 { 2005 std::unique_lock<std::mutex> lk(mV4l2BufferLock); 2006 if (mNumDequeuedV4l2Buffers == mV4L2BufferCount) { 2007 std::chrono::seconds timeout = std::chrono::seconds(kBufferWaitTimeoutSec); 2008 mLock.unlock(); 2009 auto st = mV4L2BufferReturned.wait_for(lk, timeout); 2010 mLock.lock(); 2011 if (st == std::cv_status::timeout) { 2012 ALOGE("%s: wait for V4L2 buffer return timeout!", __FUNCTION__); 2013 return ret; 2014 } 2015 } 2016 } 2017 2018 v4l2_buffer buffer{}; 2019 buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; 2020 buffer.memory = V4L2_MEMORY_MMAP; 2021 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_DQBUF, &buffer)) < 0) { 2022 ALOGE("%s: DQBUF fails: %s", __FUNCTION__, strerror(errno)); 2023 return ret; 2024 } 2025 2026 if (buffer.index >= mV4L2BufferCount) { 2027 ALOGE("%s: Invalid buffer id: %d", __FUNCTION__, buffer.index); 2028 return ret; 2029 } 2030 2031 if (buffer.flags & V4L2_BUF_FLAG_ERROR) { 2032 ALOGE("%s: v4l2 buf error! buf flag 0x%x", __FUNCTION__, buffer.flags); 2033 // TODO: try to dequeue again 2034 } 2035 2036 { 2037 std::lock_guard<std::mutex> lk(mV4l2BufferLock); 2038 mNumDequeuedV4l2Buffers++; 2039 } 2040 return new V4L2Frame( 2041 mV4l2StreamingFmt.width, mV4l2StreamingFmt.height, mV4l2StreamingFmt.fourcc, 2042 buffer.index, mV4l2Fd.get(), buffer.bytesused, buffer.m.offset); 2043} 2044 2045void ExternalCameraDeviceSession::enqueueV4l2Frame(const sp<V4L2Frame>& frame) { 2046 Mutex::Autolock _l(mLock); 2047 frame->unmap(); 2048 v4l2_buffer buffer{}; 2049 buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; 2050 buffer.memory = V4L2_MEMORY_MMAP; 2051 buffer.index = frame->mBufferIndex; 2052 if (TEMP_FAILURE_RETRY(ioctl(mV4l2Fd.get(), VIDIOC_QBUF, &buffer)) < 0) { 2053 ALOGE("%s: QBUF index %d fails: %s", __FUNCTION__, frame->mBufferIndex, strerror(errno)); 2054 return; 2055 } 2056 2057 { 2058 std::lock_guard<std::mutex> lk(mV4l2BufferLock); 2059 mNumDequeuedV4l2Buffers--; 2060 mV4L2BufferReturned.notify_one(); 2061 } 2062} 2063 2064Status ExternalCameraDeviceSession::configureStreams( 2065 const V3_2::StreamConfiguration& config, V3_3::HalStreamConfiguration* out) { 2066 if (config.operationMode != StreamConfigurationMode::NORMAL_MODE) { 2067 ALOGE("%s: unsupported operation mode: %d", __FUNCTION__, config.operationMode); 2068 return Status::ILLEGAL_ARGUMENT; 2069 } 2070 2071 if (config.streams.size() == 0) { 2072 ALOGE("%s: cannot configure zero stream", __FUNCTION__); 2073 return Status::ILLEGAL_ARGUMENT; 2074 } 2075 2076 int numProcessedStream = 0; 2077 int numStallStream = 0; 2078 for (const auto& stream : config.streams) { 2079 // Check if the format/width/height combo is supported 2080 if (!isSupported(stream)) { 2081 return Status::ILLEGAL_ARGUMENT; 2082 } 2083 if (stream.format == PixelFormat::BLOB) { 2084 numStallStream++; 2085 } else { 2086 numProcessedStream++; 2087 } 2088 } 2089 2090 if (numProcessedStream > kMaxProcessedStream) { 2091 ALOGE("%s: too many processed streams (expect <= %d, got %d)", __FUNCTION__, 2092 kMaxProcessedStream, numProcessedStream); 2093 return Status::ILLEGAL_ARGUMENT; 2094 } 2095 2096 if (numStallStream > kMaxStallStream) { 2097 ALOGE("%s: too many stall streams (expect <= %d, got %d)", __FUNCTION__, 2098 kMaxStallStream, numStallStream); 2099 return Status::ILLEGAL_ARGUMENT; 2100 } 2101 2102 Status status = initStatus(); 2103 if (status != Status::OK) { 2104 return status; 2105 } 2106 2107 Mutex::Autolock _l(mLock); 2108 if (!mInflightFrames.empty()) { 2109 ALOGE("%s: trying to configureStreams while there are still %zu inflight frames!", 2110 __FUNCTION__, mInflightFrames.size()); 2111 return Status::INTERNAL_ERROR; 2112 } 2113 2114 // Add new streams 2115 for (const auto& stream : config.streams) { 2116 if (mStreamMap.count(stream.id) == 0) { 2117 mStreamMap[stream.id] = stream; 2118 mCirculatingBuffers.emplace(stream.id, CirculatingBuffers{}); 2119 } 2120 } 2121 2122 // Cleanup removed streams 2123 for(auto it = mStreamMap.begin(); it != mStreamMap.end();) { 2124 int id = it->first; 2125 bool found = false; 2126 for (const auto& stream : config.streams) { 2127 if (id == stream.id) { 2128 found = true; 2129 break; 2130 } 2131 } 2132 if (!found) { 2133 // Unmap all buffers of deleted stream 2134 cleanupBuffersLocked(id); 2135 it = mStreamMap.erase(it); 2136 } else { 2137 ++it; 2138 } 2139 } 2140 2141 // Now select a V4L2 format to produce all output streams 2142 float desiredAr = (mCroppingType == VERTICAL) ? kMaxAspectRatio : kMinAspectRatio; 2143 uint32_t maxDim = 0; 2144 for (const auto& stream : config.streams) { 2145 float aspectRatio = ASPECT_RATIO(stream); 2146 if ((mCroppingType == VERTICAL && aspectRatio < desiredAr) || 2147 (mCroppingType == HORIZONTAL && aspectRatio > desiredAr)) { 2148 desiredAr = aspectRatio; 2149 } 2150 2151 // The dimension that's not cropped 2152 uint32_t dim = (mCroppingType == VERTICAL) ? stream.width : stream.height; 2153 if (dim > maxDim) { 2154 maxDim = dim; 2155 } 2156 } 2157 // Find the smallest format that matches the desired aspect ratio and is wide/high enough 2158 SupportedV4L2Format v4l2Fmt {.width = 0, .height = 0}; 2159 for (const auto& fmt : mSupportedFormats) { 2160 uint32_t dim = (mCroppingType == VERTICAL) ? fmt.width : fmt.height; 2161 if (dim >= maxDim) { 2162 float aspectRatio = ASPECT_RATIO(fmt); 2163 if (isAspectRatioClose(aspectRatio, desiredAr)) { 2164 v4l2Fmt = fmt; 2165 // since mSupportedFormats is sorted by width then height, the first matching fmt 2166 // will be the smallest one with matching aspect ratio 2167 break; 2168 } 2169 } 2170 } 2171 if (v4l2Fmt.width == 0) { 2172 // Cannot find exact good aspect ratio candidate, try to find a close one 2173 for (const auto& fmt : mSupportedFormats) { 2174 uint32_t dim = (mCroppingType == VERTICAL) ? fmt.width : fmt.height; 2175 if (dim >= maxDim) { 2176 float aspectRatio = ASPECT_RATIO(fmt); 2177 if ((mCroppingType == VERTICAL && aspectRatio < desiredAr) || 2178 (mCroppingType == HORIZONTAL && aspectRatio > desiredAr)) { 2179 v4l2Fmt = fmt; 2180 break; 2181 } 2182 } 2183 } 2184 } 2185 2186 if (v4l2Fmt.width == 0) { 2187 ALOGE("%s: unable to find a resolution matching (%s at least %d, aspect ratio %f)" 2188 , __FUNCTION__, (mCroppingType == VERTICAL) ? "width" : "height", 2189 maxDim, desiredAr); 2190 return Status::ILLEGAL_ARGUMENT; 2191 } 2192 2193 if (configureV4l2StreamLocked(v4l2Fmt) != 0) { 2194 ALOGE("V4L configuration failed!, format:%c%c%c%c, w %d, h %d", 2195 v4l2Fmt.fourcc & 0xFF, 2196 (v4l2Fmt.fourcc >> 8) & 0xFF, 2197 (v4l2Fmt.fourcc >> 16) & 0xFF, 2198 (v4l2Fmt.fourcc >> 24) & 0xFF, 2199 v4l2Fmt.width, v4l2Fmt.height); 2200 return Status::INTERNAL_ERROR; 2201 } 2202 2203 Size v4lSize = {v4l2Fmt.width, v4l2Fmt.height}; 2204 Size thumbSize { 0, 0 }; 2205 camera_metadata_ro_entry entry = 2206 mCameraCharacteristics.find(ANDROID_JPEG_AVAILABLE_THUMBNAIL_SIZES); 2207 for(uint32_t i = 0; i < entry.count; i += 2) { 2208 Size sz { static_cast<uint32_t>(entry.data.i32[i]), 2209 static_cast<uint32_t>(entry.data.i32[i+1]) }; 2210 if(sz.width * sz.height > thumbSize.width * thumbSize.height) { 2211 thumbSize = sz; 2212 } 2213 } 2214 2215 if (thumbSize.width * thumbSize.height == 0) { 2216 ALOGE("%s: non-zero thumbnail size not available", __FUNCTION__); 2217 return Status::INTERNAL_ERROR; 2218 } 2219 2220 status = mOutputThread->allocateIntermediateBuffers(v4lSize, 2221 mMaxThumbResolution, config.streams); 2222 if (status != Status::OK) { 2223 ALOGE("%s: allocating intermediate buffers failed!", __FUNCTION__); 2224 return status; 2225 } 2226 2227 out->streams.resize(config.streams.size()); 2228 for (size_t i = 0; i < config.streams.size(); i++) { 2229 out->streams[i].overrideDataSpace = config.streams[i].dataSpace; 2230 out->streams[i].v3_2.id = config.streams[i].id; 2231 // TODO: double check should we add those CAMERA flags 2232 mStreamMap[config.streams[i].id].usage = 2233 out->streams[i].v3_2.producerUsage = config.streams[i].usage | 2234 BufferUsage::CPU_WRITE_OFTEN | 2235 BufferUsage::CAMERA_OUTPUT; 2236 out->streams[i].v3_2.consumerUsage = 0; 2237 out->streams[i].v3_2.maxBuffers = mV4L2BufferCount; 2238 2239 switch (config.streams[i].format) { 2240 case PixelFormat::BLOB: 2241 case PixelFormat::YCBCR_420_888: 2242 case PixelFormat::YV12: // Used by SurfaceTexture 2243 // No override 2244 out->streams[i].v3_2.overrideFormat = config.streams[i].format; 2245 break; 2246 case PixelFormat::IMPLEMENTATION_DEFINED: 2247 // Override based on VIDEO or not 2248 out->streams[i].v3_2.overrideFormat = 2249 (config.streams[i].usage & BufferUsage::VIDEO_ENCODER) ? 2250 PixelFormat::YCBCR_420_888 : PixelFormat::YV12; 2251 // Save overridden formt in mStreamMap 2252 mStreamMap[config.streams[i].id].format = out->streams[i].v3_2.overrideFormat; 2253 break; 2254 default: 2255 ALOGE("%s: unsupported format 0x%x", __FUNCTION__, config.streams[i].format); 2256 return Status::ILLEGAL_ARGUMENT; 2257 } 2258 } 2259 2260 mFirstRequest = true; 2261 return Status::OK; 2262} 2263 2264bool ExternalCameraDeviceSession::isClosed() { 2265 Mutex::Autolock _l(mLock); 2266 return mClosed; 2267} 2268 2269#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) 2270#define UPDATE(md, tag, data, size) \ 2271do { \ 2272 if ((md).update((tag), (data), (size))) { \ 2273 ALOGE("Update " #tag " failed!"); \ 2274 return BAD_VALUE; \ 2275 } \ 2276} while (0) 2277 2278status_t ExternalCameraDeviceSession::initDefaultRequests() { 2279 ::android::hardware::camera::common::V1_0::helper::CameraMetadata md; 2280 2281 const uint8_t aberrationMode = ANDROID_COLOR_CORRECTION_ABERRATION_MODE_OFF; 2282 UPDATE(md, ANDROID_COLOR_CORRECTION_ABERRATION_MODE, &aberrationMode, 1); 2283 2284 const int32_t exposureCompensation = 0; 2285 UPDATE(md, ANDROID_CONTROL_AE_EXPOSURE_COMPENSATION, &exposureCompensation, 1); 2286 2287 const uint8_t videoStabilizationMode = ANDROID_CONTROL_VIDEO_STABILIZATION_MODE_OFF; 2288 UPDATE(md, ANDROID_CONTROL_VIDEO_STABILIZATION_MODE, &videoStabilizationMode, 1); 2289 2290 const uint8_t awbMode = ANDROID_CONTROL_AWB_MODE_AUTO; 2291 UPDATE(md, ANDROID_CONTROL_AWB_MODE, &awbMode, 1); 2292 2293 const uint8_t aeMode = ANDROID_CONTROL_AE_MODE_ON; 2294 UPDATE(md, ANDROID_CONTROL_AE_MODE, &aeMode, 1); 2295 2296 const uint8_t aePrecaptureTrigger = ANDROID_CONTROL_AE_PRECAPTURE_TRIGGER_IDLE; 2297 UPDATE(md, ANDROID_CONTROL_AE_PRECAPTURE_TRIGGER, &aePrecaptureTrigger, 1); 2298 2299 const uint8_t afMode = ANDROID_CONTROL_AF_MODE_AUTO; 2300 UPDATE(md, ANDROID_CONTROL_AF_MODE, &afMode, 1); 2301 2302 const uint8_t afTrigger = ANDROID_CONTROL_AF_TRIGGER_IDLE; 2303 UPDATE(md, ANDROID_CONTROL_AF_TRIGGER, &afTrigger, 1); 2304 2305 const uint8_t sceneMode = ANDROID_CONTROL_SCENE_MODE_DISABLED; 2306 UPDATE(md, ANDROID_CONTROL_SCENE_MODE, &sceneMode, 1); 2307 2308 const uint8_t effectMode = ANDROID_CONTROL_EFFECT_MODE_OFF; 2309 UPDATE(md, ANDROID_CONTROL_EFFECT_MODE, &effectMode, 1); 2310 2311 const uint8_t flashMode = ANDROID_FLASH_MODE_OFF; 2312 UPDATE(md, ANDROID_FLASH_MODE, &flashMode, 1); 2313 2314 const int32_t thumbnailSize[] = {240, 180}; 2315 UPDATE(md, ANDROID_JPEG_THUMBNAIL_SIZE, thumbnailSize, 2); 2316 2317 const uint8_t jpegQuality = 90; 2318 UPDATE(md, ANDROID_JPEG_QUALITY, &jpegQuality, 1); 2319 UPDATE(md, ANDROID_JPEG_THUMBNAIL_QUALITY, &jpegQuality, 1); 2320 2321 const int32_t jpegOrientation = 0; 2322 UPDATE(md, ANDROID_JPEG_ORIENTATION, &jpegOrientation, 1); 2323 2324 const uint8_t oisMode = ANDROID_LENS_OPTICAL_STABILIZATION_MODE_OFF; 2325 UPDATE(md, ANDROID_LENS_OPTICAL_STABILIZATION_MODE, &oisMode, 1); 2326 2327 const uint8_t nrMode = ANDROID_NOISE_REDUCTION_MODE_OFF; 2328 UPDATE(md, ANDROID_NOISE_REDUCTION_MODE, &nrMode, 1); 2329 2330 const uint8_t fdMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; 2331 UPDATE(md, ANDROID_STATISTICS_FACE_DETECT_MODE, &fdMode, 1); 2332 2333 const uint8_t hotpixelMode = ANDROID_STATISTICS_HOT_PIXEL_MAP_MODE_OFF; 2334 UPDATE(md, ANDROID_STATISTICS_HOT_PIXEL_MAP_MODE, &hotpixelMode, 1); 2335 2336 bool support30Fps = false; 2337 int32_t maxFps = std::numeric_limits<int32_t>::min(); 2338 for (const auto& supportedFormat : mSupportedFormats) { 2339 for (const auto& fr : supportedFormat.frameRates) { 2340 int32_t framerateInt = static_cast<int32_t>(fr.getDouble()); 2341 if (maxFps < framerateInt) { 2342 maxFps = framerateInt; 2343 } 2344 if (framerateInt == 30) { 2345 support30Fps = true; 2346 break; 2347 } 2348 } 2349 if (support30Fps) { 2350 break; 2351 } 2352 } 2353 int32_t defaultFramerate = support30Fps ? 30 : maxFps; 2354 int32_t defaultFpsRange[] = {defaultFramerate, defaultFramerate}; 2355 UPDATE(md, ANDROID_CONTROL_AE_TARGET_FPS_RANGE, defaultFpsRange, ARRAY_SIZE(defaultFpsRange)); 2356 2357 uint8_t antibandingMode = ANDROID_CONTROL_AE_ANTIBANDING_MODE_AUTO; 2358 UPDATE(md, ANDROID_CONTROL_AE_ANTIBANDING_MODE, &antibandingMode, 1); 2359 2360 const uint8_t controlMode = ANDROID_CONTROL_MODE_AUTO; 2361 UPDATE(md, ANDROID_CONTROL_MODE, &controlMode, 1); 2362 2363 auto requestTemplates = hidl_enum_iterator<RequestTemplate>(); 2364 for (RequestTemplate type : requestTemplates) { 2365 ::android::hardware::camera::common::V1_0::helper::CameraMetadata mdCopy = md; 2366 uint8_t intent = ANDROID_CONTROL_CAPTURE_INTENT_PREVIEW; 2367 switch (type) { 2368 case RequestTemplate::PREVIEW: 2369 intent = ANDROID_CONTROL_CAPTURE_INTENT_PREVIEW; 2370 break; 2371 case RequestTemplate::STILL_CAPTURE: 2372 intent = ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE; 2373 break; 2374 case RequestTemplate::VIDEO_RECORD: 2375 intent = ANDROID_CONTROL_CAPTURE_INTENT_VIDEO_RECORD; 2376 break; 2377 case RequestTemplate::VIDEO_SNAPSHOT: 2378 intent = ANDROID_CONTROL_CAPTURE_INTENT_VIDEO_SNAPSHOT; 2379 break; 2380 default: 2381 ALOGV("%s: unsupported RequestTemplate type %d", __FUNCTION__, type); 2382 continue; 2383 } 2384 UPDATE(mdCopy, ANDROID_CONTROL_CAPTURE_INTENT, &intent, 1); 2385 2386 camera_metadata_t* rawMd = mdCopy.release(); 2387 CameraMetadata hidlMd; 2388 hidlMd.setToExternal( 2389 (uint8_t*) rawMd, get_camera_metadata_size(rawMd)); 2390 mDefaultRequests[type] = hidlMd; 2391 free_camera_metadata(rawMd); 2392 } 2393 2394 return OK; 2395} 2396 2397status_t ExternalCameraDeviceSession::fillCaptureResult( 2398 common::V1_0::helper::CameraMetadata &md, nsecs_t timestamp) { 2399 // android.control 2400 // For USB camera, we don't know the AE state. Set the state to converged to 2401 // indicate the frame should be good to use. Then apps don't have to wait the 2402 // AE state. 2403 const uint8_t aeState = ANDROID_CONTROL_AE_STATE_CONVERGED; 2404 UPDATE(md, ANDROID_CONTROL_AE_STATE, &aeState, 1); 2405 2406 const uint8_t ae_lock = ANDROID_CONTROL_AE_LOCK_OFF; 2407 UPDATE(md, ANDROID_CONTROL_AE_LOCK, &ae_lock, 1); 2408 2409 bool afTrigger = mAfTrigger; 2410 if (md.exists(ANDROID_CONTROL_AF_TRIGGER)) { 2411 Mutex::Autolock _l(mLock); 2412 camera_metadata_entry entry = md.find(ANDROID_CONTROL_AF_TRIGGER); 2413 if (entry.data.u8[0] == ANDROID_CONTROL_AF_TRIGGER_START) { 2414 mAfTrigger = afTrigger = true; 2415 } else if (entry.data.u8[0] == ANDROID_CONTROL_AF_TRIGGER_CANCEL) { 2416 mAfTrigger = afTrigger = false; 2417 } 2418 } 2419 2420 // For USB camera, the USB camera handles everything and we don't have control 2421 // over AF. We only simply fake the AF metadata based on the request 2422 // received here. 2423 uint8_t afState; 2424 if (afTrigger) { 2425 afState = ANDROID_CONTROL_AF_STATE_FOCUSED_LOCKED; 2426 } else { 2427 afState = ANDROID_CONTROL_AF_STATE_INACTIVE; 2428 } 2429 UPDATE(md, ANDROID_CONTROL_AF_STATE, &afState, 1); 2430 2431 // Set AWB state to converged to indicate the frame should be good to use. 2432 const uint8_t awbState = ANDROID_CONTROL_AWB_STATE_CONVERGED; 2433 UPDATE(md, ANDROID_CONTROL_AWB_STATE, &awbState, 1); 2434 2435 const uint8_t awbLock = ANDROID_CONTROL_AWB_LOCK_OFF; 2436 UPDATE(md, ANDROID_CONTROL_AWB_LOCK, &awbLock, 1); 2437 2438 camera_metadata_ro_entry active_array_size = 2439 mCameraCharacteristics.find(ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE); 2440 2441 if (active_array_size.count == 0) { 2442 ALOGE("%s: cannot find active array size!", __FUNCTION__); 2443 return -EINVAL; 2444 } 2445 2446 const uint8_t flashState = ANDROID_FLASH_STATE_UNAVAILABLE; 2447 UPDATE(md, ANDROID_FLASH_STATE, &flashState, 1); 2448 2449 // android.scaler 2450 const int32_t crop_region[] = { 2451 active_array_size.data.i32[0], active_array_size.data.i32[1], 2452 active_array_size.data.i32[2], active_array_size.data.i32[3], 2453 }; 2454 UPDATE(md, ANDROID_SCALER_CROP_REGION, crop_region, ARRAY_SIZE(crop_region)); 2455 2456 // android.sensor 2457 UPDATE(md, ANDROID_SENSOR_TIMESTAMP, ×tamp, 1); 2458 2459 // android.statistics 2460 const uint8_t lensShadingMapMode = ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF; 2461 UPDATE(md, ANDROID_STATISTICS_LENS_SHADING_MAP_MODE, &lensShadingMapMode, 1); 2462 2463 const uint8_t sceneFlicker = ANDROID_STATISTICS_SCENE_FLICKER_NONE; 2464 UPDATE(md, ANDROID_STATISTICS_SCENE_FLICKER, &sceneFlicker, 1); 2465 2466 return OK; 2467} 2468 2469#undef ARRAY_SIZE 2470#undef UPDATE 2471 2472} // namespace implementation 2473} // namespace V3_4 2474} // namespace device 2475} // namespace camera 2476} // namespace hardware 2477} // namespace android 2478