RequestThreadManager.java revision f4a637d0be2e028d1e78c8bf90ad17ec3f84b5f3
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.hardware.camera2.legacy; 18 19import android.graphics.ImageFormat; 20import android.graphics.SurfaceTexture; 21import android.hardware.Camera; 22import android.hardware.camera2.CameraCharacteristics; 23import android.hardware.camera2.CaptureRequest; 24import android.hardware.camera2.impl.CameraDeviceImpl; 25import android.hardware.camera2.params.StreamConfigurationMap; 26import android.hardware.camera2.utils.LongParcelable; 27import android.hardware.camera2.utils.SizeAreaComparator; 28import android.hardware.camera2.impl.CameraMetadataNative; 29import android.os.ConditionVariable; 30import android.os.Handler; 31import android.os.Message; 32import android.os.SystemClock; 33import android.util.Log; 34import android.util.MutableLong; 35import android.util.Pair; 36import android.util.Size; 37import android.view.Surface; 38 39import java.io.IOException; 40import java.util.ArrayList; 41import java.util.Arrays; 42import java.util.Collection; 43import java.util.Collections; 44import java.util.Iterator; 45import java.util.List; 46import java.util.concurrent.TimeUnit; 47 48import static com.android.internal.util.Preconditions.*; 49 50/** 51 * This class executes requests to the {@link Camera}. 52 * 53 * <p> 54 * The main components of this class are: 55 * - A message queue of requests to the {@link Camera}. 56 * - A thread that consumes requests to the {@link Camera} and executes them. 57 * - A {@link GLThreadManager} that draws to the configured output {@link Surface}s. 58 * - An {@link CameraDeviceState} state machine that manages the callbacks for various operations. 59 * </p> 60 */ 61@SuppressWarnings("deprecation") 62public class RequestThreadManager { 63 private final String TAG; 64 private final int mCameraId; 65 private final RequestHandlerThread mRequestThread; 66 67 private static final boolean DEBUG = Log.isLoggable(LegacyCameraDevice.DEBUG_PROP, Log.DEBUG); 68 // For slightly more spammy messages that will get repeated every frame 69 private static final boolean VERBOSE = 70 Log.isLoggable(LegacyCameraDevice.DEBUG_PROP, Log.VERBOSE); 71 private final Camera mCamera; 72 private final CameraCharacteristics mCharacteristics; 73 74 private final CameraDeviceState mDeviceState; 75 private final CaptureCollector mCaptureCollector; 76 private final LegacyFocusStateMapper mFocusStateMapper; 77 private final LegacyFaceDetectMapper mFaceDetectMapper; 78 79 private static final int MSG_CONFIGURE_OUTPUTS = 1; 80 private static final int MSG_SUBMIT_CAPTURE_REQUEST = 2; 81 private static final int MSG_CLEANUP = 3; 82 83 private static final int MAX_IN_FLIGHT_REQUESTS = 2; 84 85 private static final int PREVIEW_FRAME_TIMEOUT = 1000; // ms 86 private static final int JPEG_FRAME_TIMEOUT = 3000; // ms (same as CTS for API2) 87 private static final int REQUEST_COMPLETE_TIMEOUT = 3000; // ms (same as JPEG timeout) 88 89 private static final float ASPECT_RATIO_TOLERANCE = 0.01f; 90 private boolean mPreviewRunning = false; 91 92 private final List<Surface> mPreviewOutputs = new ArrayList<>(); 93 private final List<Surface> mCallbackOutputs = new ArrayList<>(); 94 private GLThreadManager mGLThreadManager; 95 private SurfaceTexture mPreviewTexture; 96 private Camera.Parameters mParams; 97 98 private final List<Long> mJpegSurfaceIds = new ArrayList<>(); 99 100 private Size mIntermediateBufferSize; 101 102 private final RequestQueue mRequestQueue = new RequestQueue(mJpegSurfaceIds); 103 private LegacyRequest mLastRequest = null; 104 private SurfaceTexture mDummyTexture; 105 private Surface mDummySurface; 106 107 private final Object mIdleLock = new Object(); 108 private final FpsCounter mPrevCounter = new FpsCounter("Incoming Preview"); 109 private final FpsCounter mRequestCounter = new FpsCounter("Incoming Requests"); 110 111 // Stuff JPEGs into HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers to get around SW write 112 // limitations for (b/17379185). 113 private static final boolean USE_BLOB_FORMAT_OVERRIDE = true; 114 115 /** 116 * Container object for Configure messages. 117 */ 118 private static class ConfigureHolder { 119 public final ConditionVariable condition; 120 public final Collection<Pair<Surface, Size>> surfaces; 121 122 public ConfigureHolder(ConditionVariable condition, Collection<Pair<Surface, 123 Size>> surfaces) { 124 this.condition = condition; 125 this.surfaces = surfaces; 126 } 127 } 128 129 /** 130 * Counter class used to calculate and log the current FPS of frame production. 131 */ 132 public static class FpsCounter { 133 //TODO: Hook this up to SystTrace? 134 private static final String TAG = "FpsCounter"; 135 private int mFrameCount = 0; 136 private long mLastTime = 0; 137 private long mLastPrintTime = 0; 138 private double mLastFps = 0; 139 private final String mStreamType; 140 private static final long NANO_PER_SECOND = 1000000000; //ns 141 142 public FpsCounter(String streamType) { 143 mStreamType = streamType; 144 } 145 146 public synchronized void countFrame() { 147 mFrameCount++; 148 long nextTime = SystemClock.elapsedRealtimeNanos(); 149 if (mLastTime == 0) { 150 mLastTime = nextTime; 151 } 152 if (nextTime > mLastTime + NANO_PER_SECOND) { 153 long elapsed = nextTime - mLastTime; 154 mLastFps = mFrameCount * (NANO_PER_SECOND / (double) elapsed); 155 mFrameCount = 0; 156 mLastTime = nextTime; 157 } 158 } 159 160 public synchronized double checkFps() { 161 return mLastFps; 162 } 163 164 public synchronized void staggeredLog() { 165 if (mLastTime > mLastPrintTime + 5 * NANO_PER_SECOND) { 166 mLastPrintTime = mLastTime; 167 Log.d(TAG, "FPS for " + mStreamType + " stream: " + mLastFps ); 168 } 169 } 170 171 public synchronized void countAndLog() { 172 countFrame(); 173 staggeredLog(); 174 } 175 } 176 /** 177 * Fake preview for jpeg captures when there is no active preview 178 */ 179 private void createDummySurface() { 180 if (mDummyTexture == null || mDummySurface == null) { 181 mDummyTexture = new SurfaceTexture(/*ignored*/0); 182 // TODO: use smallest default sizes 183 mDummyTexture.setDefaultBufferSize(640, 480); 184 mDummySurface = new Surface(mDummyTexture); 185 } 186 } 187 188 private final Camera.ErrorCallback mErrorCallback = new Camera.ErrorCallback() { 189 @Override 190 public void onError(int i, Camera camera) { 191 Log.e(TAG, "Received error " + i + " from the Camera1 ErrorCallback"); 192 mDeviceState.setError(CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 193 } 194 }; 195 196 private final ConditionVariable mReceivedJpeg = new ConditionVariable(false); 197 198 private final Camera.PictureCallback mJpegCallback = new Camera.PictureCallback() { 199 @Override 200 public void onPictureTaken(byte[] data, Camera camera) { 201 Log.i(TAG, "Received jpeg."); 202 Pair<RequestHolder, Long> captureInfo = mCaptureCollector.jpegProduced(); 203 if (captureInfo == null || captureInfo.first == null) { 204 Log.e(TAG, "Dropping jpeg frame."); 205 return; 206 } 207 RequestHolder holder = captureInfo.first; 208 long timestamp = captureInfo.second; 209 for (Surface s : holder.getHolderTargets()) { 210 try { 211 if (LegacyCameraDevice.containsSurfaceId(s, mJpegSurfaceIds)) { 212 Log.i(TAG, "Producing jpeg buffer..."); 213 214 int totalSize = data.length + LegacyCameraDevice.nativeGetJpegFooterSize(); 215 totalSize = (totalSize + 3) & ~0x3; // round up to nearest octonibble 216 LegacyCameraDevice.setNextTimestamp(s, timestamp); 217 218 if (USE_BLOB_FORMAT_OVERRIDE) { 219 // Override to RGBA_8888 format. 220 LegacyCameraDevice.setSurfaceFormat(s, 221 LegacyMetadataMapper.HAL_PIXEL_FORMAT_RGBA_8888); 222 223 int dimen = (int) Math.ceil(Math.sqrt(totalSize)); 224 dimen = (dimen + 0xf) & ~0xf; // round up to nearest multiple of 16 225 LegacyCameraDevice.setSurfaceDimens(s, dimen, dimen); 226 LegacyCameraDevice.produceFrame(s, data, dimen, dimen, 227 CameraMetadataNative.NATIVE_JPEG_FORMAT); 228 } else { 229 LegacyCameraDevice.setSurfaceDimens(s, totalSize, /*height*/1); 230 LegacyCameraDevice.produceFrame(s, data, totalSize, /*height*/1, 231 CameraMetadataNative.NATIVE_JPEG_FORMAT); 232 } 233 } 234 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 235 Log.w(TAG, "Surface abandoned, dropping frame. ", e); 236 } 237 } 238 239 mReceivedJpeg.open(); 240 } 241 }; 242 243 private final Camera.ShutterCallback mJpegShutterCallback = new Camera.ShutterCallback() { 244 @Override 245 public void onShutter() { 246 mCaptureCollector.jpegCaptured(SystemClock.elapsedRealtimeNanos()); 247 } 248 }; 249 250 private final SurfaceTexture.OnFrameAvailableListener mPreviewCallback = 251 new SurfaceTexture.OnFrameAvailableListener() { 252 @Override 253 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 254 if (DEBUG) { 255 mPrevCounter.countAndLog(); 256 } 257 mGLThreadManager.queueNewFrame(); 258 } 259 }; 260 261 private void stopPreview() { 262 if (VERBOSE) { 263 Log.v(TAG, "stopPreview - preview running? " + mPreviewRunning); 264 } 265 if (mPreviewRunning) { 266 mCamera.stopPreview(); 267 mPreviewRunning = false; 268 } 269 } 270 271 private void startPreview() { 272 if (VERBOSE) { 273 Log.v(TAG, "startPreview - preview running? " + mPreviewRunning); 274 } 275 if (!mPreviewRunning) { 276 // XX: CameraClient:;startPreview is not getting called after a stop 277 mCamera.startPreview(); 278 mPreviewRunning = true; 279 } 280 } 281 282 private void doJpegCapturePrepare(RequestHolder request) throws IOException { 283 if (DEBUG) Log.d(TAG, "doJpegCapturePrepare - preview running? " + mPreviewRunning); 284 285 if (!mPreviewRunning) { 286 if (DEBUG) Log.d(TAG, "doJpegCapture - create fake surface"); 287 288 createDummySurface(); 289 mCamera.setPreviewTexture(mDummyTexture); 290 startPreview(); 291 } 292 } 293 294 private void doJpegCapture(RequestHolder request) { 295 if (DEBUG) Log.d(TAG, "doJpegCapturePrepare"); 296 297 mCamera.takePicture(mJpegShutterCallback, /*raw*/null, mJpegCallback); 298 mPreviewRunning = false; 299 } 300 301 private void doPreviewCapture(RequestHolder request) throws IOException { 302 if (VERBOSE) { 303 Log.v(TAG, "doPreviewCapture - preview running? " + mPreviewRunning); 304 } 305 306 if (mPreviewRunning) { 307 return; // Already running 308 } 309 310 if (mPreviewTexture == null) { 311 throw new IllegalStateException( 312 "Preview capture called with no preview surfaces configured."); 313 } 314 315 mPreviewTexture.setDefaultBufferSize(mIntermediateBufferSize.getWidth(), 316 mIntermediateBufferSize.getHeight()); 317 mCamera.setPreviewTexture(mPreviewTexture); 318 319 startPreview(); 320 } 321 322 private void configureOutputs(Collection<Pair<Surface, Size>> outputs) { 323 if (DEBUG) { 324 String outputsStr = outputs == null ? "null" : (outputs.size() + " surfaces"); 325 Log.d(TAG, "configureOutputs with " + outputsStr); 326 } 327 328 stopPreview(); 329 /* 330 * Try to release the previous preview's surface texture earlier if we end up 331 * using a different one; this also reduces the likelihood of getting into a deadlock 332 * when disconnecting from the old previous texture at a later time. 333 */ 334 try { 335 mCamera.setPreviewTexture(/*surfaceTexture*/null); 336 } catch (IOException e) { 337 Log.w(TAG, "Failed to clear prior SurfaceTexture, may cause GL deadlock: ", e); 338 } 339 340 if (mGLThreadManager != null) { 341 mGLThreadManager.waitUntilStarted(); 342 mGLThreadManager.ignoreNewFrames(); 343 mGLThreadManager.waitUntilIdle(); 344 } 345 resetJpegSurfaceFormats(mCallbackOutputs); 346 mPreviewOutputs.clear(); 347 mCallbackOutputs.clear(); 348 mJpegSurfaceIds.clear(); 349 mPreviewTexture = null; 350 351 List<Size> previewOutputSizes = new ArrayList<>(); 352 List<Size> callbackOutputSizes = new ArrayList<>(); 353 354 int facing = mCharacteristics.get(CameraCharacteristics.LENS_FACING); 355 int orientation = mCharacteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); 356 if (outputs != null) { 357 for (Pair<Surface, Size> outPair : outputs) { 358 Surface s = outPair.first; 359 Size outSize = outPair.second; 360 try { 361 int format = LegacyCameraDevice.detectSurfaceType(s); 362 LegacyCameraDevice.setSurfaceOrientation(s, facing, orientation); 363 switch (format) { 364 case CameraMetadataNative.NATIVE_JPEG_FORMAT: 365 if (USE_BLOB_FORMAT_OVERRIDE) { 366 // Override to RGBA_8888 format. 367 LegacyCameraDevice.setSurfaceFormat(s, 368 LegacyMetadataMapper.HAL_PIXEL_FORMAT_RGBA_8888); 369 } 370 mJpegSurfaceIds.add(LegacyCameraDevice.getSurfaceId(s)); 371 mCallbackOutputs.add(s); 372 callbackOutputSizes.add(outSize); 373 break; 374 default: 375 mPreviewOutputs.add(s); 376 previewOutputSizes.add(outSize); 377 break; 378 } 379 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 380 Log.w(TAG, "Surface abandoned, skipping...", e); 381 } 382 } 383 } 384 try { 385 mParams = mCamera.getParameters(); 386 } catch (RuntimeException e) { 387 Log.e(TAG, "Received device exception: ", e); 388 mDeviceState.setError( 389 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 390 return; 391 } 392 393 List<int[]> supportedFpsRanges = mParams.getSupportedPreviewFpsRange(); 394 int[] bestRange = getPhotoPreviewFpsRange(supportedFpsRanges); 395 if (DEBUG) { 396 Log.d(TAG, "doPreviewCapture - Selected range [" + 397 bestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] + "," + 398 bestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] + "]"); 399 } 400 mParams.setPreviewFpsRange(bestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], 401 bestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); 402 403 if (previewOutputSizes.size() > 0) { 404 405 Size largestOutput = SizeAreaComparator.findLargestByArea(previewOutputSizes); 406 407 // Find largest jpeg dimension - assume to have the same aspect ratio as sensor. 408 Size largestJpegDimen = ParameterUtils.getLargestSupportedJpegSizeByArea(mParams); 409 410 List<Size> supportedPreviewSizes = ParameterUtils.convertSizeList( 411 mParams.getSupportedPreviewSizes()); 412 413 // Use smallest preview dimension with same aspect ratio as sensor that is >= than all 414 // of the configured output dimensions. If none exists, fall back to using the largest 415 // supported preview size. 416 long largestOutputArea = largestOutput.getHeight() * (long) largestOutput.getWidth(); 417 Size bestPreviewDimen = SizeAreaComparator.findLargestByArea(supportedPreviewSizes); 418 for (Size s : supportedPreviewSizes) { 419 long currArea = s.getWidth() * s.getHeight(); 420 long bestArea = bestPreviewDimen.getWidth() * bestPreviewDimen.getHeight(); 421 if (checkAspectRatiosMatch(largestJpegDimen, s) && (currArea < bestArea && 422 currArea >= largestOutputArea)) { 423 bestPreviewDimen = s; 424 } 425 } 426 427 mIntermediateBufferSize = bestPreviewDimen; 428 mParams.setPreviewSize(mIntermediateBufferSize.getWidth(), 429 mIntermediateBufferSize.getHeight()); 430 431 if (DEBUG) { 432 Log.d(TAG, "Intermediate buffer selected with dimens: " + 433 bestPreviewDimen.toString()); 434 } 435 } else { 436 mIntermediateBufferSize = null; 437 if (DEBUG) { 438 Log.d(TAG, "No Intermediate buffer selected, no preview outputs were configured"); 439 } 440 } 441 442 Size smallestSupportedJpegSize = calculatePictureSize(mCallbackOutputs, 443 callbackOutputSizes, mParams); 444 if (smallestSupportedJpegSize != null) { 445 /* 446 * Set takePicture size to the smallest supported JPEG size large enough 447 * to scale/crop out of for the bounding rectangle of the configured JPEG sizes. 448 */ 449 450 Log.i(TAG, "configureOutputs - set take picture size to " + smallestSupportedJpegSize); 451 mParams.setPictureSize( 452 smallestSupportedJpegSize.getWidth(), smallestSupportedJpegSize.getHeight()); 453 } 454 455 // TODO: Detect and optimize single-output paths here to skip stream teeing. 456 if (mGLThreadManager == null) { 457 mGLThreadManager = new GLThreadManager(mCameraId, facing, mDeviceState); 458 mGLThreadManager.start(); 459 } 460 mGLThreadManager.waitUntilStarted(); 461 List<Pair<Surface, Size>> previews = new ArrayList<>(); 462 Iterator<Size> previewSizeIter = previewOutputSizes.iterator(); 463 for (Surface p : mPreviewOutputs) { 464 previews.add(new Pair<>(p, previewSizeIter.next())); 465 } 466 mGLThreadManager.setConfigurationAndWait(previews, mCaptureCollector); 467 mGLThreadManager.allowNewFrames(); 468 mPreviewTexture = mGLThreadManager.getCurrentSurfaceTexture(); 469 if (mPreviewTexture != null) { 470 mPreviewTexture.setOnFrameAvailableListener(mPreviewCallback); 471 } 472 473 mCamera.setParameters(mParams); 474 } 475 476 private void resetJpegSurfaceFormats(Collection<Surface> surfaces) { 477 if (!USE_BLOB_FORMAT_OVERRIDE || surfaces == null) { 478 return; 479 } 480 for(Surface s : surfaces) { 481 try { 482 LegacyCameraDevice.setSurfaceFormat(s, LegacyMetadataMapper.HAL_PIXEL_FORMAT_BLOB); 483 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 484 Log.w(TAG, "Surface abandoned, skipping...", e); 485 } 486 } 487 } 488 489 /** 490 * Find a JPEG size (that is supported by the legacy camera device) which is equal to or larger 491 * than all of the configured {@code JPEG} outputs (by both width and height). 492 * 493 * <p>If multiple supported JPEG sizes are larger, select the smallest of them which 494 * still satisfies the above constraint.</p> 495 * 496 * <p>As a result, the returned size is guaranteed to be usable without needing 497 * to upscale any of the outputs. If only one {@code JPEG} surface is used, 498 * then no scaling/cropping is necessary between the taken picture and 499 * the {@code JPEG} output surface.</p> 500 * 501 * @param callbackOutputs a non-{@code null} list of {@code Surface}s with any image formats 502 * @param params api1 parameters (used for reading only) 503 * 504 * @return a size large enough to fit all of the configured {@code JPEG} outputs, or 505 * {@code null} if the {@code callbackOutputs} did not have any {@code JPEG} 506 * surfaces. 507 */ 508 private Size calculatePictureSize( List<Surface> callbackOutputs, 509 List<Size> callbackSizes, Camera.Parameters params) { 510 /* 511 * Find the largest JPEG size (if any), from the configured outputs: 512 * - the api1 picture size should be set to the smallest legal size that's at least as large 513 * as the largest configured JPEG size 514 */ 515 if (callbackOutputs.size() != callbackSizes.size()) { 516 throw new IllegalStateException("Input collections must be same length"); 517 } 518 List<Size> configuredJpegSizes = new ArrayList<>(); 519 Iterator<Size> sizeIterator = callbackSizes.iterator(); 520 for (Surface callbackSurface : callbackOutputs) { 521 Size jpegSize = sizeIterator.next(); 522 if (!LegacyCameraDevice.containsSurfaceId(callbackSurface, mJpegSurfaceIds)) { 523 continue; // Ignore non-JPEG callback formats 524 } 525 526 configuredJpegSizes.add(jpegSize); 527 } 528 if (!configuredJpegSizes.isEmpty()) { 529 /* 530 * Find the largest configured JPEG width, and height, independently 531 * of the rest. 532 * 533 * The rest of the JPEG streams can be cropped out of this smallest bounding 534 * rectangle. 535 */ 536 int maxConfiguredJpegWidth = -1; 537 int maxConfiguredJpegHeight = -1; 538 for (Size jpegSize : configuredJpegSizes) { 539 maxConfiguredJpegWidth = jpegSize.getWidth() > maxConfiguredJpegWidth ? 540 jpegSize.getWidth() : maxConfiguredJpegWidth; 541 maxConfiguredJpegHeight = jpegSize.getHeight() > maxConfiguredJpegHeight ? 542 jpegSize.getHeight() : maxConfiguredJpegHeight; 543 } 544 Size smallestBoundJpegSize = new Size(maxConfiguredJpegWidth, maxConfiguredJpegHeight); 545 546 List<Size> supportedJpegSizes = ParameterUtils.convertSizeList( 547 params.getSupportedPictureSizes()); 548 549 /* 550 * Find the smallest supported JPEG size that can fit the smallest bounding 551 * rectangle for the configured JPEG sizes. 552 */ 553 List<Size> candidateSupportedJpegSizes = new ArrayList<>(); 554 for (Size supportedJpegSize : supportedJpegSizes) { 555 if (supportedJpegSize.getWidth() >= maxConfiguredJpegWidth && 556 supportedJpegSize.getHeight() >= maxConfiguredJpegHeight) { 557 candidateSupportedJpegSizes.add(supportedJpegSize); 558 } 559 } 560 561 if (candidateSupportedJpegSizes.isEmpty()) { 562 throw new AssertionError( 563 "Could not find any supported JPEG sizes large enough to fit " + 564 smallestBoundJpegSize); 565 } 566 567 Size smallestSupportedJpegSize = Collections.min(candidateSupportedJpegSizes, 568 new SizeAreaComparator()); 569 570 if (!smallestSupportedJpegSize.equals(smallestBoundJpegSize)) { 571 Log.w(TAG, 572 String.format( 573 "configureOutputs - Will need to crop picture %s into " 574 + "smallest bound size %s", 575 smallestSupportedJpegSize, smallestBoundJpegSize)); 576 } 577 578 return smallestSupportedJpegSize; 579 } 580 581 return null; 582 } 583 584 private static boolean checkAspectRatiosMatch(Size a, Size b) { 585 float aAspect = a.getWidth() / (float) a.getHeight(); 586 float bAspect = b.getWidth() / (float) b.getHeight(); 587 588 return Math.abs(aAspect - bAspect) < ASPECT_RATIO_TOLERANCE; 589 } 590 591 // Calculate the highest FPS range supported 592 private int[] getPhotoPreviewFpsRange(List<int[]> frameRates) { 593 if (frameRates.size() == 0) { 594 Log.e(TAG, "No supported frame rates returned!"); 595 return null; 596 } 597 598 int bestMin = 0; 599 int bestMax = 0; 600 int bestIndex = 0; 601 int index = 0; 602 for (int[] rate : frameRates) { 603 int minFps = rate[Camera.Parameters.PREVIEW_FPS_MIN_INDEX]; 604 int maxFps = rate[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]; 605 if (maxFps > bestMax || (maxFps == bestMax && minFps > bestMin)) { 606 bestMin = minFps; 607 bestMax = maxFps; 608 bestIndex = index; 609 } 610 index++; 611 } 612 613 return frameRates.get(bestIndex); 614 } 615 616 private final Handler.Callback mRequestHandlerCb = new Handler.Callback() { 617 private boolean mCleanup = false; 618 private final LegacyResultMapper mMapper = new LegacyResultMapper(); 619 620 @Override 621 public boolean handleMessage(Message msg) { 622 if (mCleanup) { 623 return true; 624 } 625 626 if (DEBUG) { 627 Log.d(TAG, "Request thread handling message:" + msg.what); 628 } 629 long startTime = 0; 630 if (DEBUG) { 631 startTime = SystemClock.elapsedRealtimeNanos(); 632 } 633 switch (msg.what) { 634 case MSG_CONFIGURE_OUTPUTS: 635 ConfigureHolder config = (ConfigureHolder) msg.obj; 636 int sizes = config.surfaces != null ? config.surfaces.size() : 0; 637 Log.i(TAG, "Configure outputs: " + sizes + " surfaces configured."); 638 639 try { 640 boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT, 641 TimeUnit.MILLISECONDS); 642 if (!success) { 643 Log.e(TAG, "Timed out while queueing configure request."); 644 mCaptureCollector.failAll(); 645 } 646 } catch (InterruptedException e) { 647 Log.e(TAG, "Interrupted while waiting for requests to complete."); 648 mDeviceState.setError( 649 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 650 break; 651 } 652 653 configureOutputs(config.surfaces); 654 config.condition.open(); 655 if (DEBUG) { 656 long totalTime = SystemClock.elapsedRealtimeNanos() - startTime; 657 Log.d(TAG, "Configure took " + totalTime + " ns"); 658 } 659 break; 660 case MSG_SUBMIT_CAPTURE_REQUEST: 661 Handler handler = RequestThreadManager.this.mRequestThread.getHandler(); 662 663 // Get the next burst from the request queue. 664 Pair<BurstHolder, Long> nextBurst = mRequestQueue.getNext(); 665 666 if (nextBurst == null) { 667 // If there are no further requests queued, wait for any currently executing 668 // requests to complete, then switch to idle state. 669 try { 670 boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT, 671 TimeUnit.MILLISECONDS); 672 if (!success) { 673 Log.e(TAG, 674 "Timed out while waiting for prior requests to complete."); 675 mCaptureCollector.failAll(); 676 } 677 } catch (InterruptedException e) { 678 Log.e(TAG, "Interrupted while waiting for requests to complete: ", e); 679 mDeviceState.setError( 680 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 681 break; 682 } 683 684 synchronized (mIdleLock) { 685 // Retry the the request queue. 686 nextBurst = mRequestQueue.getNext(); 687 688 // If we still have no queued requests, go idle. 689 if (nextBurst == null) { 690 mDeviceState.setIdle(); 691 break; 692 } 693 } 694 } 695 696 if (nextBurst != null) { 697 // Queue another capture if we did not get the last burst. 698 handler.sendEmptyMessage(MSG_SUBMIT_CAPTURE_REQUEST); 699 } 700 701 // Complete each request in the burst 702 List<RequestHolder> requests = 703 nextBurst.first.produceRequestHolders(nextBurst.second); 704 for (RequestHolder holder : requests) { 705 CaptureRequest request = holder.getRequest(); 706 707 boolean paramsChanged = false; 708 709 // Only update parameters if the request has changed 710 if (mLastRequest == null || mLastRequest.captureRequest != request) { 711 712 // The intermediate buffer is sometimes null, but we always need 713 // the Camera1 API configured preview size 714 Size previewSize = ParameterUtils.convertSize(mParams.getPreviewSize()); 715 716 LegacyRequest legacyRequest = new LegacyRequest(mCharacteristics, 717 request, previewSize, mParams); // params are copied 718 719 720 // Parameters are mutated as a side-effect 721 LegacyMetadataMapper.convertRequestMetadata(/*inout*/legacyRequest); 722 723 // If the parameters have changed, set them in the Camera1 API. 724 if (!mParams.same(legacyRequest.parameters)) { 725 try { 726 mCamera.setParameters(legacyRequest.parameters); 727 } catch (RuntimeException e) { 728 // If setting the parameters failed, report a request error to 729 // the camera client, and skip any further work for this request 730 Log.e(TAG, "Exception while setting camera parameters: ", e); 731 holder.failRequest(); 732 mDeviceState.setCaptureStart(holder, /*timestamp*/0, 733 CameraDeviceImpl.CameraDeviceCallbacks. 734 ERROR_CAMERA_REQUEST); 735 continue; 736 } 737 paramsChanged = true; 738 mParams = legacyRequest.parameters; 739 } 740 741 mLastRequest = legacyRequest; 742 } 743 744 try { 745 boolean success = mCaptureCollector.queueRequest(holder, 746 mLastRequest, JPEG_FRAME_TIMEOUT, TimeUnit.MILLISECONDS); 747 748 if (!success) { 749 // Report a request error if we timed out while queuing this. 750 Log.e(TAG, "Timed out while queueing capture request."); 751 holder.failRequest(); 752 mDeviceState.setCaptureStart(holder, /*timestamp*/0, 753 CameraDeviceImpl.CameraDeviceCallbacks. 754 ERROR_CAMERA_REQUEST); 755 continue; 756 } 757 758 // Starting the preview needs to happen before enabling 759 // face detection or auto focus 760 if (holder.hasPreviewTargets()) { 761 doPreviewCapture(holder); 762 } 763 if (holder.hasJpegTargets()) { 764 while(!mCaptureCollector.waitForPreviewsEmpty(PREVIEW_FRAME_TIMEOUT, 765 TimeUnit.MILLISECONDS)) { 766 // Fail preview requests until the queue is empty. 767 Log.e(TAG, "Timed out while waiting for preview requests to " + 768 "complete."); 769 mCaptureCollector.failNextPreview(); 770 } 771 mReceivedJpeg.close(); 772 doJpegCapturePrepare(holder); 773 } 774 775 /* 776 * Do all the actions that require a preview to have been started 777 */ 778 779 // Toggle face detection on/off 780 // - do this before AF to give AF a chance to use faces 781 mFaceDetectMapper.processFaceDetectMode(request, /*in*/mParams); 782 783 // Unconditionally process AF triggers, since they're non-idempotent 784 // - must be done after setting the most-up-to-date AF mode 785 mFocusStateMapper.processRequestTriggers(request, mParams); 786 787 if (holder.hasJpegTargets()) { 788 doJpegCapture(holder); 789 if (!mReceivedJpeg.block(JPEG_FRAME_TIMEOUT)) { 790 Log.e(TAG, "Hit timeout for jpeg callback!"); 791 mCaptureCollector.failNextJpeg(); 792 } 793 } 794 795 } catch (IOException e) { 796 Log.e(TAG, "Received device exception: ", e); 797 mDeviceState.setError( 798 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 799 break; 800 } catch (InterruptedException e) { 801 Log.e(TAG, "Interrupted during capture: ", e); 802 mDeviceState.setError( 803 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 804 break; 805 } 806 807 if (paramsChanged) { 808 if (DEBUG) { 809 Log.d(TAG, "Params changed -- getting new Parameters from HAL."); 810 } 811 try { 812 mParams = mCamera.getParameters(); 813 } catch (RuntimeException e) { 814 Log.e(TAG, "Received device exception: ", e); 815 mDeviceState.setError( 816 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 817 break; 818 } 819 820 // Update parameters to the latest that we think the camera is using 821 mLastRequest.setParameters(mParams); 822 } 823 824 MutableLong timestampMutable = new MutableLong(/*value*/0L); 825 try { 826 boolean success = mCaptureCollector.waitForRequestCompleted(holder, 827 REQUEST_COMPLETE_TIMEOUT, TimeUnit.MILLISECONDS, 828 /*out*/timestampMutable); 829 830 if (!success) { 831 Log.e(TAG, "Timed out while waiting for request to complete."); 832 mCaptureCollector.failAll(); 833 } 834 } catch (InterruptedException e) { 835 Log.e(TAG, "Interrupted waiting for request completion: ", e); 836 mDeviceState.setError( 837 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 838 break; 839 } 840 841 CameraMetadataNative result = mMapper.cachedConvertResultMetadata( 842 mLastRequest, timestampMutable.value); 843 /* 844 * Order matters: The default result mapper is state-less; the 845 * other mappers carry state and may override keys set by the default 846 * mapper with their own values. 847 */ 848 849 // Update AF state 850 mFocusStateMapper.mapResultTriggers(result); 851 // Update face-related results 852 mFaceDetectMapper.mapResultFaces(result, mLastRequest); 853 854 if (!holder.requestFailed()) { 855 mDeviceState.setCaptureResult(holder, result, 856 CameraDeviceState.NO_CAPTURE_ERROR); 857 } 858 } 859 if (DEBUG) { 860 long totalTime = SystemClock.elapsedRealtimeNanos() - startTime; 861 Log.d(TAG, "Capture request took " + totalTime + " ns"); 862 mRequestCounter.countAndLog(); 863 } 864 break; 865 case MSG_CLEANUP: 866 mCleanup = true; 867 try { 868 boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT, 869 TimeUnit.MILLISECONDS); 870 if (!success) { 871 Log.e(TAG, "Timed out while queueing cleanup request."); 872 mCaptureCollector.failAll(); 873 } 874 } catch (InterruptedException e) { 875 Log.e(TAG, "Interrupted while waiting for requests to complete: ", e); 876 mDeviceState.setError( 877 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 878 } 879 if (mGLThreadManager != null) { 880 mGLThreadManager.quit(); 881 } 882 if (mCamera != null) { 883 mCamera.release(); 884 } 885 resetJpegSurfaceFormats(mCallbackOutputs); 886 break; 887 case RequestHandlerThread.MSG_POKE_IDLE_HANDLER: 888 // OK: Ignore message. 889 break; 890 default: 891 throw new AssertionError("Unhandled message " + msg.what + 892 " on RequestThread."); 893 } 894 return true; 895 } 896 }; 897 898 /** 899 * Create a new RequestThreadManager. 900 * 901 * @param cameraId the id of the camera to use. 902 * @param camera an open camera object. The RequestThreadManager takes ownership of this camera 903 * object, and is responsible for closing it. 904 * @param characteristics the static camera characteristics corresponding to this camera device 905 * @param deviceState a {@link CameraDeviceState} state machine. 906 */ 907 public RequestThreadManager(int cameraId, Camera camera, CameraCharacteristics characteristics, 908 CameraDeviceState deviceState) { 909 mCamera = checkNotNull(camera, "camera must not be null"); 910 mCameraId = cameraId; 911 mCharacteristics = checkNotNull(characteristics, "characteristics must not be null"); 912 String name = String.format("RequestThread-%d", cameraId); 913 TAG = name; 914 mDeviceState = checkNotNull(deviceState, "deviceState must not be null"); 915 mFocusStateMapper = new LegacyFocusStateMapper(mCamera); 916 mFaceDetectMapper = new LegacyFaceDetectMapper(mCamera, mCharacteristics); 917 mCaptureCollector = new CaptureCollector(MAX_IN_FLIGHT_REQUESTS, mDeviceState); 918 mRequestThread = new RequestHandlerThread(name, mRequestHandlerCb); 919 mCamera.setErrorCallback(mErrorCallback); 920 } 921 922 /** 923 * Start the request thread. 924 */ 925 public void start() { 926 mRequestThread.start(); 927 } 928 929 /** 930 * Flush any pending requests. 931 * 932 * @return the last frame number. 933 */ 934 public long flush() { 935 Log.i(TAG, "Flushing all pending requests."); 936 long lastFrame = mRequestQueue.stopRepeating(); 937 mCaptureCollector.failAll(); 938 return lastFrame; 939 } 940 941 /** 942 * Quit the request thread, and clean up everything. 943 */ 944 public void quit() { 945 Handler handler = mRequestThread.waitAndGetHandler(); 946 handler.sendMessageAtFrontOfQueue(handler.obtainMessage(MSG_CLEANUP)); 947 mRequestThread.quitSafely(); 948 try { 949 mRequestThread.join(); 950 } catch (InterruptedException e) { 951 Log.e(TAG, String.format("Thread %s (%d) interrupted while quitting.", 952 mRequestThread.getName(), mRequestThread.getId())); 953 } 954 } 955 956 /** 957 * Submit the given burst of requests to be captured. 958 * 959 * <p>If the burst is repeating, replace the current repeating burst.</p> 960 * 961 * @param requests the burst of requests to add to the queue. 962 * @param repeating true if the burst is repeating. 963 * @param frameNumber an output argument that contains either the frame number of the last frame 964 * that will be returned for this request, or the frame number of the last 965 * frame that will be returned for the current repeating request if this 966 * burst is set to be repeating. 967 * @return the request id. 968 */ 969 public int submitCaptureRequests(List<CaptureRequest> requests, boolean repeating, 970 /*out*/LongParcelable frameNumber) { 971 Handler handler = mRequestThread.waitAndGetHandler(); 972 int ret; 973 synchronized (mIdleLock) { 974 ret = mRequestQueue.submit(requests, repeating, frameNumber); 975 handler.sendEmptyMessage(MSG_SUBMIT_CAPTURE_REQUEST); 976 } 977 return ret; 978 } 979 980 /** 981 * Cancel a repeating request. 982 * 983 * @param requestId the id of the repeating request to cancel. 984 * @return the last frame to be returned from the HAL for the given repeating request, or 985 * {@code INVALID_FRAME} if none exists. 986 */ 987 public long cancelRepeating(int requestId) { 988 return mRequestQueue.stopRepeating(requestId); 989 } 990 991 /** 992 * Configure with the current list of output Surfaces. 993 * 994 * <p> 995 * This operation blocks until the configuration is complete. 996 * </p> 997 * 998 * <p>Using a {@code null} or empty {@code outputs} list is the equivalent of unconfiguring.</p> 999 * 1000 * @param outputs a {@link java.util.Collection} of outputs to configure. 1001 */ 1002 public void configure(Collection<Pair<Surface, Size>> outputs) { 1003 Handler handler = mRequestThread.waitAndGetHandler(); 1004 final ConditionVariable condition = new ConditionVariable(/*closed*/false); 1005 ConfigureHolder holder = new ConfigureHolder(condition, outputs); 1006 handler.sendMessage(handler.obtainMessage(MSG_CONFIGURE_OUTPUTS, 0, 0, holder)); 1007 condition.block(); 1008 } 1009} 1010