RequestThreadManager.java revision d85e1a6ced452c9bd0d805f6ce19f50c9ea9b0a6
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.hardware.camera2.legacy; 18 19import android.graphics.ImageFormat; 20import android.graphics.SurfaceTexture; 21import android.hardware.Camera; 22import android.hardware.camera2.CaptureRequest; 23import android.hardware.camera2.CaptureResult; 24import android.hardware.camera2.utils.LongParcelable; 25import android.hardware.camera2.impl.CameraMetadataNative; 26import android.os.ConditionVariable; 27import android.os.Handler; 28import android.os.Message; 29import android.os.SystemClock; 30import android.util.Log; 31import android.util.Pair; 32import android.util.Size; 33import android.view.Surface; 34 35import java.io.IOError; 36import java.io.IOException; 37import java.util.ArrayList; 38import java.util.Collection; 39import java.util.Collections; 40import java.util.Comparator; 41import java.util.List; 42 43/** 44 * This class executes requests to the {@link Camera}. 45 * 46 * <p> 47 * The main components of this class are: 48 * - A message queue of requests to the {@link Camera}. 49 * - A thread that consumes requests to the {@link Camera} and executes them. 50 * - A {@link GLThreadManager} that draws to the configured output {@link Surface}s. 51 * - An {@link CameraDeviceState} state machine that manages the callbacks for various operations. 52 * </p> 53 */ 54public class RequestThreadManager { 55 private final String TAG; 56 private final int mCameraId; 57 private final RequestHandlerThread mRequestThread; 58 59 private static final boolean DEBUG = Log.isLoggable(LegacyCameraDevice.DEBUG_PROP, Log.DEBUG); 60 private final Camera mCamera; 61 62 private final CameraDeviceState mDeviceState; 63 64 private static final int MSG_CONFIGURE_OUTPUTS = 1; 65 private static final int MSG_SUBMIT_CAPTURE_REQUEST = 2; 66 private static final int MSG_CLEANUP = 3; 67 68 private static final int PREVIEW_FRAME_TIMEOUT = 300; // ms 69 private static final int JPEG_FRAME_TIMEOUT = 1000; // ms 70 71 private static final float ASPECT_RATIO_TOLERANCE = 0.01f; 72 private boolean mPreviewRunning = false; 73 74 private volatile RequestHolder mInFlightPreview; 75 private volatile RequestHolder mInFlightJpeg; 76 77 private List<Surface> mPreviewOutputs = new ArrayList<Surface>(); 78 private List<Surface> mCallbackOutputs = new ArrayList<Surface>(); 79 private GLThreadManager mGLThreadManager; 80 private SurfaceTexture mPreviewTexture; 81 82 private Size mIntermediateBufferSize; 83 84 private final RequestQueue mRequestQueue = new RequestQueue(); 85 private SurfaceTexture mDummyTexture; 86 private Surface mDummySurface; 87 88 private final FpsCounter mPrevCounter = new FpsCounter("Incoming Preview"); 89 90 /** 91 * Container object for Configure messages. 92 */ 93 private static class ConfigureHolder { 94 public final ConditionVariable condition; 95 public final Collection<Surface> surfaces; 96 97 public ConfigureHolder(ConditionVariable condition, Collection<Surface> surfaces) { 98 this.condition = condition; 99 this.surfaces = surfaces; 100 } 101 } 102 103 104 /** 105 * Comparator for {@link Size} objects. 106 * 107 * <p>This comparator compares by rectangle area. Tiebreaks on width.</p> 108 */ 109 private static class SizeComparator implements Comparator<Size> { 110 @Override 111 public int compare(Size size, Size size2) { 112 if (size == null || size2 == null) { 113 throw new NullPointerException("Null argument passed to compare"); 114 } 115 if (size.equals(size2)) return 0; 116 long width = size.getWidth(); 117 long width2 = size2.getWidth(); 118 long area = width * size.getHeight(); 119 long area2 = width2 * size2.getHeight(); 120 if (area == area2) { 121 return (width > width2) ? 1 : -1; 122 } 123 return (area > area2) ? 1 : -1; 124 125 } 126 } 127 128 /** 129 * Counter class used to calculate and log the current FPS of frame production. 130 */ 131 public static class FpsCounter { 132 //TODO: Hook this up to SystTrace? 133 private static final String TAG = "FpsCounter"; 134 private int mFrameCount = 0; 135 private long mLastTime = 0; 136 private long mLastPrintTime = 0; 137 private double mLastFps = 0; 138 private final String mStreamType; 139 private static final long NANO_PER_SECOND = 1000000000; //ns 140 141 public FpsCounter(String streamType) { 142 mStreamType = streamType; 143 } 144 145 public synchronized void countFrame() { 146 mFrameCount++; 147 long nextTime = SystemClock.elapsedRealtimeNanos(); 148 if (mLastTime == 0) { 149 mLastTime = nextTime; 150 } 151 if (nextTime > mLastTime + NANO_PER_SECOND) { 152 long elapsed = nextTime - mLastTime; 153 mLastFps = mFrameCount * (NANO_PER_SECOND / (double) elapsed); 154 mFrameCount = 0; 155 mLastTime = nextTime; 156 } 157 } 158 159 public synchronized double checkFps() { 160 return mLastFps; 161 } 162 163 public synchronized void staggeredLog() { 164 if (mLastTime > mLastPrintTime + 5 * NANO_PER_SECOND) { 165 mLastPrintTime = mLastTime; 166 Log.d(TAG, "FPS for " + mStreamType + " stream: " + mLastFps ); 167 } 168 } 169 170 public synchronized void countAndLog() { 171 countFrame(); 172 staggeredLog(); 173 } 174 } 175 /** 176 * Fake preview for jpeg captures when there is no active preview 177 */ 178 private void createDummySurface() { 179 if (mDummyTexture == null || mDummySurface == null) { 180 mDummyTexture = new SurfaceTexture(/*ignored*/0); 181 // TODO: use smallest default sizes 182 mDummyTexture.setDefaultBufferSize(640, 480); 183 mDummySurface = new Surface(mDummyTexture); 184 } 185 } 186 187 private final ConditionVariable mReceivedJpeg = new ConditionVariable(false); 188 private final ConditionVariable mReceivedPreview = new ConditionVariable(false); 189 190 private final Camera.PictureCallback mJpegCallback = new Camera.PictureCallback() { 191 @Override 192 public void onPictureTaken(byte[] data, Camera camera) { 193 Log.i(TAG, "Received jpeg."); 194 RequestHolder holder = mInFlightJpeg; 195 if (holder == null) { 196 Log.w(TAG, "Dropping jpeg frame."); 197 mInFlightJpeg = null; 198 return; 199 } 200 for (Surface s : holder.getHolderTargets()) { 201 if (RequestHolder.jpegType(s)) { 202 Log.i(TAG, "Producing jpeg buffer..."); 203 LegacyCameraDevice.nativeSetSurfaceDimens(s, data.length, /*height*/1); 204 LegacyCameraDevice.nativeProduceFrame(s, data, data.length, /*height*/1, 205 CameraMetadataNative.NATIVE_JPEG_FORMAT); 206 } 207 } 208 mReceivedJpeg.open(); 209 } 210 }; 211 212 private final SurfaceTexture.OnFrameAvailableListener mPreviewCallback = 213 new SurfaceTexture.OnFrameAvailableListener() { 214 @Override 215 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 216 if (DEBUG) { 217 mPrevCounter.countAndLog(); 218 } 219 RequestHolder holder = mInFlightPreview; 220 if (holder == null) { 221 Log.w(TAG, "Dropping preview frame."); 222 mInFlightPreview = null; 223 return; 224 } 225 if (holder.hasPreviewTargets()) { 226 mGLThreadManager.queueNewFrame(holder.getHolderTargets()); 227 } 228 229 mReceivedPreview.open(); 230 } 231 }; 232 233 private void stopPreview() { 234 if (mPreviewRunning) { 235 mCamera.stopPreview(); 236 mPreviewRunning = false; 237 } 238 } 239 240 private void startPreview() { 241 if (!mPreviewRunning) { 242 mCamera.startPreview(); 243 mPreviewRunning = true; 244 } 245 } 246 247 private void doJpegCapture(RequestHolder request) throws IOException { 248 if (!mPreviewRunning) { 249 createDummySurface(); 250 mCamera.setPreviewTexture(mDummyTexture); 251 startPreview(); 252 } 253 mInFlightJpeg = request; 254 // TODO: Hook up shutter callback to CameraDeviceStateListener#onCaptureStarted 255 mCamera.takePicture(/*shutter*/null, /*raw*/null, mJpegCallback); 256 mPreviewRunning = false; 257 } 258 259 private void doPreviewCapture(RequestHolder request) throws IOException { 260 mInFlightPreview = request; 261 if (mPreviewRunning) { 262 return; // Already running 263 } 264 265 if (mPreviewTexture == null) { 266 throw new IllegalStateException( 267 "Preview capture called with no preview surfaces configured."); 268 } 269 270 mPreviewTexture.setDefaultBufferSize(mIntermediateBufferSize.getWidth(), 271 mIntermediateBufferSize.getHeight()); 272 mCamera.setPreviewTexture(mPreviewTexture); 273 Camera.Parameters params = mCamera.getParameters(); 274 List<int[]> supportedFpsRanges = params.getSupportedPreviewFpsRange(); 275 int[] bestRange = getPhotoPreviewFpsRange(supportedFpsRanges); 276 if (DEBUG) { 277 Log.d(TAG, "doPreviewCapture - Selected range [" + 278 bestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] + "," + 279 bestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] + "]"); 280 } 281 params.setPreviewFpsRange(bestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], 282 bestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); 283 params.setRecordingHint(true); 284 mCamera.setParameters(params); 285 286 startPreview(); 287 } 288 289 290 private void configureOutputs(Collection<Surface> outputs) throws IOException { 291 stopPreview(); 292 if (mGLThreadManager != null) { 293 mGLThreadManager.waitUntilStarted(); 294 mGLThreadManager.ignoreNewFrames(); 295 mGLThreadManager.waitUntilIdle(); 296 } 297 mPreviewOutputs.clear(); 298 mCallbackOutputs.clear(); 299 mPreviewTexture = null; 300 mInFlightPreview = null; 301 mInFlightJpeg = null; 302 303 304 for (Surface s : outputs) { 305 int format = LegacyCameraDevice.nativeDetectSurfaceType(s); 306 switch (format) { 307 case CameraMetadataNative.NATIVE_JPEG_FORMAT: 308 mCallbackOutputs.add(s); 309 break; 310 default: 311 mPreviewOutputs.add(s); 312 break; 313 } 314 } 315 316 if (mPreviewOutputs.size() > 0) { 317 List<Size> outputSizes = new ArrayList<>(outputs.size()); 318 for (Surface s : mPreviewOutputs) { 319 int[] dimens = {0, 0}; 320 LegacyCameraDevice.nativeDetectSurfaceDimens(s, dimens); 321 outputSizes.add(new Size(dimens[0], dimens[1])); 322 } 323 324 Size largestOutput = findLargestByArea(outputSizes); 325 326 Camera.Parameters params = mCamera.getParameters(); 327 328 // Find largest jpeg dimension - assume to have the same aspect ratio as sensor. 329 List<Size> supportedJpegSizes = convertSizeList(params.getSupportedPictureSizes()); 330 Size largestJpegDimen = findLargestByArea(supportedJpegSizes); 331 332 List<Size> supportedPreviewSizes = convertSizeList(params.getSupportedPreviewSizes()); 333 334 // Use smallest preview dimension with same aspect ratio as sensor that is >= than all 335 // of the configured output dimensions. If none exists, fall back to using the largest 336 // supported preview size. 337 long largestOutputArea = largestOutput.getHeight() * (long) largestOutput.getWidth(); 338 Size bestPreviewDimen = findLargestByArea(supportedPreviewSizes); 339 for (Size s : supportedPreviewSizes) { 340 long currArea = s.getWidth() * s.getHeight(); 341 long bestArea = bestPreviewDimen.getWidth() * bestPreviewDimen.getHeight(); 342 if (checkAspectRatiosMatch(largestJpegDimen, s) && (currArea < bestArea && 343 currArea >= largestOutputArea)) { 344 bestPreviewDimen = s; 345 } 346 } 347 348 mIntermediateBufferSize = bestPreviewDimen; 349 if (DEBUG) { 350 Log.d(TAG, "Intermediate buffer selected with dimens: " + 351 bestPreviewDimen.toString()); 352 } 353 } else { 354 mIntermediateBufferSize = null; 355 if (DEBUG) { 356 Log.d(TAG, "No Intermediate buffer selected, no preview outputs were configured"); 357 } 358 } 359 360 361 362 // TODO: Detect and optimize single-output paths here to skip stream teeing. 363 if (mGLThreadManager == null) { 364 mGLThreadManager = new GLThreadManager(mCameraId); 365 mGLThreadManager.start(); 366 } 367 mGLThreadManager.waitUntilStarted(); 368 mGLThreadManager.setConfigurationAndWait(mPreviewOutputs); 369 mGLThreadManager.allowNewFrames(); 370 mPreviewTexture = mGLThreadManager.getCurrentSurfaceTexture(); 371 if (mPreviewTexture != null) { 372 mPreviewTexture.setOnFrameAvailableListener(mPreviewCallback); 373 } 374 } 375 376 private static Size findLargestByArea(List<Size> sizes) { 377 return Collections.max(sizes, new SizeComparator()); 378 } 379 380 private static boolean checkAspectRatiosMatch(Size a, Size b) { 381 float aAspect = a.getWidth() / (float) a.getHeight(); 382 float bAspect = b.getWidth() / (float) b.getHeight(); 383 384 return Math.abs(aAspect - bAspect) < ASPECT_RATIO_TOLERANCE; 385 } 386 387 private static List<Size> convertSizeList(List<Camera.Size> sizeList) { 388 List<Size> sizes = new ArrayList<>(sizeList.size()); 389 for (Camera.Size s : sizeList) { 390 sizes.add(new Size(s.width, s.height)); 391 } 392 return sizes; 393 } 394 395 // Calculate the highest FPS range supported 396 private int[] getPhotoPreviewFpsRange(List<int[]> frameRates) { 397 if (frameRates.size() == 0) { 398 Log.e(TAG, "No supported frame rates returned!"); 399 return null; 400 } 401 402 int bestMin = 0; 403 int bestMax = 0; 404 int bestIndex = 0; 405 int index = 0; 406 for (int[] rate : frameRates) { 407 int minFps = rate[Camera.Parameters.PREVIEW_FPS_MIN_INDEX]; 408 int maxFps = rate[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]; 409 if (maxFps > bestMax || (maxFps == bestMax && minFps > bestMin)) { 410 bestMin = minFps; 411 bestMax = maxFps; 412 bestIndex = index; 413 } 414 index++; 415 } 416 417 return frameRates.get(bestIndex); 418 } 419 420 private final Handler.Callback mRequestHandlerCb = new Handler.Callback() { 421 private boolean mCleanup = false; 422 private List<RequestHolder> mRepeating = null; 423 424 @SuppressWarnings("unchecked") 425 @Override 426 public boolean handleMessage(Message msg) { 427 if (mCleanup) { 428 return true; 429 } 430 431 switch (msg.what) { 432 case MSG_CONFIGURE_OUTPUTS: 433 ConfigureHolder config = (ConfigureHolder) msg.obj; 434 Log.i(TAG, "Configure outputs: " + config.surfaces.size() + 435 " surfaces configured."); 436 try { 437 configureOutputs(config.surfaces); 438 } catch (IOException e) { 439 // TODO: report error to CameraDevice 440 throw new IOError(e); 441 } 442 config.condition.open(); 443 break; 444 case MSG_SUBMIT_CAPTURE_REQUEST: 445 Handler handler = RequestThreadManager.this.mRequestThread.getHandler(); 446 447 // Get the next burst from the request queue. 448 Pair<BurstHolder, Long> nextBurst = mRequestQueue.getNext(); 449 if (nextBurst == null) { 450 mDeviceState.setIdle(); 451 stopPreview(); 452 break; 453 } else { 454 // Queue another capture if we did not get the last burst. 455 handler.sendEmptyMessage(MSG_SUBMIT_CAPTURE_REQUEST); 456 } 457 458 // Complete each request in the burst 459 List<RequestHolder> requests = 460 nextBurst.first.produceRequestHolders(nextBurst.second); 461 for (RequestHolder holder : requests) { 462 mDeviceState.setCaptureStart(holder); 463 try { 464 if (holder.hasPreviewTargets()) { 465 mReceivedPreview.close(); 466 doPreviewCapture(holder); 467 if (!mReceivedPreview.block(PREVIEW_FRAME_TIMEOUT)) { 468 // TODO: report error to CameraDevice 469 Log.e(TAG, "Hit timeout for preview callback!"); 470 } 471 } 472 if (holder.hasJpegTargets()) { 473 mReceivedJpeg.close(); 474 doJpegCapture(holder); 475 mReceivedJpeg.block(); 476 if (!mReceivedJpeg.block(JPEG_FRAME_TIMEOUT)) { 477 // TODO: report error to CameraDevice 478 Log.e(TAG, "Hit timeout for jpeg callback!"); 479 } 480 mInFlightJpeg = null; 481 } 482 } catch (IOException e) { 483 // TODO: err handling 484 throw new IOError(e); 485 } 486 Camera.Parameters params = mCamera.getParameters(); 487 CameraMetadataNative result = convertResultMetadata(params, 488 holder.getRequest()); 489 mDeviceState.setCaptureResult(holder, result); 490 } 491 break; 492 case MSG_CLEANUP: 493 mCleanup = true; 494 if (mGLThreadManager != null) { 495 mGLThreadManager.quit(); 496 } 497 if (mCamera != null) { 498 mCamera.release(); 499 } 500 break; 501 default: 502 throw new AssertionError("Unhandled message " + msg.what + 503 " on RequestThread."); 504 } 505 return true; 506 } 507 }; 508 509 private CameraMetadataNative convertResultMetadata(Camera.Parameters params, 510 CaptureRequest request) { 511 CameraMetadataNative result = new CameraMetadataNative(); 512 result.set(CaptureResult.LENS_FOCAL_LENGTH, params.getFocalLength()); 513 514 // TODO: Remaining result metadata tags conversions. 515 return result; 516 } 517 518 /** 519 * Create a new RequestThreadManager. 520 * 521 * @param cameraId the id of the camera to use. 522 * @param camera an open camera object. The RequestThreadManager takes ownership of this camera 523 * object, and is responsible for closing it. 524 * @param deviceState a {@link CameraDeviceState} state machine. 525 */ 526 public RequestThreadManager(int cameraId, Camera camera, 527 CameraDeviceState deviceState) { 528 mCamera = camera; 529 mCameraId = cameraId; 530 String name = String.format("RequestThread-%d", cameraId); 531 TAG = name; 532 mDeviceState = deviceState; 533 mRequestThread = new RequestHandlerThread(name, mRequestHandlerCb); 534 } 535 536 /** 537 * Start the request thread. 538 */ 539 public void start() { 540 mRequestThread.start(); 541 } 542 543 /** 544 * Flush the pending requests. 545 */ 546 public void flush() { 547 // TODO: Implement flush. 548 Log.e(TAG, "flush not yet implemented."); 549 } 550 551 /** 552 * Quit the request thread, and clean up everything. 553 */ 554 public void quit() { 555 Handler handler = mRequestThread.waitAndGetHandler(); 556 handler.sendMessageAtFrontOfQueue(handler.obtainMessage(MSG_CLEANUP)); 557 mRequestThread.quitSafely(); 558 try { 559 mRequestThread.join(); 560 } catch (InterruptedException e) { 561 Log.e(TAG, String.format("Thread %s (%d) interrupted while quitting.", 562 mRequestThread.getName(), mRequestThread.getId())); 563 } 564 } 565 566 /** 567 * Submit the given burst of requests to be captured. 568 * 569 * <p>If the burst is repeating, replace the current repeating burst.</p> 570 * 571 * @param requests the burst of requests to add to the queue. 572 * @param repeating true if the burst is repeating. 573 * @param frameNumber an output argument that contains either the frame number of the last frame 574 * that will be returned for this request, or the frame number of the last 575 * frame that will be returned for the current repeating request if this 576 * burst is set to be repeating. 577 * @return the request id. 578 */ 579 public int submitCaptureRequests(List<CaptureRequest> requests, boolean repeating, 580 /*out*/LongParcelable frameNumber) { 581 Handler handler = mRequestThread.waitAndGetHandler(); 582 int ret = mRequestQueue.submit(requests, repeating, frameNumber); 583 handler.sendEmptyMessage(MSG_SUBMIT_CAPTURE_REQUEST); 584 return ret; 585 } 586 587 /** 588 * Cancel a repeating request. 589 * 590 * @param requestId the id of the repeating request to cancel. 591 * @return the last frame to be returned from the HAL for the given repeating request, or 592 * {@code INVALID_FRAME} if none exists. 593 */ 594 public long cancelRepeating(int requestId) { 595 return mRequestQueue.stopRepeating(requestId); 596 } 597 598 599 /** 600 * Configure with the current output Surfaces. 601 * 602 * <p> 603 * This operation blocks until the configuration is complete. 604 * </p> 605 * 606 * @param outputs a {@link java.util.Collection} of outputs to configure. 607 */ 608 public void configure(Collection<Surface> outputs) { 609 Handler handler = mRequestThread.waitAndGetHandler(); 610 final ConditionVariable condition = new ConditionVariable(/*closed*/false); 611 ConfigureHolder holder = new ConfigureHolder(condition, outputs); 612 handler.sendMessage(handler.obtainMessage(MSG_CONFIGURE_OUTPUTS, 0, 0, holder)); 613 condition.block(); 614 } 615} 616