ItsService.java revision 12339cdd90b2c98eb4f7adf794ddca8de53992b8
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.camera2.its; 18 19import android.app.Service; 20import android.content.Context; 21import android.content.Intent; 22import android.graphics.ImageFormat; 23import android.hardware.camera2.CameraAccessException; 24import android.hardware.camera2.CameraDevice; 25import android.hardware.camera2.CameraManager; 26import android.hardware.camera2.CameraProperties; 27import android.hardware.camera2.CaptureRequest; 28import android.hardware.camera2.CaptureResult; 29import android.media.Image; 30import android.media.ImageReader; 31import android.net.Uri; 32import android.os.ConditionVariable; 33import android.os.Handler; 34import android.os.HandlerThread; 35import android.os.IBinder; 36import android.os.Message; 37import android.util.Log; 38import android.view.Surface; 39 40import org.json.JSONObject; 41 42import java.io.File; 43import java.nio.ByteBuffer; 44import java.util.ArrayList; 45import java.util.Arrays; 46import java.util.List; 47import java.util.concurrent.CountDownLatch; 48import java.util.concurrent.TimeUnit; 49 50public class ItsService extends Service { 51 public static final String TAG = ItsService.class.getSimpleName(); 52 public static final String PYTAG = "CAMERA-ITS-PY"; 53 54 // Supported intents 55 public static final String ACTION_CAPTURE = "com.android.camera2.its.CAPTURE"; 56 public static final String ACTION_3A = "com.android.camera2.its.3A"; 57 public static final String ACTION_GETPROPS = "com.android.camera2.its.GETPROPS"; 58 private static final int MESSAGE_CAPTURE = 1; 59 private static final int MESSAGE_3A = 2; 60 private static final int MESSAGE_GETPROPS = 3; 61 62 // Timeouts, in seconds. 63 public static final int TIMEOUT_CAPTURE = 10; 64 public static final int TIMEOUT_3A = 10; 65 66 private static final int MAX_CONCURRENT_READER_BUFFERS = 8; 67 68 public static final String REGION_KEY = "regions"; 69 public static final String REGION_AE_KEY = "ae"; 70 public static final String REGION_AWB_KEY = "awb"; 71 public static final String REGION_AF_KEY = "af"; 72 73 private CameraManager mCameraManager = null; 74 private CameraDevice mCamera = null; 75 private ImageReader mCaptureReader = null; 76 private CameraProperties mCameraProperties = null; 77 78 private HandlerThread mCommandThread; 79 private Handler mCommandHandler; 80 private HandlerThread mSaveThread; 81 private Handler mSaveHandler; 82 83 private ConditionVariable mInterlock3A = new ConditionVariable(true); 84 private volatile boolean mIssuedRequest3A = false; 85 private volatile boolean mConvergedAE = false; 86 private volatile boolean mConvergedAF = false; 87 private volatile boolean mConvergedAWB = false; 88 89 private CountDownLatch mCaptureCallbackLatch; 90 91 public interface CaptureListener { 92 void onCaptureAvailable(Image capture); 93 } 94 95 public interface CaptureResultListener extends CameraDevice.CaptureListener {} 96 97 @Override 98 public IBinder onBind(Intent intent) { 99 return null; 100 } 101 102 @Override 103 public void onCreate() { 104 105 try { 106 // Get handle to camera manager. 107 mCameraManager = (CameraManager) this.getSystemService(Context.CAMERA_SERVICE); 108 if (mCameraManager == null) { 109 throw new ItsException("Failed to connect to camera manager"); 110 } 111 112 // Open the camera device, and get its properties. 113 String[] devices; 114 try { 115 devices = mCameraManager.getDeviceIdList(); 116 if (devices == null || devices.length == 0) { 117 throw new ItsException("No camera devices"); 118 } 119 120 // TODO: Add support for specifying which device to open. 121 mCamera = mCameraManager.openCamera(devices[0]); 122 mCameraProperties = mCamera.getProperties(); 123 } catch (CameraAccessException e) { 124 throw new ItsException("Failed to get device ID list"); 125 } 126 127 // Create a thread to receive images and save them. 128 mSaveThread = new HandlerThread("SaveThread"); 129 mSaveThread.start(); 130 mSaveHandler = new Handler(mSaveThread.getLooper()); 131 132 // Create a thread to process commands. 133 mCommandThread = new HandlerThread("CaptureThread"); 134 mCommandThread.start(); 135 mCommandHandler = new Handler(mCommandThread.getLooper(), new Handler.Callback() { 136 @Override 137 public boolean handleMessage(Message msg) { 138 try { 139 switch (msg.what) { 140 case MESSAGE_CAPTURE: 141 doCapture((Uri) msg.obj); 142 break; 143 case MESSAGE_3A: 144 do3A((Uri) msg.obj); 145 break; 146 case MESSAGE_GETPROPS: 147 doGetProps(); 148 break; 149 default: 150 throw new ItsException("Unknown message type"); 151 } 152 Log.i(PYTAG, "### DONE"); 153 return true; 154 } 155 catch (ItsException e) { 156 Log.e(TAG, "Script failed: ", e); 157 Log.e(PYTAG, "### FAIL"); 158 return true; 159 } 160 } 161 }); 162 } catch (ItsException e) { 163 Log.e(TAG, "Script failed: ", e); 164 Log.e(PYTAG, "### FAIL"); 165 } 166 } 167 168 @Override 169 public void onDestroy() { 170 try { 171 if (mCommandThread != null) { 172 mCommandThread.quit(); 173 mCommandThread = null; 174 } 175 if (mSaveThread != null) { 176 mSaveThread.quit(); 177 mSaveThread = null; 178 } 179 180 try { 181 mCamera.close(); 182 } catch (Exception e) { 183 throw new ItsException("Failed to close device"); 184 } 185 } catch (ItsException e) { 186 Log.e(TAG, "Script failed: ", e); 187 Log.e(PYTAG, "### FAIL"); 188 } 189 } 190 191 @Override 192 public int onStartCommand(Intent intent, int flags, int startId) { 193 try { 194 Log.i(PYTAG, "### RECV"); 195 String action = intent.getAction(); 196 if (ACTION_CAPTURE.equals(action)) { 197 Uri uri = intent.getData(); 198 Message m = mCommandHandler.obtainMessage(MESSAGE_CAPTURE, uri); 199 mCommandHandler.sendMessage(m); 200 } else if (ACTION_3A.equals(action)) { 201 Uri uri = intent.getData(); 202 Message m = mCommandHandler.obtainMessage(MESSAGE_3A, uri); 203 mCommandHandler.sendMessage(m); 204 } else if (ACTION_GETPROPS.equals(action)) { 205 Uri uri = intent.getData(); 206 Message m = mCommandHandler.obtainMessage(MESSAGE_GETPROPS, uri); 207 mCommandHandler.sendMessage(m); 208 } else { 209 throw new ItsException("Unhandled intent: " + intent.toString()); 210 } 211 } catch (ItsException e) { 212 Log.e(TAG, "Script failed: ", e); 213 Log.e(PYTAG, "### FAIL"); 214 } 215 return START_STICKY; 216 } 217 218 public void idleCamera() throws ItsException { 219 try { 220 mCamera.stopRepeating(); 221 mCamera.waitUntilIdle(); 222 } catch (CameraAccessException e) { 223 throw new ItsException("Error waiting for camera idle", e); 224 } 225 } 226 227 private ImageReader.OnImageAvailableListener 228 createAvailableListener(final CaptureListener listener) { 229 return new ImageReader.OnImageAvailableListener() { 230 @Override 231 public void onImageAvailable(ImageReader reader) { 232 Image i = reader.getNextImage(); 233 listener.onCaptureAvailable(i); 234 i.close(); 235 } 236 }; 237 } 238 239 private ImageReader.OnImageAvailableListener 240 createAvailableListenerDropper(final CaptureListener listener) { 241 return new ImageReader.OnImageAvailableListener() { 242 @Override 243 public void onImageAvailable(ImageReader reader) { 244 Image i = reader.getNextImage(); 245 i.close(); 246 } 247 }; 248 } 249 250 private void doGetProps() throws ItsException { 251 String fileName = ItsUtils.getMetadataFileName(0); 252 File mdFile = ItsUtils.getOutputFile(ItsService.this, fileName); 253 ItsUtils.storeCameraProperties(mCameraProperties, mdFile); 254 Log.i(PYTAG, 255 String.format("### FILE %s", 256 ItsUtils.getExternallyVisiblePath(ItsService.this, mdFile.toString()))); 257 } 258 259 private void do3A(Uri uri) throws ItsException { 260 try { 261 if (uri == null || !uri.toString().endsWith(".json")) { 262 throw new ItsException("Invalid URI: " + uri); 263 } 264 265 idleCamera(); 266 267 // Start a 3A action, and wait for it to converge. 268 // Get the converged values for each "A", and package into JSON result for caller. 269 270 // 3A happens on full-res frames. 271 android.hardware.camera2.Size sizes[] = mCameraProperties.get( 272 CameraProperties.SCALER_AVAILABLE_JPEG_SIZES); 273 int width = sizes[0].getWidth(); 274 int height = sizes[0].getHeight(); 275 int format = ImageFormat.YUV_420_888; 276 277 if (mCaptureReader == null || mCaptureReader.getWidth() != width 278 || mCaptureReader.getHeight() != height) { 279 if (mCaptureReader != null) { 280 mCaptureReader.close(); 281 } 282 mCaptureReader = new ImageReader(width, height, format, 283 MAX_CONCURRENT_READER_BUFFERS); 284 } 285 286 List<Surface> outputSurfaces = new ArrayList<Surface>(1); 287 outputSurfaces.add(mCaptureReader.getSurface()); 288 mCamera.configureOutputs(outputSurfaces); 289 290 // Add a listener that just recycles buffers; they aren't saved anywhere. 291 ImageReader.OnImageAvailableListener readerListener = 292 createAvailableListenerDropper(mCaptureListener); 293 mCaptureReader.setImageAvailableListener(readerListener, mSaveHandler); 294 295 // Get the user-specified regions for AE, AWB, AF. 296 // Note that the user specifies normalized [x,y,w,h], which is converted below 297 // to an [x0,y0,x1,y1] region in sensor coords. The capture request region 298 // also has a fifth "weight" element: [x0,y0,x1,y1,w]. 299 int[] regionAE = new int[]{0,0,width-1,height-1,1}; 300 int[] regionAF = new int[]{0,0,width-1,height-1,1}; 301 int[] regionAWB = new int[]{0,0,width-1,height-1,1}; 302 JSONObject params = ItsUtils.loadJsonFile(uri); 303 if (params.has(REGION_KEY)) { 304 JSONObject regions = params.getJSONObject(REGION_KEY); 305 if (params.has(REGION_AE_KEY)) { 306 int[] r = ItsUtils.getJsonRectFromArray( 307 params.getJSONArray(REGION_AE_KEY), true, width, height); 308 regionAE = new int[]{r[0],r[1],r[0]+r[2]-1,r[1]+r[3]-1,1}; 309 } 310 if (params.has(REGION_AF_KEY)) { 311 int[] r = ItsUtils.getJsonRectFromArray( 312 params.getJSONArray(REGION_AF_KEY), true, width, height); 313 regionAF = new int[]{r[0],r[1],r[0]+r[2]-1,r[1]+r[3]-1,1}; 314 } 315 if (params.has(REGION_AWB_KEY)) { 316 int[] r = ItsUtils.getJsonRectFromArray( 317 params.getJSONArray(REGION_AWB_KEY), true, width, height); 318 regionAWB = new int[]{r[0],r[1],r[0]+r[2]-1,r[1]+r[3]-1,1}; 319 } 320 } 321 Log.i(TAG, "AE region: " + Arrays.toString(regionAE)); 322 Log.i(TAG, "AF region: " + Arrays.toString(regionAF)); 323 Log.i(TAG, "AWB region: " + Arrays.toString(regionAWB)); 324 325 mInterlock3A.open(); 326 mIssuedRequest3A = false; 327 mConvergedAE = false; 328 mConvergedAWB = false; 329 mConvergedAF = false; 330 long tstart = System.currentTimeMillis(); 331 boolean triggeredAE = false; 332 boolean triggeredAF = false; 333 334 // Keep issuing capture requests until 3A has converged. 335 // First do AE, then do AF and AWB together. 336 while (true) { 337 338 // Block until can take the next 3A frame. Only want one outstanding frame 339 // at a time, to simplify the logic here. 340 if (!mInterlock3A.block(TIMEOUT_3A * 1000) || 341 System.currentTimeMillis() - tstart > TIMEOUT_3A * 1000) { 342 throw new ItsException("3A failed to converge (timeout)"); 343 } 344 mInterlock3A.close(); 345 346 // If not converged yet, issue another capture request. 347 if (!mConvergedAE || !mConvergedAWB || !mConvergedAF) { 348 349 // Baseline capture request for 3A. 350 CaptureRequest req = mCamera.createCaptureRequest( 351 CameraDevice.TEMPLATE_PREVIEW); 352 req.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_OFF); 353 req.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO); 354 req.set(CaptureRequest.CONTROL_CAPTURE_INTENT, 355 CaptureRequest.CONTROL_CAPTURE_INTENT_PREVIEW); 356 req.set(CaptureRequest.CONTROL_AE_MODE, 357 CaptureRequest.CONTROL_AE_MODE_ON); 358 req.set(CaptureRequest.CONTROL_AE_EXPOSURE_COMPENSATION, 0); 359 req.set(CaptureRequest.CONTROL_AE_LOCK, false); 360 req.set(CaptureRequest.CONTROL_AE_REGIONS, regionAE); 361 req.set(CaptureRequest.CONTROL_AF_MODE, 362 CaptureRequest.CONTROL_AF_MODE_AUTO); 363 req.set(CaptureRequest.CONTROL_AF_REGIONS, regionAF); 364 req.set(CaptureRequest.CONTROL_AWB_MODE, 365 CaptureRequest.CONTROL_AWB_MODE_AUTO); 366 req.set(CaptureRequest.CONTROL_AWB_LOCK, false); 367 req.set(CaptureRequest.CONTROL_AWB_REGIONS, regionAWB); 368 369 // Trigger AE first. 370 if (!triggeredAE) { 371 Log.i(TAG, "Triggering AE"); 372 req.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, 373 CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_START); 374 triggeredAE = true; 375 } 376 377 // After AE has converged, trigger AF. 378 if (!triggeredAF && triggeredAE && mConvergedAE) { 379 Log.i(TAG, "Triggering AF"); 380 req.set(CaptureRequest.CONTROL_AF_TRIGGER, 381 CaptureRequest.CONTROL_AF_TRIGGER_START); 382 triggeredAF = true; 383 } 384 385 req.addTarget(mCaptureReader.getSurface()); 386 387 mIssuedRequest3A = true; 388 mCamera.capture(req, mCaptureResultListener); 389 } else { 390 Log.i(TAG, "3A converged"); 391 break; 392 } 393 } 394 } catch (android.hardware.camera2.CameraAccessException e) { 395 throw new ItsException("Access error: ", e); 396 } catch (org.json.JSONException e) { 397 throw new ItsException("JSON error: ", e); 398 } 399 } 400 401 private void doCapture(Uri uri) throws ItsException { 402 try { 403 if (uri == null || !uri.toString().endsWith(".json")) { 404 throw new ItsException("Invalid URI: " + uri); 405 } 406 407 idleCamera(); 408 409 // Parse the JSON to get the list of capture requests. 410 List<CaptureRequest> requests = ItsUtils.loadRequestList(mCamera, uri); 411 412 // Set the output surface and listeners. 413 try { 414 // Default: 415 // Capture full-frame images. Use the reported JPEG size rather than the sensor 416 // size since this is more likely to be the unscaled size; the crop from sensor 417 // size is probably for the ISP (e.g. demosaicking) rather than the encoder. 418 android.hardware.camera2.Size sizes[] = mCameraProperties.get( 419 CameraProperties.SCALER_AVAILABLE_JPEG_SIZES); 420 int width = sizes[0].getWidth(); 421 int height = sizes[0].getHeight(); 422 int format = ImageFormat.YUV_420_888; 423 424 JSONObject jsonOutputSpecs = ItsUtils.getOutputSpecs(uri); 425 if (jsonOutputSpecs != null) { 426 // Use the user's JSON capture spec. 427 int width2 = jsonOutputSpecs.optInt("width"); 428 int height2 = jsonOutputSpecs.optInt("height"); 429 if (width2 > 0) { 430 width = width2; 431 } 432 if (height2 > 0) { 433 height = height2; 434 } 435 String sformat = jsonOutputSpecs.optString("format"); 436 if ("yuv".equals(sformat)) { 437 format = ImageFormat.YUV_420_888; 438 } else if ("jpg".equals(sformat) || "jpeg".equals(sformat)) { 439 format = ImageFormat.JPEG; 440 } else if ("".equals(sformat)) { 441 // No format specified. 442 } else { 443 throw new ItsException("Unsupported format: " + sformat); 444 } 445 } 446 447 Log.i(PYTAG, String.format("### SIZE %d %d", width, height)); 448 449 if (mCaptureReader == null || mCaptureReader.getWidth() != width 450 || mCaptureReader.getHeight() != height) { 451 if (mCaptureReader != null) { 452 mCaptureReader.close(); 453 } 454 mCaptureReader = new ImageReader(width, height, format, 455 MAX_CONCURRENT_READER_BUFFERS); 456 } 457 458 // TODO: support multiple surfaces 459 List<Surface> outputSurfaces = new ArrayList<Surface>(1); 460 outputSurfaces.add(mCaptureReader.getSurface()); 461 mCamera.configureOutputs(outputSurfaces); 462 463 ImageReader.OnImageAvailableListener readerListener = 464 createAvailableListener(mCaptureListener); 465 mCaptureReader.setImageAvailableListener(readerListener, mSaveHandler); 466 467 // Plan for how many callbacks need to be received throughout the duration of this 468 // sequence of capture requests. 469 int numCaptures = requests.size(); 470 mCaptureCallbackLatch = new CountDownLatch( 471 numCaptures * ItsUtils.getCallbacksPerCapture(format)); 472 473 } catch (CameraAccessException e) { 474 throw new ItsException("Error configuring outputs", e); 475 } 476 477 // Initiate the captures. 478 for (int i = 0; i < requests.size(); i++) { 479 CaptureRequest req = requests.get(i); 480 Log.i(PYTAG, String.format("### CAPT %d of %d", i+1, requests.size())); 481 req.addTarget(mCaptureReader.getSurface()); 482 mCamera.capture(req, mCaptureResultListener); 483 } 484 485 // Make sure all callbacks have been hit (wait until captures are done). 486 try { 487 if (!mCaptureCallbackLatch.await(TIMEOUT_CAPTURE, TimeUnit.SECONDS)) { 488 throw new ItsException( 489 "Timeout hit, but all callbacks not received"); 490 } 491 } catch (InterruptedException e) { 492 throw new ItsException("Interrupted: ", e); 493 } 494 495 } catch (android.hardware.camera2.CameraAccessException e) { 496 throw new ItsException("Access error: ", e); 497 } 498 } 499 500 private final CaptureListener mCaptureListener = new CaptureListener() { 501 @Override 502 public void onCaptureAvailable(Image capture) { 503 try { 504 int format = capture.getFormat(); 505 String extFileName = null; 506 if (format == ImageFormat.JPEG) { 507 String fileName = ItsUtils.getJpegFileName(capture.getTimestamp()); 508 ByteBuffer buf = capture.getPlanes()[0].getBuffer(); 509 extFileName = ItsUtils.writeImageToFile(ItsService.this, buf, fileName); 510 } else if (format == ImageFormat.YUV_420_888) { 511 String fileName = ItsUtils.getYuvFileName(capture.getTimestamp()); 512 byte[] img = ItsUtils.getDataFromImage(capture); 513 ByteBuffer buf = ByteBuffer.wrap(img); 514 extFileName = ItsUtils.writeImageToFile(ItsService.this, buf, fileName); 515 } else { 516 throw new ItsException("Unsupported image format: " + format); 517 } 518 Log.i(PYTAG, String.format("### FILE %s", extFileName)); 519 mCaptureCallbackLatch.countDown(); 520 } catch (ItsException e) { 521 Log.e(TAG, "Script error: " + e); 522 Log.e(PYTAG, "### FAIL"); 523 } 524 } 525 }; 526 527 private final CaptureResultListener mCaptureResultListener = new CaptureResultListener() { 528 @Override 529 public void onCaptureComplete(CameraDevice camera, CaptureRequest request, 530 CaptureResult result) { 531 try { 532 // Currently result has all 0 values. 533 if (request == null || result == null) { 534 throw new ItsException("Request/result is invalid"); 535 } 536 537 Log.i(TAG, String.format( 538 "Capture result: AE=%d, AF=%d, AWB=%d, sens=%d, exp=%dms, dur=%dms", 539 result.get(CaptureResult.CONTROL_AE_STATE), 540 result.get(CaptureResult.CONTROL_AF_STATE), 541 result.get(CaptureResult.CONTROL_AWB_STATE), 542 result.get(CaptureResult.SENSOR_SENSITIVITY), 543 result.get(CaptureResult.SENSOR_EXPOSURE_TIME).intValue(), 544 result.get(CaptureResult.SENSOR_FRAME_DURATION).intValue())); 545 546 mConvergedAE = result.get(CaptureResult.CONTROL_AE_STATE) == 547 CaptureResult.CONTROL_AE_STATE_CONVERGED; 548 mConvergedAF = result.get(CaptureResult.CONTROL_AF_STATE) == 549 CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED; 550 mConvergedAWB = result.get(CaptureResult.CONTROL_AWB_STATE) == 551 CaptureResult.CONTROL_AWB_STATE_CONVERGED; 552 553 if (mIssuedRequest3A) { 554 mIssuedRequest3A = false; 555 mInterlock3A.open(); 556 } else { 557 String fileName = ItsUtils.getMetadataFileName( 558 result.get(CaptureResult.SENSOR_TIMESTAMP)); 559 File mdFile = ItsUtils.getOutputFile(ItsService.this, fileName); 560 ItsUtils.storeResults(mCameraProperties, request, result, mdFile); 561 mCaptureCallbackLatch.countDown(); 562 } 563 } catch (ItsException e) { 564 Log.e(TAG, "Script error: " + e); 565 Log.e(PYTAG, "### FAIL"); 566 } catch (Exception e) { 567 Log.e(TAG, "Script error: " + e); 568 Log.e(PYTAG, "### FAIL"); 569 } 570 } 571 572 @Override 573 public void onCaptureFailed(CameraDevice camera, CaptureRequest request) { 574 mCaptureCallbackLatch.countDown(); 575 Log.e(TAG, "Script error: capture failed"); 576 Log.e(PYTAG, "### FAIL"); 577 } 578 }; 579 580} 581 582