ItsService.java revision 7c6e563c9fdaedc03b838bf9f3a3806cf396be35
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.camera2.its; 18 19import android.app.Service; 20import android.content.Context; 21import android.content.Intent; 22import android.graphics.ImageFormat; 23import android.hardware.camera2.CameraAccessException; 24import android.hardware.camera2.CameraDevice; 25import android.hardware.camera2.CameraManager; 26import android.hardware.camera2.CameraProperties; 27import android.hardware.camera2.CaptureRequest; 28import android.hardware.camera2.CaptureResult; 29import android.media.Image; 30import android.media.ImageReader; 31import android.net.Uri; 32import android.os.ConditionVariable; 33import android.os.Handler; 34import android.os.HandlerThread; 35import android.os.IBinder; 36import android.os.Message; 37import android.util.Log; 38import android.view.Surface; 39 40import org.json.JSONObject; 41 42import java.io.File; 43import java.nio.ByteBuffer; 44import java.util.ArrayList; 45import java.util.Arrays; 46import java.util.List; 47import java.util.concurrent.CountDownLatch; 48import java.util.concurrent.TimeUnit; 49 50public class ItsService extends Service { 51 public static final String TAG = ItsService.class.getSimpleName(); 52 public static final String PYTAG = "CAMERA-ITS-PY"; 53 54 // Supported intents 55 public static final String ACTION_CAPTURE = "com.android.camera2.its.CAPTURE"; 56 public static final String ACTION_3A = "com.android.camera2.its.3A"; 57 public static final String ACTION_GETPROPS = "com.android.camera2.its.GETPROPS"; 58 private static final int MESSAGE_CAPTURE = 1; 59 private static final int MESSAGE_3A = 2; 60 private static final int MESSAGE_GETPROPS = 3; 61 62 // Timeouts, in seconds. 63 public static final int TIMEOUT_CAPTURE = 10; 64 public static final int TIMEOUT_3A = 10; 65 66 private static final int MAX_CONCURRENT_READER_BUFFERS = 8; 67 68 public static final String REGION_KEY = "regions"; 69 public static final String REGION_AE_KEY = "ae"; 70 public static final String REGION_AWB_KEY = "awb"; 71 public static final String REGION_AF_KEY = "af"; 72 73 private CameraManager mCameraManager = null; 74 private CameraDevice mCamera = null; 75 private ImageReader mCaptureReader = null; 76 private CameraProperties mCameraProperties = null; 77 78 private HandlerThread mCommandThread; 79 private Handler mCommandHandler; 80 private HandlerThread mSaveThread; 81 private Handler mSaveHandler; 82 83 private ConditionVariable mInterlock3A = new ConditionVariable(true); 84 private volatile boolean mIssuedRequest3A = false; 85 private volatile boolean mConvergedAE = false; 86 private volatile boolean mConvergedAF = false; 87 private volatile boolean mConvergedAWB = false; 88 89 private CountDownLatch mCaptureCallbackLatch; 90 91 public interface CaptureListener { 92 void onCaptureAvailable(Image capture); 93 } 94 95 public interface CaptureResultListener extends CameraDevice.CaptureListener {} 96 97 @Override 98 public IBinder onBind(Intent intent) { 99 return null; 100 } 101 102 @Override 103 public void onCreate() { 104 105 try { 106 // Get handle to camera manager. 107 mCameraManager = (CameraManager) this.getSystemService(Context.CAMERA_SERVICE); 108 if (mCameraManager == null) { 109 throw new ItsException("Failed to connect to camera manager"); 110 } 111 112 // Open the camera device, and get its properties. 113 String[] devices; 114 try { 115 devices = mCameraManager.getDeviceIdList(); 116 if (devices == null || devices.length == 0) { 117 throw new ItsException("No camera devices"); 118 } 119 120 // TODO: Add support for specifying which device to open. 121 mCamera = mCameraManager.openCamera(devices[0]); 122 mCameraProperties = mCamera.getProperties(); 123 } catch (CameraAccessException e) { 124 throw new ItsException("Failed to get device ID list"); 125 } 126 127 // Create a thread to receive images and save them. 128 mSaveThread = new HandlerThread("SaveThread"); 129 mSaveThread.start(); 130 mSaveHandler = new Handler(mSaveThread.getLooper()); 131 132 // Create a thread to process commands. 133 mCommandThread = new HandlerThread("CaptureThread"); 134 mCommandThread.start(); 135 mCommandHandler = new Handler(mCommandThread.getLooper(), new Handler.Callback() { 136 @Override 137 public boolean handleMessage(Message msg) { 138 try { 139 switch (msg.what) { 140 case MESSAGE_CAPTURE: 141 doCapture((Uri) msg.obj); 142 break; 143 case MESSAGE_3A: 144 do3A((Uri) msg.obj); 145 break; 146 case MESSAGE_GETPROPS: 147 doGetProps(); 148 break; 149 default: 150 throw new ItsException("Unknown message type"); 151 } 152 Log.i(PYTAG, "### DONE"); 153 return true; 154 } 155 catch (ItsException e) { 156 Log.e(TAG, "Script failed: ", e); 157 Log.e(PYTAG, "### FAIL"); 158 return true; 159 } 160 } 161 }); 162 } catch (ItsException e) { 163 Log.e(TAG, "Script failed: ", e); 164 Log.e(PYTAG, "### FAIL"); 165 } 166 } 167 168 @Override 169 public void onDestroy() { 170 try { 171 if (mCommandThread != null) { 172 mCommandThread.quit(); 173 mCommandThread = null; 174 } 175 if (mSaveThread != null) { 176 mSaveThread.quit(); 177 mSaveThread = null; 178 } 179 180 try { 181 mCamera.close(); 182 } catch (Exception e) { 183 throw new ItsException("Failed to close device"); 184 } 185 } catch (ItsException e) { 186 Log.e(TAG, "Script failed: ", e); 187 Log.e(PYTAG, "### FAIL"); 188 } 189 } 190 191 @Override 192 public int onStartCommand(Intent intent, int flags, int startId) { 193 try { 194 Log.i(PYTAG, "### RECV"); 195 String action = intent.getAction(); 196 if (ACTION_CAPTURE.equals(action)) { 197 Uri uri = intent.getData(); 198 Message m = mCommandHandler.obtainMessage(MESSAGE_CAPTURE, uri); 199 mCommandHandler.sendMessage(m); 200 } else if (ACTION_3A.equals(action)) { 201 Uri uri = intent.getData(); 202 Message m = mCommandHandler.obtainMessage(MESSAGE_3A, uri); 203 mCommandHandler.sendMessage(m); 204 } else if (ACTION_GETPROPS.equals(action)) { 205 Uri uri = intent.getData(); 206 Message m = mCommandHandler.obtainMessage(MESSAGE_GETPROPS, uri); 207 mCommandHandler.sendMessage(m); 208 } else { 209 throw new ItsException("Unhandled intent: " + intent.toString()); 210 } 211 } catch (ItsException e) { 212 Log.e(TAG, "Script failed: ", e); 213 Log.e(PYTAG, "### FAIL"); 214 } 215 return START_STICKY; 216 } 217 218 public void idleCamera() throws ItsException { 219 try { 220 mCamera.stopRepeating(); 221 mCamera.waitUntilIdle(); 222 } catch (CameraAccessException e) { 223 throw new ItsException("Error waiting for camera idle", e); 224 } 225 } 226 227 private ImageReader.OnImageAvailableListener 228 createAvailableListener(final CaptureListener listener) { 229 return new ImageReader.OnImageAvailableListener() { 230 @Override 231 public void onImageAvailable(ImageReader reader) { 232 Image i = reader.getNextImage(); 233 listener.onCaptureAvailable(i); 234 i.close(); 235 } 236 }; 237 } 238 239 private ImageReader.OnImageAvailableListener 240 createAvailableListenerDropper(final CaptureListener listener) { 241 return new ImageReader.OnImageAvailableListener() { 242 @Override 243 public void onImageAvailable(ImageReader reader) { 244 Image i = reader.getNextImage(); 245 i.close(); 246 } 247 }; 248 } 249 250 private void doGetProps() throws ItsException { 251 String fileName = ItsUtils.getMetadataFileName(0); 252 File mdFile = ItsUtils.getOutputFile(ItsService.this, fileName); 253 ItsUtils.storeCameraProperties(mCameraProperties, mdFile); 254 Log.i(PYTAG, 255 String.format("### FILE %s", 256 ItsUtils.getExternallyVisiblePath(ItsService.this, mdFile.toString()))); 257 } 258 259 private void prepareCaptureReader(int width, int height, int format) { 260 if (mCaptureReader == null 261 || mCaptureReader.getWidth() != width 262 || mCaptureReader.getHeight() != height 263 || mCaptureReader.getImageFormat() != format) { 264 if (mCaptureReader != null) { 265 mCaptureReader.close(); 266 } 267 mCaptureReader = new ImageReader(width, height, format, 268 MAX_CONCURRENT_READER_BUFFERS); 269 } 270 } 271 272 private void do3A(Uri uri) throws ItsException { 273 try { 274 if (uri == null || !uri.toString().endsWith(".json")) { 275 throw new ItsException("Invalid URI: " + uri); 276 } 277 278 idleCamera(); 279 280 // Start a 3A action, and wait for it to converge. 281 // Get the converged values for each "A", and package into JSON result for caller. 282 283 // 3A happens on full-res frames. 284 android.hardware.camera2.Size sizes[] = mCameraProperties.get( 285 CameraProperties.SCALER_AVAILABLE_JPEG_SIZES); 286 int width = sizes[0].getWidth(); 287 int height = sizes[0].getHeight(); 288 int format = ImageFormat.YUV_420_888; 289 290 prepareCaptureReader(width, height, format); 291 List<Surface> outputSurfaces = new ArrayList<Surface>(1); 292 outputSurfaces.add(mCaptureReader.getSurface()); 293 mCamera.configureOutputs(outputSurfaces); 294 295 // Add a listener that just recycles buffers; they aren't saved anywhere. 296 ImageReader.OnImageAvailableListener readerListener = 297 createAvailableListenerDropper(mCaptureListener); 298 mCaptureReader.setImageAvailableListener(readerListener, mSaveHandler); 299 300 // Get the user-specified regions for AE, AWB, AF. 301 // Note that the user specifies normalized [x,y,w,h], which is converted below 302 // to an [x0,y0,x1,y1] region in sensor coords. The capture request region 303 // also has a fifth "weight" element: [x0,y0,x1,y1,w]. 304 int[] regionAE = new int[]{0,0,width-1,height-1,1}; 305 int[] regionAF = new int[]{0,0,width-1,height-1,1}; 306 int[] regionAWB = new int[]{0,0,width-1,height-1,1}; 307 JSONObject params = ItsUtils.loadJsonFile(uri); 308 if (params.has(REGION_KEY)) { 309 JSONObject regions = params.getJSONObject(REGION_KEY); 310 if (params.has(REGION_AE_KEY)) { 311 int[] r = ItsUtils.getJsonRectFromArray( 312 params.getJSONArray(REGION_AE_KEY), true, width, height); 313 regionAE = new int[]{r[0],r[1],r[0]+r[2]-1,r[1]+r[3]-1,1}; 314 } 315 if (params.has(REGION_AF_KEY)) { 316 int[] r = ItsUtils.getJsonRectFromArray( 317 params.getJSONArray(REGION_AF_KEY), true, width, height); 318 regionAF = new int[]{r[0],r[1],r[0]+r[2]-1,r[1]+r[3]-1,1}; 319 } 320 if (params.has(REGION_AWB_KEY)) { 321 int[] r = ItsUtils.getJsonRectFromArray( 322 params.getJSONArray(REGION_AWB_KEY), true, width, height); 323 regionAWB = new int[]{r[0],r[1],r[0]+r[2]-1,r[1]+r[3]-1,1}; 324 } 325 } 326 Log.i(TAG, "AE region: " + Arrays.toString(regionAE)); 327 Log.i(TAG, "AF region: " + Arrays.toString(regionAF)); 328 Log.i(TAG, "AWB region: " + Arrays.toString(regionAWB)); 329 330 mInterlock3A.open(); 331 mIssuedRequest3A = false; 332 mConvergedAE = false; 333 mConvergedAWB = false; 334 mConvergedAF = false; 335 long tstart = System.currentTimeMillis(); 336 boolean triggeredAE = false; 337 boolean triggeredAF = false; 338 339 // Keep issuing capture requests until 3A has converged. 340 // First do AE, then do AF and AWB together. 341 while (true) { 342 343 // Block until can take the next 3A frame. Only want one outstanding frame 344 // at a time, to simplify the logic here. 345 if (!mInterlock3A.block(TIMEOUT_3A * 1000) || 346 System.currentTimeMillis() - tstart > TIMEOUT_3A * 1000) { 347 throw new ItsException("3A failed to converge (timeout)"); 348 } 349 mInterlock3A.close(); 350 351 // If not converged yet, issue another capture request. 352 if (!mConvergedAE || !mConvergedAWB || !mConvergedAF) { 353 354 // Baseline capture request for 3A. 355 CaptureRequest req = mCamera.createCaptureRequest( 356 CameraDevice.TEMPLATE_PREVIEW); 357 req.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_OFF); 358 req.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO); 359 req.set(CaptureRequest.CONTROL_CAPTURE_INTENT, 360 CaptureRequest.CONTROL_CAPTURE_INTENT_PREVIEW); 361 req.set(CaptureRequest.CONTROL_AE_MODE, 362 CaptureRequest.CONTROL_AE_MODE_ON); 363 req.set(CaptureRequest.CONTROL_AE_EXPOSURE_COMPENSATION, 0); 364 req.set(CaptureRequest.CONTROL_AE_LOCK, false); 365 req.set(CaptureRequest.CONTROL_AE_REGIONS, regionAE); 366 req.set(CaptureRequest.CONTROL_AF_MODE, 367 CaptureRequest.CONTROL_AF_MODE_AUTO); 368 req.set(CaptureRequest.CONTROL_AF_REGIONS, regionAF); 369 req.set(CaptureRequest.CONTROL_AWB_MODE, 370 CaptureRequest.CONTROL_AWB_MODE_AUTO); 371 req.set(CaptureRequest.CONTROL_AWB_LOCK, false); 372 req.set(CaptureRequest.CONTROL_AWB_REGIONS, regionAWB); 373 374 // Trigger AE first. 375 if (!triggeredAE) { 376 Log.i(TAG, "Triggering AE"); 377 req.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, 378 CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_START); 379 triggeredAE = true; 380 } 381 382 // After AE has converged, trigger AF. 383 if (!triggeredAF && triggeredAE && mConvergedAE) { 384 Log.i(TAG, "Triggering AF"); 385 req.set(CaptureRequest.CONTROL_AF_TRIGGER, 386 CaptureRequest.CONTROL_AF_TRIGGER_START); 387 triggeredAF = true; 388 } 389 390 req.addTarget(mCaptureReader.getSurface()); 391 392 mIssuedRequest3A = true; 393 mCamera.capture(req, mCaptureResultListener); 394 } else { 395 Log.i(TAG, "3A converged"); 396 break; 397 } 398 } 399 } catch (android.hardware.camera2.CameraAccessException e) { 400 throw new ItsException("Access error: ", e); 401 } catch (org.json.JSONException e) { 402 throw new ItsException("JSON error: ", e); 403 } 404 } 405 406 private void doCapture(Uri uri) throws ItsException { 407 try { 408 if (uri == null || !uri.toString().endsWith(".json")) { 409 throw new ItsException("Invalid URI: " + uri); 410 } 411 412 idleCamera(); 413 414 // Parse the JSON to get the list of capture requests. 415 List<CaptureRequest> requests = ItsUtils.loadRequestList(mCamera, uri); 416 417 // Set the output surface and listeners. 418 try { 419 // Default: 420 // Capture full-frame images. Use the reported JPEG size rather than the sensor 421 // size since this is more likely to be the unscaled size; the crop from sensor 422 // size is probably for the ISP (e.g. demosaicking) rather than the encoder. 423 android.hardware.camera2.Size sizes[] = mCameraProperties.get( 424 CameraProperties.SCALER_AVAILABLE_JPEG_SIZES); 425 int width = sizes[0].getWidth(); 426 int height = sizes[0].getHeight(); 427 int format = ImageFormat.YUV_420_888; 428 429 JSONObject jsonOutputSpecs = ItsUtils.getOutputSpecs(uri); 430 if (jsonOutputSpecs != null) { 431 // Use the user's JSON capture spec. 432 int width2 = jsonOutputSpecs.optInt("width"); 433 int height2 = jsonOutputSpecs.optInt("height"); 434 if (width2 > 0) { 435 width = width2; 436 } 437 if (height2 > 0) { 438 height = height2; 439 } 440 String sformat = jsonOutputSpecs.optString("format"); 441 if ("yuv".equals(sformat)) { 442 format = ImageFormat.YUV_420_888; 443 } else if ("jpg".equals(sformat) || "jpeg".equals(sformat)) { 444 format = ImageFormat.JPEG; 445 } else if ("".equals(sformat)) { 446 // No format specified. 447 } else { 448 throw new ItsException("Unsupported format: " + sformat); 449 } 450 } 451 452 Log.i(PYTAG, String.format("### SIZE %d %d", width, height)); 453 454 prepareCaptureReader(width, height, format); 455 List<Surface> outputSurfaces = new ArrayList<Surface>(1); 456 outputSurfaces.add(mCaptureReader.getSurface()); 457 mCamera.configureOutputs(outputSurfaces); 458 459 ImageReader.OnImageAvailableListener readerListener = 460 createAvailableListener(mCaptureListener); 461 mCaptureReader.setImageAvailableListener(readerListener, mSaveHandler); 462 463 // Plan for how many callbacks need to be received throughout the duration of this 464 // sequence of capture requests. 465 int numCaptures = requests.size(); 466 mCaptureCallbackLatch = new CountDownLatch( 467 numCaptures * ItsUtils.getCallbacksPerCapture(format)); 468 469 } catch (CameraAccessException e) { 470 throw new ItsException("Error configuring outputs", e); 471 } 472 473 // Initiate the captures. 474 for (int i = 0; i < requests.size(); i++) { 475 CaptureRequest req = requests.get(i); 476 Log.i(PYTAG, String.format("### CAPT %d of %d", i+1, requests.size())); 477 req.addTarget(mCaptureReader.getSurface()); 478 mCamera.capture(req, mCaptureResultListener); 479 } 480 481 // Make sure all callbacks have been hit (wait until captures are done). 482 try { 483 if (!mCaptureCallbackLatch.await(TIMEOUT_CAPTURE, TimeUnit.SECONDS)) { 484 throw new ItsException( 485 "Timeout hit, but all callbacks not received"); 486 } 487 } catch (InterruptedException e) { 488 throw new ItsException("Interrupted: ", e); 489 } 490 491 } catch (android.hardware.camera2.CameraAccessException e) { 492 throw new ItsException("Access error: ", e); 493 } 494 } 495 496 private final CaptureListener mCaptureListener = new CaptureListener() { 497 @Override 498 public void onCaptureAvailable(Image capture) { 499 try { 500 int format = capture.getFormat(); 501 String extFileName = null; 502 if (format == ImageFormat.JPEG) { 503 String fileName = ItsUtils.getJpegFileName(capture.getTimestamp()); 504 ByteBuffer buf = capture.getPlanes()[0].getBuffer(); 505 extFileName = ItsUtils.writeImageToFile(ItsService.this, buf, fileName); 506 } else if (format == ImageFormat.YUV_420_888) { 507 String fileName = ItsUtils.getYuvFileName(capture.getTimestamp()); 508 byte[] img = ItsUtils.getDataFromImage(capture); 509 ByteBuffer buf = ByteBuffer.wrap(img); 510 extFileName = ItsUtils.writeImageToFile(ItsService.this, buf, fileName); 511 } else { 512 throw new ItsException("Unsupported image format: " + format); 513 } 514 Log.i(PYTAG, String.format("### FILE %s", extFileName)); 515 mCaptureCallbackLatch.countDown(); 516 } catch (ItsException e) { 517 Log.e(TAG, "Script error: " + e); 518 Log.e(PYTAG, "### FAIL"); 519 } 520 } 521 }; 522 523 private final CaptureResultListener mCaptureResultListener = new CaptureResultListener() { 524 @Override 525 public void onCaptureComplete(CameraDevice camera, CaptureRequest request, 526 CaptureResult result) { 527 try { 528 // Currently result has all 0 values. 529 if (request == null || result == null) { 530 throw new ItsException("Request/result is invalid"); 531 } 532 533 Log.i(TAG, String.format( 534 "Capture result: AE=%d, AF=%d, AWB=%d, sens=%d, exp=%dms, dur=%dms", 535 result.get(CaptureResult.CONTROL_AE_STATE), 536 result.get(CaptureResult.CONTROL_AF_STATE), 537 result.get(CaptureResult.CONTROL_AWB_STATE), 538 result.get(CaptureResult.SENSOR_SENSITIVITY), 539 result.get(CaptureResult.SENSOR_EXPOSURE_TIME).intValue(), 540 result.get(CaptureResult.SENSOR_FRAME_DURATION).intValue())); 541 542 mConvergedAE = result.get(CaptureResult.CONTROL_AE_STATE) == 543 CaptureResult.CONTROL_AE_STATE_CONVERGED; 544 mConvergedAF = result.get(CaptureResult.CONTROL_AF_STATE) == 545 CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED; 546 mConvergedAWB = result.get(CaptureResult.CONTROL_AWB_STATE) == 547 CaptureResult.CONTROL_AWB_STATE_CONVERGED; 548 549 if (mIssuedRequest3A) { 550 mIssuedRequest3A = false; 551 mInterlock3A.open(); 552 } else { 553 String fileName = ItsUtils.getMetadataFileName( 554 result.get(CaptureResult.SENSOR_TIMESTAMP)); 555 File mdFile = ItsUtils.getOutputFile(ItsService.this, fileName); 556 ItsUtils.storeResults(mCameraProperties, request, result, mdFile); 557 mCaptureCallbackLatch.countDown(); 558 } 559 } catch (ItsException e) { 560 Log.e(TAG, "Script error: " + e); 561 Log.e(PYTAG, "### FAIL"); 562 } catch (Exception e) { 563 Log.e(TAG, "Script error: " + e); 564 Log.e(PYTAG, "### FAIL"); 565 } 566 } 567 568 @Override 569 public void onCaptureFailed(CameraDevice camera, CaptureRequest request) { 570 mCaptureCallbackLatch.countDown(); 571 Log.e(TAG, "Script error: capture failed"); 572 Log.e(PYTAG, "### FAIL"); 573 } 574 }; 575 576} 577 578