ItsService.java revision 7f84538d28ba6df5e8f55b3770ef78476c9aab74
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.camera2.its; 18 19import android.app.Service; 20import android.content.Context; 21import android.content.Intent; 22import android.graphics.ImageFormat; 23import android.hardware.camera2.CameraAccessException; 24import android.hardware.camera2.CameraDevice; 25import android.hardware.camera2.CameraManager; 26import android.hardware.camera2.CameraProperties; 27import android.hardware.camera2.CaptureRequest; 28import android.hardware.camera2.CaptureResult; 29import android.media.Image; 30import android.media.ImageReader; 31import android.net.Uri; 32import android.os.ConditionVariable; 33import android.os.Handler; 34import android.os.HandlerThread; 35import android.os.IBinder; 36import android.os.Message; 37import android.util.Log; 38import android.view.Surface; 39 40import org.json.JSONObject; 41 42import java.io.File; 43import java.nio.ByteBuffer; 44import java.util.ArrayList; 45import java.util.Arrays; 46import java.util.List; 47import java.util.concurrent.CountDownLatch; 48import java.util.concurrent.TimeUnit; 49 50public class ItsService extends Service { 51 public static final String TAG = ItsService.class.getSimpleName(); 52 public static final String PYTAG = "CAMERA-ITS-PY"; 53 54 // Supported intents 55 public static final String ACTION_CAPTURE = "com.android.camera2.its.CAPTURE"; 56 public static final String ACTION_3A = "com.android.camera2.its.3A"; 57 public static final String ACTION_GETPROPS = "com.android.camera2.its.GETPROPS"; 58 private static final int MESSAGE_CAPTURE = 1; 59 private static final int MESSAGE_3A = 2; 60 private static final int MESSAGE_GETPROPS = 3; 61 62 // Timeouts, in seconds. 63 public static final int TIMEOUT_CAPTURE = 10; 64 public static final int TIMEOUT_3A = 10; 65 66 private static final int MAX_CONCURRENT_READER_BUFFERS = 8; 67 68 public static final String REGION_KEY = "regions"; 69 public static final String REGION_AE_KEY = "ae"; 70 public static final String REGION_AWB_KEY = "awb"; 71 public static final String REGION_AF_KEY = "af"; 72 73 private CameraManager mCameraManager = null; 74 private CameraDevice mCamera = null; 75 private ImageReader mCaptureReader = null; 76 private CameraProperties mCameraProperties = null; 77 78 private HandlerThread mCommandThread; 79 private Handler mCommandHandler; 80 private HandlerThread mSaveThread; 81 private Handler mSaveHandler; 82 private HandlerThread mResultThread; 83 private Handler mResultHandler; 84 85 private ConditionVariable mInterlock3A = new ConditionVariable(true); 86 private volatile boolean mIssuedRequest3A = false; 87 private volatile boolean mConvergedAE = false; 88 private volatile boolean mConvergedAF = false; 89 private volatile boolean mConvergedAWB = false; 90 91 private CountDownLatch mCaptureCallbackLatch; 92 93 public interface CaptureListener { 94 void onCaptureAvailable(Image capture); 95 } 96 97 public abstract class CaptureResultListener extends CameraDevice.CaptureListener {} 98 99 @Override 100 public IBinder onBind(Intent intent) { 101 return null; 102 } 103 104 @Override 105 public void onCreate() { 106 107 try { 108 // Get handle to camera manager. 109 mCameraManager = (CameraManager) this.getSystemService(Context.CAMERA_SERVICE); 110 if (mCameraManager == null) { 111 throw new ItsException("Failed to connect to camera manager"); 112 } 113 114 // Open the camera device, and get its properties. 115 String[] devices; 116 try { 117 devices = mCameraManager.getCameraIdList(); 118 if (devices == null || devices.length == 0) { 119 throw new ItsException("No camera devices"); 120 } 121 122 // TODO: Add support for specifying which device to open. 123 mCamera = mCameraManager.openCamera(devices[0]); 124 mCameraProperties = mCamera.getProperties(); 125 } catch (CameraAccessException e) { 126 throw new ItsException("Failed to get device ID list"); 127 } 128 129 // Create a thread to receive images and save them. 130 mSaveThread = new HandlerThread("SaveThread"); 131 mSaveThread.start(); 132 mSaveHandler = new Handler(mSaveThread.getLooper()); 133 134 // Create a thread to receive capture results and process them 135 mResultThread = new HandlerThread("ResultThread"); 136 mResultThread.start(); 137 mResultHandler = new Handler(mResultThread.getLooper()); 138 139 // Create a thread to process commands. 140 mCommandThread = new HandlerThread("CaptureThread"); 141 mCommandThread.start(); 142 mCommandHandler = new Handler(mCommandThread.getLooper(), new Handler.Callback() { 143 @Override 144 public boolean handleMessage(Message msg) { 145 try { 146 switch (msg.what) { 147 case MESSAGE_CAPTURE: 148 doCapture((Uri) msg.obj); 149 break; 150 case MESSAGE_3A: 151 do3A((Uri) msg.obj); 152 break; 153 case MESSAGE_GETPROPS: 154 doGetProps(); 155 break; 156 default: 157 throw new ItsException("Unknown message type"); 158 } 159 Log.i(PYTAG, "### DONE"); 160 return true; 161 } 162 catch (ItsException e) { 163 Log.e(TAG, "Script failed: ", e); 164 Log.e(PYTAG, "### FAIL"); 165 return true; 166 } 167 } 168 }); 169 } catch (ItsException e) { 170 Log.e(TAG, "Script failed: ", e); 171 Log.e(PYTAG, "### FAIL"); 172 } 173 } 174 175 @Override 176 public void onDestroy() { 177 try { 178 if (mCommandThread != null) { 179 mCommandThread.quit(); 180 mCommandThread = null; 181 } 182 if (mSaveThread != null) { 183 mSaveThread.quit(); 184 mSaveThread = null; 185 } 186 187 try { 188 mCamera.close(); 189 } catch (Exception e) { 190 throw new ItsException("Failed to close device"); 191 } 192 } catch (ItsException e) { 193 Log.e(TAG, "Script failed: ", e); 194 Log.e(PYTAG, "### FAIL"); 195 } 196 } 197 198 @Override 199 public int onStartCommand(Intent intent, int flags, int startId) { 200 try { 201 Log.i(PYTAG, "### RECV"); 202 String action = intent.getAction(); 203 if (ACTION_CAPTURE.equals(action)) { 204 Uri uri = intent.getData(); 205 Message m = mCommandHandler.obtainMessage(MESSAGE_CAPTURE, uri); 206 mCommandHandler.sendMessage(m); 207 } else if (ACTION_3A.equals(action)) { 208 Uri uri = intent.getData(); 209 Message m = mCommandHandler.obtainMessage(MESSAGE_3A, uri); 210 mCommandHandler.sendMessage(m); 211 } else if (ACTION_GETPROPS.equals(action)) { 212 Uri uri = intent.getData(); 213 Message m = mCommandHandler.obtainMessage(MESSAGE_GETPROPS, uri); 214 mCommandHandler.sendMessage(m); 215 } else { 216 throw new ItsException("Unhandled intent: " + intent.toString()); 217 } 218 } catch (ItsException e) { 219 Log.e(TAG, "Script failed: ", e); 220 Log.e(PYTAG, "### FAIL"); 221 } 222 return START_STICKY; 223 } 224 225 public void idleCamera() throws ItsException { 226 try { 227 mCamera.stopRepeating(); 228 mCamera.waitUntilIdle(); 229 } catch (CameraAccessException e) { 230 throw new ItsException("Error waiting for camera idle", e); 231 } 232 } 233 234 private ImageReader.OnImageAvailableListener 235 createAvailableListener(final CaptureListener listener) { 236 return new ImageReader.OnImageAvailableListener() { 237 @Override 238 public void onImageAvailable(ImageReader reader) { 239 Image i = reader.getNextImage(); 240 listener.onCaptureAvailable(i); 241 i.close(); 242 } 243 }; 244 } 245 246 private ImageReader.OnImageAvailableListener 247 createAvailableListenerDropper(final CaptureListener listener) { 248 return new ImageReader.OnImageAvailableListener() { 249 @Override 250 public void onImageAvailable(ImageReader reader) { 251 Image i = reader.getNextImage(); 252 i.close(); 253 } 254 }; 255 } 256 257 private void doGetProps() throws ItsException { 258 String fileName = ItsUtils.getMetadataFileName(0); 259 File mdFile = ItsUtils.getOutputFile(ItsService.this, fileName); 260 ItsUtils.storeCameraProperties(mCameraProperties, mdFile); 261 Log.i(PYTAG, 262 String.format("### FILE %s", 263 ItsUtils.getExternallyVisiblePath(ItsService.this, mdFile.toString()))); 264 } 265 266 private void prepareCaptureReader(int width, int height, int format) { 267 if (mCaptureReader == null 268 || mCaptureReader.getWidth() != width 269 || mCaptureReader.getHeight() != height 270 || mCaptureReader.getImageFormat() != format) { 271 if (mCaptureReader != null) { 272 mCaptureReader.close(); 273 } 274 mCaptureReader = new ImageReader(width, height, format, 275 MAX_CONCURRENT_READER_BUFFERS); 276 } 277 } 278 279 private void do3A(Uri uri) throws ItsException { 280 try { 281 if (uri == null || !uri.toString().endsWith(".json")) { 282 throw new ItsException("Invalid URI: " + uri); 283 } 284 285 idleCamera(); 286 287 // Start a 3A action, and wait for it to converge. 288 // Get the converged values for each "A", and package into JSON result for caller. 289 290 // 3A happens on full-res frames. 291 android.hardware.camera2.Size sizes[] = mCameraProperties.get( 292 CameraProperties.SCALER_AVAILABLE_JPEG_SIZES); 293 int width = sizes[0].getWidth(); 294 int height = sizes[0].getHeight(); 295 int format = ImageFormat.YUV_420_888; 296 297 prepareCaptureReader(width, height, format); 298 List<Surface> outputSurfaces = new ArrayList<Surface>(1); 299 outputSurfaces.add(mCaptureReader.getSurface()); 300 mCamera.configureOutputs(outputSurfaces); 301 302 // Add a listener that just recycles buffers; they aren't saved anywhere. 303 ImageReader.OnImageAvailableListener readerListener = 304 createAvailableListenerDropper(mCaptureListener); 305 mCaptureReader.setImageAvailableListener(readerListener, mSaveHandler); 306 307 // Get the user-specified regions for AE, AWB, AF. 308 // Note that the user specifies normalized [x,y,w,h], which is converted below 309 // to an [x0,y0,x1,y1] region in sensor coords. The capture request region 310 // also has a fifth "weight" element: [x0,y0,x1,y1,w]. 311 int[] regionAE = new int[]{0,0,width-1,height-1,1}; 312 int[] regionAF = new int[]{0,0,width-1,height-1,1}; 313 int[] regionAWB = new int[]{0,0,width-1,height-1,1}; 314 JSONObject params = ItsUtils.loadJsonFile(uri); 315 if (params.has(REGION_KEY)) { 316 JSONObject regions = params.getJSONObject(REGION_KEY); 317 if (params.has(REGION_AE_KEY)) { 318 int[] r = ItsUtils.getJsonRectFromArray( 319 params.getJSONArray(REGION_AE_KEY), true, width, height); 320 regionAE = new int[]{r[0],r[1],r[0]+r[2]-1,r[1]+r[3]-1,1}; 321 } 322 if (params.has(REGION_AF_KEY)) { 323 int[] r = ItsUtils.getJsonRectFromArray( 324 params.getJSONArray(REGION_AF_KEY), true, width, height); 325 regionAF = new int[]{r[0],r[1],r[0]+r[2]-1,r[1]+r[3]-1,1}; 326 } 327 if (params.has(REGION_AWB_KEY)) { 328 int[] r = ItsUtils.getJsonRectFromArray( 329 params.getJSONArray(REGION_AWB_KEY), true, width, height); 330 regionAWB = new int[]{r[0],r[1],r[0]+r[2]-1,r[1]+r[3]-1,1}; 331 } 332 } 333 Log.i(TAG, "AE region: " + Arrays.toString(regionAE)); 334 Log.i(TAG, "AF region: " + Arrays.toString(regionAF)); 335 Log.i(TAG, "AWB region: " + Arrays.toString(regionAWB)); 336 337 mInterlock3A.open(); 338 mIssuedRequest3A = false; 339 mConvergedAE = false; 340 mConvergedAWB = false; 341 mConvergedAF = false; 342 long tstart = System.currentTimeMillis(); 343 boolean triggeredAE = false; 344 boolean triggeredAF = false; 345 346 // Keep issuing capture requests until 3A has converged. 347 // First do AE, then do AF and AWB together. 348 while (true) { 349 350 // Block until can take the next 3A frame. Only want one outstanding frame 351 // at a time, to simplify the logic here. 352 if (!mInterlock3A.block(TIMEOUT_3A * 1000) || 353 System.currentTimeMillis() - tstart > TIMEOUT_3A * 1000) { 354 throw new ItsException("3A failed to converge (timeout)"); 355 } 356 mInterlock3A.close(); 357 358 // If not converged yet, issue another capture request. 359 if (!mConvergedAE || !mConvergedAWB || !mConvergedAF) { 360 361 // Baseline capture request for 3A. 362 CaptureRequest.Builder req = mCamera.createCaptureRequest( 363 CameraDevice.TEMPLATE_PREVIEW); 364 req.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_OFF); 365 req.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO); 366 req.set(CaptureRequest.CONTROL_CAPTURE_INTENT, 367 CaptureRequest.CONTROL_CAPTURE_INTENT_PREVIEW); 368 req.set(CaptureRequest.CONTROL_AE_MODE, 369 CaptureRequest.CONTROL_AE_MODE_ON); 370 req.set(CaptureRequest.CONTROL_AE_EXPOSURE_COMPENSATION, 0); 371 req.set(CaptureRequest.CONTROL_AE_LOCK, false); 372 req.set(CaptureRequest.CONTROL_AE_REGIONS, regionAE); 373 req.set(CaptureRequest.CONTROL_AF_MODE, 374 CaptureRequest.CONTROL_AF_MODE_AUTO); 375 req.set(CaptureRequest.CONTROL_AF_REGIONS, regionAF); 376 req.set(CaptureRequest.CONTROL_AWB_MODE, 377 CaptureRequest.CONTROL_AWB_MODE_AUTO); 378 req.set(CaptureRequest.CONTROL_AWB_LOCK, false); 379 req.set(CaptureRequest.CONTROL_AWB_REGIONS, regionAWB); 380 381 // Trigger AE first. 382 if (!triggeredAE) { 383 Log.i(TAG, "Triggering AE"); 384 req.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, 385 CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_START); 386 triggeredAE = true; 387 } 388 389 // After AE has converged, trigger AF. 390 if (!triggeredAF && triggeredAE && mConvergedAE) { 391 Log.i(TAG, "Triggering AF"); 392 req.set(CaptureRequest.CONTROL_AF_TRIGGER, 393 CaptureRequest.CONTROL_AF_TRIGGER_START); 394 triggeredAF = true; 395 } 396 397 req.addTarget(mCaptureReader.getSurface()); 398 399 mIssuedRequest3A = true; 400 mCamera.capture(req.build(), mCaptureResultListener, mResultHandler); 401 } else { 402 Log.i(TAG, "3A converged"); 403 break; 404 } 405 } 406 } catch (android.hardware.camera2.CameraAccessException e) { 407 throw new ItsException("Access error: ", e); 408 } catch (org.json.JSONException e) { 409 throw new ItsException("JSON error: ", e); 410 } 411 } 412 413 private void doCapture(Uri uri) throws ItsException { 414 try { 415 if (uri == null || !uri.toString().endsWith(".json")) { 416 throw new ItsException("Invalid URI: " + uri); 417 } 418 419 idleCamera(); 420 421 // Parse the JSON to get the list of capture requests. 422 List<CaptureRequest.Builder> requests = ItsUtils.loadRequestList(mCamera, uri); 423 424 // Set the output surface and listeners. 425 try { 426 // Default: 427 // Capture full-frame images. Use the reported JPEG size rather than the sensor 428 // size since this is more likely to be the unscaled size; the crop from sensor 429 // size is probably for the ISP (e.g. demosaicking) rather than the encoder. 430 android.hardware.camera2.Size sizes[] = mCameraProperties.get( 431 CameraProperties.SCALER_AVAILABLE_JPEG_SIZES); 432 int width = sizes[0].getWidth(); 433 int height = sizes[0].getHeight(); 434 int format = ImageFormat.YUV_420_888; 435 436 JSONObject jsonOutputSpecs = ItsUtils.getOutputSpecs(uri); 437 if (jsonOutputSpecs != null) { 438 // Use the user's JSON capture spec. 439 int width2 = jsonOutputSpecs.optInt("width"); 440 int height2 = jsonOutputSpecs.optInt("height"); 441 if (width2 > 0) { 442 width = width2; 443 } 444 if (height2 > 0) { 445 height = height2; 446 } 447 String sformat = jsonOutputSpecs.optString("format"); 448 if ("yuv".equals(sformat)) { 449 format = ImageFormat.YUV_420_888; 450 } else if ("jpg".equals(sformat) || "jpeg".equals(sformat)) { 451 format = ImageFormat.JPEG; 452 } else if ("".equals(sformat)) { 453 // No format specified. 454 } else { 455 throw new ItsException("Unsupported format: " + sformat); 456 } 457 } 458 459 Log.i(PYTAG, String.format("### SIZE %d %d", width, height)); 460 461 prepareCaptureReader(width, height, format); 462 List<Surface> outputSurfaces = new ArrayList<Surface>(1); 463 outputSurfaces.add(mCaptureReader.getSurface()); 464 mCamera.configureOutputs(outputSurfaces); 465 466 ImageReader.OnImageAvailableListener readerListener = 467 createAvailableListener(mCaptureListener); 468 mCaptureReader.setImageAvailableListener(readerListener, mSaveHandler); 469 470 // Plan for how many callbacks need to be received throughout the duration of this 471 // sequence of capture requests. 472 int numCaptures = requests.size(); 473 mCaptureCallbackLatch = new CountDownLatch( 474 numCaptures * ItsUtils.getCallbacksPerCapture(format)); 475 476 } catch (CameraAccessException e) { 477 throw new ItsException("Error configuring outputs", e); 478 } 479 480 // Initiate the captures. 481 for (int i = 0; i < requests.size(); i++) { 482 CaptureRequest.Builder req = requests.get(i); 483 Log.i(PYTAG, String.format("### CAPT %d of %d", i+1, requests.size())); 484 req.addTarget(mCaptureReader.getSurface()); 485 mCamera.capture(req.build(), mCaptureResultListener, mResultHandler); 486 } 487 488 // Make sure all callbacks have been hit (wait until captures are done). 489 try { 490 if (!mCaptureCallbackLatch.await(TIMEOUT_CAPTURE, TimeUnit.SECONDS)) { 491 throw new ItsException( 492 "Timeout hit, but all callbacks not received"); 493 } 494 } catch (InterruptedException e) { 495 throw new ItsException("Interrupted: ", e); 496 } 497 498 } catch (android.hardware.camera2.CameraAccessException e) { 499 throw new ItsException("Access error: ", e); 500 } 501 } 502 503 private final CaptureListener mCaptureListener = new CaptureListener() { 504 @Override 505 public void onCaptureAvailable(Image capture) { 506 try { 507 int format = capture.getFormat(); 508 String extFileName = null; 509 if (format == ImageFormat.JPEG) { 510 String fileName = ItsUtils.getJpegFileName(capture.getTimestamp()); 511 ByteBuffer buf = capture.getPlanes()[0].getBuffer(); 512 extFileName = ItsUtils.writeImageToFile(ItsService.this, buf, fileName); 513 } else if (format == ImageFormat.YUV_420_888) { 514 String fileName = ItsUtils.getYuvFileName(capture.getTimestamp()); 515 byte[] img = ItsUtils.getDataFromImage(capture); 516 ByteBuffer buf = ByteBuffer.wrap(img); 517 extFileName = ItsUtils.writeImageToFile(ItsService.this, buf, fileName); 518 } else { 519 throw new ItsException("Unsupported image format: " + format); 520 } 521 Log.i(PYTAG, String.format("### FILE %s", extFileName)); 522 mCaptureCallbackLatch.countDown(); 523 } catch (ItsException e) { 524 Log.e(TAG, "Script error: " + e); 525 Log.e(PYTAG, "### FAIL"); 526 } 527 } 528 }; 529 530 private final CaptureResultListener mCaptureResultListener = new CaptureResultListener() { 531 @Override 532 public void onCaptureStarted(CameraDevice camera, CaptureRequest request, long timestamp) { 533 } 534 535 @Override 536 public void onCaptureCompleted(CameraDevice camera, CaptureRequest request, 537 CaptureResult result) { 538 try { 539 // Currently result has all 0 values. 540 if (request == null || result == null) { 541 throw new ItsException("Request/result is invalid"); 542 } 543 544 Log.i(TAG, String.format( 545 "Capture result: AE=%d, AF=%d, AWB=%d, sens=%d, exp=%dms, dur=%dms", 546 result.get(CaptureResult.CONTROL_AE_STATE), 547 result.get(CaptureResult.CONTROL_AF_STATE), 548 result.get(CaptureResult.CONTROL_AWB_STATE), 549 result.get(CaptureResult.SENSOR_SENSITIVITY), 550 result.get(CaptureResult.SENSOR_EXPOSURE_TIME).intValue(), 551 result.get(CaptureResult.SENSOR_FRAME_DURATION).intValue())); 552 553 mConvergedAE = result.get(CaptureResult.CONTROL_AE_STATE) == 554 CaptureResult.CONTROL_AE_STATE_CONVERGED; 555 mConvergedAF = result.get(CaptureResult.CONTROL_AF_STATE) == 556 CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED; 557 mConvergedAWB = result.get(CaptureResult.CONTROL_AWB_STATE) == 558 CaptureResult.CONTROL_AWB_STATE_CONVERGED; 559 560 if (mIssuedRequest3A) { 561 mIssuedRequest3A = false; 562 mInterlock3A.open(); 563 } else { 564 String fileName = ItsUtils.getMetadataFileName( 565 result.get(CaptureResult.SENSOR_TIMESTAMP)); 566 File mdFile = ItsUtils.getOutputFile(ItsService.this, fileName); 567 ItsUtils.storeResults(mCameraProperties, request, result, mdFile); 568 mCaptureCallbackLatch.countDown(); 569 } 570 } catch (ItsException e) { 571 Log.e(TAG, "Script error: " + e); 572 Log.e(PYTAG, "### FAIL"); 573 } catch (Exception e) { 574 Log.e(TAG, "Script error: " + e); 575 Log.e(PYTAG, "### FAIL"); 576 } 577 } 578 579 @Override 580 public void onCaptureFailed(CameraDevice camera, CaptureRequest request) { 581 mCaptureCallbackLatch.countDown(); 582 Log.e(TAG, "Script error: capture failed"); 583 Log.e(PYTAG, "### FAIL"); 584 } 585 }; 586 587} 588