AndroidCamera2AgentImpl.java revision f9feab9a826e5b33d811e757bdfdbfa0738fcfa5 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.ex.camera2.portability;
18
19import android.annotation.TargetApi;
20import android.content.Context;
21import android.graphics.ImageFormat;
22import android.graphics.Matrix;
23import android.graphics.Rect;
24import android.graphics.RectF;
25import android.graphics.SurfaceTexture;
26import android.hardware.camera2.CameraAccessException;
27import android.hardware.camera2.CameraCaptureSession;
28import android.hardware.camera2.CameraCharacteristics;
29import android.hardware.camera2.CameraDevice;
30import android.hardware.camera2.CameraManager;
31import android.hardware.camera2.CaptureFailure;
32import android.hardware.camera2.CaptureRequest;
33import android.hardware.camera2.CaptureResult;
34import android.hardware.camera2.TotalCaptureResult;
35import android.hardware.camera2.params.MeteringRectangle;
36import android.media.Image;
37import android.media.ImageReader;
38import android.os.Build;
39import android.os.Handler;
40import android.os.HandlerThread;
41import android.os.Looper;
42import android.os.Message;
43import android.view.Surface;
44
45import com.android.ex.camera2.portability.debug.Log;
46import com.android.ex.camera2.utils.Camera2RequestSettingsSet;
47
48import java.nio.ByteBuffer;
49import java.util.ArrayList;
50import java.util.Arrays;
51import java.util.HashSet;
52import java.util.List;
53import java.util.Set;
54
55/**
56 * A class to implement {@link CameraAgent} of the Android camera2 framework.
57 */
58class AndroidCamera2AgentImpl extends CameraAgent {
59    private static final Log.Tag TAG = new Log.Tag("AndCam2AgntImp");
60
61    private final Camera2Handler mCameraHandler;
62    private final HandlerThread mCameraHandlerThread;
63    private final CameraStateHolder mCameraState;
64    private final DispatchThread mDispatchThread;
65    private final CameraManager mCameraManager;
66
67    /**
68     * Number of camera devices.  The length of {@code mCameraDevices} does not reveal this
69     * information because that list may contain since-invalidated indices.
70     */
71    private int mNumCameraDevices;
72
73    /**
74     * Transformation between integral camera indices and the {@link java.lang.String} indices used
75     * by the underlying API.  Note that devices may disappear because they've been disconnected or
76     * have otherwise gone offline.  Because we need to keep the meanings of whatever indices we
77     * expose stable, we cannot simply remove them in such a case; instead, we insert {@code null}s
78     * to invalidate any such indices.  Whenever new devices appear, they are appended to the end of
79     * the list, and thereby assigned the lowest index that has never yet been used.
80     */
81    private final List<String> mCameraDevices;
82
83    AndroidCamera2AgentImpl(Context context) {
84        mCameraHandlerThread = new HandlerThread("Camera2 Handler Thread");
85        mCameraHandlerThread.start();
86        mCameraHandler = new Camera2Handler(mCameraHandlerThread.getLooper());
87        mCameraState = new AndroidCamera2StateHolder();
88        mDispatchThread = new DispatchThread(mCameraHandler, mCameraHandlerThread);
89        mDispatchThread.start();
90        mCameraManager = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE);
91
92        mNumCameraDevices = 0;
93        mCameraDevices = new ArrayList<String>();
94        updateCameraDevices();
95    }
96
97    /**
98     * Updates the camera device index assignments stored in {@link mCameraDevices}, without
99     * reappropriating any currently-assigned index.
100     * @return Whether the operation was successful
101     */
102    private boolean updateCameraDevices() {
103        try {
104            String[] currentCameraDevices = mCameraManager.getCameraIdList();
105            Set<String> currentSet = new HashSet<String>(Arrays.asList(currentCameraDevices));
106
107            // Invalidate the indices assigned to any camera devices that are no longer present
108            for (int index = 0; index < mCameraDevices.size(); ++index) {
109                if (!currentSet.contains(mCameraDevices.get(index))) {
110                    mCameraDevices.set(index, null);
111                    --mNumCameraDevices;
112                }
113            }
114
115            // Assign fresh indices to any new camera devices
116            currentSet.removeAll(mCameraDevices); // The devices we didn't know about
117            for (String device : currentCameraDevices) {
118                if (currentSet.contains(device)) {
119                    mCameraDevices.add(device);
120                    ++mNumCameraDevices;
121                }
122            }
123
124            return true;
125        } catch (CameraAccessException ex) {
126            Log.e(TAG, "Could not get device listing from camera subsystem", ex);
127            return false;
128        }
129    }
130
131    // TODO: Implement
132    @Override
133    public void setCameraDefaultExceptionCallback(CameraExceptionCallback callback,
134            Handler handler) {}
135
136    // TODO: Implement
137    @Override
138    public void recycle() {}
139
140    // TODO: Some indices may now be invalid; ensure everyone can handle that and update the docs
141    @Override
142    public CameraDeviceInfo getCameraDeviceInfo() {
143        updateCameraDevices();
144        return new AndroidCamera2DeviceInfo(mCameraManager, mCameraDevices.toArray(new String[0]),
145                mNumCameraDevices);
146    }
147
148    @Override
149    protected Handler getCameraHandler() {
150        return mCameraHandler;
151    }
152
153    @Override
154    protected DispatchThread getDispatchThread() {
155        return mDispatchThread;
156    }
157
158    private static abstract class CaptureAvailableListener
159            extends CameraCaptureSession.CaptureListener
160            implements ImageReader.OnImageAvailableListener {};
161
162    private class Camera2Handler extends HistoryHandler {
163        // Caller-provided when leaving CAMERA_UNOPENED state:
164        private CameraOpenCallback mOpenCallback;
165        private int mCameraIndex;
166        private String mCameraId;
167
168        // Available in CAMERA_UNCONFIGURED state and above:
169        private CameraDevice mCamera;
170        private AndroidCamera2ProxyImpl mCameraProxy;
171        private Camera2RequestSettingsSet mPersistentSettings;
172        private Rect mActiveArray;
173        private boolean mLegacyDevice;
174
175        // Available in CAMERA_CONFIGURED state and above:
176        private Size mPreviewSize;
177        private Size mPhotoSize;
178
179        // Available in PREVIEW_READY state and above:
180        private SurfaceTexture mPreviewTexture;
181        private Surface mPreviewSurface;
182        private CameraCaptureSession mSession;
183        private ImageReader mCaptureReader;
184
185        // Available from the beginning of PREVIEW_ACTIVE until the first preview frame arrives:
186        private CameraStartPreviewCallback mOneshotPreviewingCallback;
187
188        // Available in FOCUS_LOCKED between AF trigger receipt and whenever the lens stops moving:
189        private CameraAFCallback mOneshotAfCallback;
190
191        // Available when taking picture between AE trigger receipt and autoexposure convergence
192        private CaptureAvailableListener mOneshotCaptureCallback;
193
194        // Available whenever setAutoFocusMoveCallback() was last invoked with a non-null argument:
195        private CameraAFMoveCallback mPassiveAfCallback;
196
197        Camera2Handler(Looper looper) {
198            super(looper);
199        }
200
201        @Override
202        public void handleMessage(final Message msg) {
203            super.handleMessage(msg);
204            try {
205                switch(msg.what) {
206                    case CameraActions.OPEN_CAMERA:
207                    case CameraActions.RECONNECT: {
208                        CameraOpenCallback openCallback = (CameraOpenCallback) msg.obj;
209                        int cameraIndex = msg.arg1;
210
211                        if (mCameraState.getState() != AndroidCamera2StateHolder.CAMERA_UNOPENED) {
212                            openCallback.onDeviceOpenedAlready(cameraIndex,
213                                    generateHistoryString(cameraIndex));
214                            break;
215                        }
216
217                        mOpenCallback = openCallback;
218                        mCameraIndex = cameraIndex;
219                        mCameraId = mCameraDevices.get(mCameraIndex);
220
221                        if (mCameraId == null) {
222                            mOpenCallback.onCameraDisabled(msg.arg1);
223                            break;
224                        }
225                        mCameraManager.openCamera(mCameraId, mCameraDeviceStateListener, this);
226
227                        break;
228                    }
229
230                    case CameraActions.RELEASE: {
231                        if (mCameraState.getState() == AndroidCamera2StateHolder.CAMERA_UNOPENED) {
232                            Log.w(TAG, "Ignoring release at inappropriate time");
233                            break;
234                        }
235
236                        if (mSession != null) {
237                            closePreviewSession();
238                            mSession = null;
239                        }
240                        if (mCamera != null) {
241                            mCamera.close();
242                            mCamera = null;
243                        }
244                        mCameraProxy = null;
245                        mPersistentSettings = null;
246                        mActiveArray = null;
247                        if (mPreviewSurface != null) {
248                            mPreviewSurface.release();
249                            mPreviewSurface = null;
250                        }
251                        mPreviewTexture = null;
252                        if (mCaptureReader != null) {
253                            mCaptureReader.close();
254                            mCaptureReader = null;
255                        }
256                        mPreviewSize = null;
257                        mPhotoSize = null;
258                        mCameraIndex = 0;
259                        mCameraId = null;
260                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_UNOPENED);
261                        break;
262                    }
263
264                    /*case CameraActions.UNLOCK: {
265                        break;
266                    }
267
268                    case CameraActions.LOCK: {
269                        break;
270                    }*/
271
272                    case CameraActions.SET_PREVIEW_TEXTURE_ASYNC: {
273                        setPreviewTexture((SurfaceTexture) msg.obj);
274                        break;
275                    }
276
277                    case CameraActions.START_PREVIEW_ASYNC: {
278                        if (mCameraState.getState() !=
279                                        AndroidCamera2StateHolder.CAMERA_PREVIEW_READY) {
280                            // TODO: Provide better feedback here?
281                            Log.w(TAG, "Refusing to start preview at inappropriate time");
282                            break;
283                        }
284
285                        mOneshotPreviewingCallback = (CameraStartPreviewCallback) msg.obj;
286                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE);
287                        try {
288                            mSession.setRepeatingRequest(
289                                    mPersistentSettings.createRequest(mCamera,
290                                            CameraDevice.TEMPLATE_PREVIEW, mPreviewSurface),
291                                    /*listener*/mCameraFocusStateListener, /*handler*/this);
292                        } catch(CameraAccessException ex) {
293                            Log.w(TAG, "Unable to start preview", ex);
294                            mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_READY);
295                        }
296                        break;
297                    }
298
299                    case CameraActions.STOP_PREVIEW: {
300                        if (mCameraState.getState() <
301                                        AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
302                            Log.w(TAG, "Refusing to stop preview at inappropriate time");
303                            break;
304                        }
305
306                        mSession.stopRepeating();
307                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_READY);
308                        break;
309                    }
310
311                    /*case CameraActions.SET_PREVIEW_CALLBACK_WITH_BUFFER: {
312                        break;
313                    }
314
315                    case CameraActions.ADD_CALLBACK_BUFFER: {
316                        break;
317                    }
318
319                    case CameraActions.SET_PREVIEW_DISPLAY_ASYNC: {
320                        break;
321                    }
322
323                    case CameraActions.SET_PREVIEW_CALLBACK: {
324                        break;
325                    }
326
327                    case CameraActions.SET_ONE_SHOT_PREVIEW_CALLBACK: {
328                        break;
329                    }
330
331                    case CameraActions.SET_PARAMETERS: {
332                        break;
333                    }
334
335                    case CameraActions.GET_PARAMETERS: {
336                        break;
337                    }
338
339                    case CameraActions.REFRESH_PARAMETERS: {
340                        break;
341                    }*/
342
343                    case CameraActions.APPLY_SETTINGS: {
344                        AndroidCamera2Settings settings = (AndroidCamera2Settings) msg.obj;
345                        applyToRequest(settings);
346                        break;
347                    }
348
349                    case CameraActions.AUTO_FOCUS: {
350                        // We only support locking the focus while a preview is being displayed.
351                        // However, it can be requested multiple times in succession; the effect of
352                        // the subsequent invocations is determined by the focus mode defined in the
353                        // provided CameraSettings object. In passive (CONTINUOUS_*) mode, the
354                        // duplicate requests are no-ops and leave the lens locked at its current
355                        // position, but in active (AUTO) mode, they perform another scan and lock
356                        // once that is finished. In any manual focus mode, this call is a no-op,
357                        // and most notably, this is the only case where the callback isn't invoked.
358                        if (mCameraState.getState() <
359                                        AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
360                            Log.w(TAG, "Ignoring attempt to autofocus without preview");
361                            break;
362                        }
363
364                        // The earliest we can reliably tell whether the autofocus has locked in
365                        // response to our latest request is when our one-time capture completes.
366                        // However, it will probably take longer than that, so once that happens,
367                        // just start checking the repeating preview requests as they complete.
368                        final CameraAFCallback callback = (CameraAFCallback) msg.obj;
369                        CameraCaptureSession.CaptureListener deferredCallbackSetter =
370                                new CameraCaptureSession.CaptureListener() {
371                            @Override
372                            public void onCaptureCompleted(CameraCaptureSession session,
373                                                           CaptureRequest request,
374                                                           TotalCaptureResult result) {
375                                // Now our mCameraFocusStateListener will invoke the callback the
376                                // first time it finds the focus motor to be locked.
377                                mOneshotAfCallback = callback;
378                            }
379
380                            @Override
381                            public void onCaptureFailed(CameraCaptureSession session,
382                                                        CaptureRequest request,
383                                                        CaptureFailure failure) {
384                                Log.e(TAG, "Focusing failed with reason " + failure.getReason());
385                                callback.onAutoFocus(false, mCameraProxy);
386                            }};
387
388                        // Send a one-time capture to trigger the camera driver to lock focus.
389                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED);
390                        Camera2RequestSettingsSet trigger =
391                                new Camera2RequestSettingsSet(mPersistentSettings);
392                        trigger.set(CaptureRequest.CONTROL_AF_TRIGGER,
393                                CaptureRequest.CONTROL_AF_TRIGGER_START);
394                        try {
395                            mSession.capture(
396                                    trigger.createRequest(mCamera, CameraDevice.TEMPLATE_PREVIEW,
397                                            mPreviewSurface),
398                                    /*listener*/deferredCallbackSetter, /*handler*/ this);
399                        } catch(CameraAccessException ex) {
400                            Log.e(TAG, "Unable to lock autofocus", ex);
401                            mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE);
402                        }
403                        break;
404                    }
405
406                    case CameraActions.CANCEL_AUTO_FOCUS: {
407                        // Why would you want to unlock the lens if it isn't already locked?
408                        if (mCameraState.getState() <
409                                AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
410                            Log.w(TAG, "Ignoring attempt to release focus lock without preview");
411                            break;
412                        }
413
414                        // Send a one-time capture to trigger the camera driver to resume scanning.
415                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE);
416                        Camera2RequestSettingsSet cancel =
417                                new Camera2RequestSettingsSet(mPersistentSettings);
418                        cancel.set(CaptureRequest.CONTROL_AF_TRIGGER,
419                                CaptureRequest.CONTROL_AF_TRIGGER_CANCEL);
420                        try {
421                            mSession.capture(
422                                    cancel.createRequest(mCamera, CameraDevice.TEMPLATE_PREVIEW,
423                                            mPreviewSurface),
424                                    /*listener*/null, /*handler*/this);
425                        } catch(CameraAccessException ex) {
426                            Log.e(TAG, "Unable to cancel autofocus", ex);
427                            mCameraState.setState(
428                                    AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED);
429                        }
430                        break;
431                    }
432
433                    case CameraActions.SET_AUTO_FOCUS_MOVE_CALLBACK: {
434                        mPassiveAfCallback = (CameraAFMoveCallback) msg.obj;
435                        break;
436                    }
437
438                    /*case CameraActions.SET_ZOOM_CHANGE_LISTENER: {
439                        break;
440                    }
441
442                    case CameraActions.SET_FACE_DETECTION_LISTENER: {
443                        break;
444                    }
445
446                    case CameraActions.START_FACE_DETECTION: {
447                        break;
448                    }
449
450                    case CameraActions.STOP_FACE_DETECTION: {
451                        break;
452                    }
453
454                    case CameraActions.SET_ERROR_CALLBACK: {
455                        break;
456                    }
457
458                    case CameraActions.ENABLE_SHUTTER_SOUND: {
459                        break;
460                    }*/
461
462                    case CameraActions.SET_DISPLAY_ORIENTATION: {
463                        // Only set the JPEG capture orientation if requested to do so; otherwise,
464                        // capture in the sensor's physical orientation
465                        mPersistentSettings.set(CaptureRequest.JPEG_ORIENTATION, msg.arg2 > 0 ?
466                                mCameraProxy.getCharacteristics().getJpegOrientation(msg.arg1) : 0);
467                        break;
468                    }
469
470                    case CameraActions.CAPTURE_PHOTO: {
471                        if (mCameraState.getState() <
472                                        AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
473                            Log.e(TAG, "Photos may only be taken when a preview is active");
474                            break;
475                        }
476                        if (mCameraState.getState() !=
477                                AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED) {
478                            Log.w(TAG, "Taking a (likely blurry) photo without the lens locked");
479                        }
480
481                        final CaptureAvailableListener listener =
482                                (CaptureAvailableListener) msg.obj;
483                        if (mLegacyDevice) {
484                            // Just snap the shot
485                            mCaptureReader.setOnImageAvailableListener(listener, /*handler*/this);
486                            try {
487                                mSession.capture(
488                                        mPersistentSettings.createRequest(mCamera,
489                                                CameraDevice.TEMPLATE_STILL_CAPTURE,
490                                                mCaptureReader.getSurface()),
491                                        listener, /*handler*/this);
492                            } catch (CameraAccessException ex) {
493                                Log.e(TAG, "Unable to initiate legacy capture", ex);
494                            }
495                        } else {
496                            // Not a legacy device, so we need to let AE converge before capturing
497                            CameraCaptureSession.CaptureListener deferredCallbackSetter =
498                                    new CameraCaptureSession.CaptureListener() {
499                                @Override
500                                public void onCaptureCompleted(CameraCaptureSession session,
501                                                               CaptureRequest request,
502                                                               TotalCaptureResult result) {
503                                    mOneshotCaptureCallback = listener;
504                                }
505
506                                @Override
507                                public void onCaptureFailed(CameraCaptureSession session,
508                                                            CaptureRequest request,
509                                                            CaptureFailure failure) {
510                                    Log.e(TAG, "Autoexposure and capture failed with reason " +
511                                            failure.getReason());
512                                    // TODO: Make an error callback?
513                                }};
514
515                            // Set a one-time capture to trigger the camera driver's autoexposure:
516                            Camera2RequestSettingsSet expose =
517                                    new Camera2RequestSettingsSet(mPersistentSettings);
518                            expose.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER,
519                                    CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_START);
520                            try {
521                                mSession.capture(
522                                        expose.createRequest(mCamera, CameraDevice.TEMPLATE_PREVIEW,
523                                                mPreviewSurface),
524                                        /*listener*/deferredCallbackSetter, /*handler*/this);
525                            } catch (CameraAccessException ex) {
526                                Log.e(TAG, "Unable to run autoexposure and perform capture", ex);
527                            }
528                        }
529                        break;
530                    }
531
532                    default: {
533                        // TODO: Rephrase once everything has been implemented
534                        throw new RuntimeException("Unimplemented CameraProxy message=" + msg.what);
535                    }
536                }
537            } catch (final Exception ex) {
538                if (msg.what != CameraActions.RELEASE && mCamera != null) {
539                    // TODO: Handle this better
540                    mCamera.close();
541                    mCamera = null;
542                } else if (mCamera == null) {
543                    if (msg.what == CameraActions.OPEN_CAMERA) {
544                        if (mOpenCallback != null) {
545                            mOpenCallback.onDeviceOpenFailure(mCameraIndex,
546                                    generateHistoryString(mCameraIndex));
547                        }
548                    } else {
549                        Log.w(TAG, "Cannot handle message " + msg.what + ", mCamera is null");
550                    }
551                    return;
552                }
553
554                if (ex instanceof RuntimeException) {
555                    post(new Runnable() {
556                        @Override
557                        public void run() {
558                            sCameraExceptionCallback.onCameraException((RuntimeException) ex);
559                        }});
560                }
561            }
562        }
563
564        public CameraSettings buildSettings(AndroidCamera2Capabilities caps) {
565            try {
566                return new AndroidCamera2Settings(mCamera, CameraDevice.TEMPLATE_PREVIEW,
567                        mActiveArray, mPreviewSize, mPhotoSize);
568            } catch (CameraAccessException ex) {
569                Log.e(TAG, "Unable to query camera device to build settings representation");
570                return null;
571            }
572        }
573
574        /**
575         * Simply propagates settings from provided {@link CameraSettings}
576         * object to our {@link CaptureRequest.Builder} for use in captures.
577         * <p>Most conversions to match the API 2 formats are performed by
578         * {@link AndroidCamera2Capabilities.IntegralStringifier}; otherwise
579         * any final adjustments are done here before updating the builder.</p>
580         *
581         * @param settings The new/updated settings
582         */
583        private void applyToRequest(AndroidCamera2Settings settings) {
584            // TODO: If invoked when in PREVIEW_READY state, a new preview size will not take effect
585
586            mPersistentSettings.union(settings.getRequestSettings());
587            mPreviewSize = settings.getCurrentPreviewSize();
588            mPhotoSize = settings.getCurrentPhotoSize();
589
590            if (mCameraState.getState() >= AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
591                // If we're already previewing, reflect most settings immediately
592                try {
593                    mSession.setRepeatingRequest(
594                            mPersistentSettings.createRequest(mCamera,
595                                    CameraDevice.TEMPLATE_PREVIEW, mPreviewSurface),
596                            /*listener*/mCameraFocusStateListener, /*handler*/this);
597                } catch (CameraAccessException ex) {
598                    Log.e(TAG, "Failed to apply updated request settings", ex);
599                }
600            } else if (mCameraState.getState() < AndroidCamera2StateHolder.CAMERA_PREVIEW_READY) {
601                // If we're already ready to preview, this doesn't regress our state
602                mCameraState.setState(AndroidCamera2StateHolder.CAMERA_CONFIGURED);
603            }
604        }
605
606        private void setPreviewTexture(SurfaceTexture surfaceTexture) {
607            // TODO: Must be called after providing a .*Settings populated with sizes
608            // TODO: We don't technically offer a selection of sizes tailored to SurfaceTextures!
609
610            // TODO: Handle this error condition with a callback or exception
611            if (mCameraState.getState() < AndroidCamera2StateHolder.CAMERA_CONFIGURED) {
612                Log.w(TAG, "Ignoring texture setting at inappropriate time");
613                return;
614            }
615
616            // Avoid initializing another capture session unless we absolutely have to
617            if (surfaceTexture == mPreviewTexture) {
618                Log.i(TAG, "Optimizing out redundant preview texture setting");
619                return;
620            }
621
622            if (mSession != null) {
623                closePreviewSession();
624            }
625
626            mPreviewTexture = surfaceTexture;
627            surfaceTexture.setDefaultBufferSize(mPreviewSize.width(), mPreviewSize.height());
628
629            if (mPreviewSurface != null) {
630                mPreviewSurface.release();
631            }
632            mPreviewSurface = new Surface(surfaceTexture);
633
634            if (mCaptureReader != null) {
635                mCaptureReader.close();
636            }
637            mCaptureReader = ImageReader.newInstance(
638                    mPhotoSize.width(), mPhotoSize.height(), ImageFormat.JPEG, 1);
639
640            try {
641                mCamera.createCaptureSession(
642                        Arrays.asList(mPreviewSurface, mCaptureReader.getSurface()),
643                        mCameraPreviewStateListener, this);
644            } catch (CameraAccessException ex) {
645                Log.e(TAG, "Failed to create camera capture session", ex);
646            }
647        }
648
649        private void closePreviewSession() {
650            try {
651                mSession.abortCaptures();
652                mSession = null;
653            } catch (CameraAccessException ex) {
654                Log.e(TAG, "Failed to close existing camera capture session", ex);
655            }
656            mCameraState.setState(AndroidCamera2StateHolder.CAMERA_CONFIGURED);
657        }
658
659        // This listener monitors our connection to and disconnection from camera devices.
660        private CameraDevice.StateListener mCameraDeviceStateListener =
661                new CameraDevice.StateListener() {
662            @Override
663            public void onOpened(CameraDevice camera) {
664                mCamera = camera;
665                if (mOpenCallback != null) {
666                    try {
667                        CameraCharacteristics props =
668                                mCameraManager.getCameraCharacteristics(mCameraId);
669                        mCameraProxy = new AndroidCamera2ProxyImpl(mCameraIndex, mCamera,
670                                    getCameraDeviceInfo().getCharacteristics(mCameraIndex), props);
671                        mPersistentSettings = new Camera2RequestSettingsSet();
672                        mActiveArray =
673                                props.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE);
674                        mLegacyDevice =
675                                props.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL) ==
676                                        CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY;
677                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_UNCONFIGURED);
678                        mOpenCallback.onCameraOpened(mCameraProxy);
679                    } catch (CameraAccessException ex) {
680                        mOpenCallback.onDeviceOpenFailure(mCameraIndex,
681                                generateHistoryString(mCameraIndex));
682                    }
683                }
684            }
685
686            @Override
687            public void onDisconnected(CameraDevice camera) {
688                Log.w(TAG, "Camera device '" + mCameraIndex + "' was disconnected");
689            }
690
691            @Override
692            public void onError(CameraDevice camera, int error) {
693                Log.e(TAG, "Camera device '" + mCameraIndex + "' encountered error code '" +
694                        error + '\'');
695                if (mOpenCallback != null) {
696                    mOpenCallback.onDeviceOpenFailure(mCameraIndex,
697                            generateHistoryString(mCameraIndex));
698                }
699            }};
700
701        // This listener monitors our camera session (i.e. our transition into and out of preview).
702        private CameraCaptureSession.StateListener mCameraPreviewStateListener =
703                new CameraCaptureSession.StateListener() {
704            @Override
705            public void onConfigured(CameraCaptureSession session) {
706                mSession = session;
707                mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_READY);
708            }
709
710            @Override
711            public void onConfigureFailed(CameraCaptureSession session) {
712                // TODO: Invoke a callback
713                Log.e(TAG, "Failed to configure the camera for capture");
714            }
715
716            @Override
717            public void onActive(CameraCaptureSession session) {
718                if (mOneshotPreviewingCallback != null) {
719                    // The session is up and processing preview requests. Inform the caller.
720                    mOneshotPreviewingCallback.onPreviewStarted();
721                    mOneshotPreviewingCallback = null;
722                }
723            }};
724
725        // This listener monitors requested captures and notifies any relevant callbacks.
726        private CameraCaptureSession.CaptureListener mCameraFocusStateListener =
727                new CameraCaptureSession.CaptureListener() {
728            private int mLastAfState = -1;
729
730            @Override
731            public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request,
732                                           TotalCaptureResult result) {
733                Integer afStateMaybe = result.get(CaptureResult.CONTROL_AF_STATE);
734                if (afStateMaybe != null) {
735                    int afState = afStateMaybe;
736                    boolean afStateChanged = false;
737                    if (afState != mLastAfState) {
738                        mLastAfState = afState;
739                        afStateChanged = true;
740                    }
741
742                    switch (afState) {
743                        case CaptureResult.CONTROL_AF_STATE_PASSIVE_SCAN:
744                        case CaptureResult.CONTROL_AF_STATE_PASSIVE_FOCUSED:
745                        case CaptureResult.CONTROL_AF_STATE_PASSIVE_UNFOCUSED: {
746                            if (afStateChanged && mPassiveAfCallback != null) {
747                                // A CameraAFMoveCallback is attached. If we just started to scan,
748                                // the motor is moving; otherwise, it has settled.
749                                mPassiveAfCallback.onAutoFocusMoving(
750                                        afState == CaptureResult.CONTROL_AF_STATE_PASSIVE_SCAN,
751                                        mCameraProxy);
752                            }
753                            break;
754                        }
755
756                        case CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED:
757                        case CaptureResult.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED: {
758                            if (mOneshotAfCallback != null) {
759                                // A call to autoFocus() was just made to request a focus lock.
760                                // Notify the caller that the lens is now indefinitely fixed, and
761                                // report whether the image we're now stuck with is in focus.
762                                mOneshotAfCallback.onAutoFocus(
763                                        afState == CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED,
764                                        mCameraProxy);
765                                mOneshotAfCallback = null;
766                            }
767                            break;
768                        }
769                    }
770                }
771
772                Integer aeStateMaybe = result.get(CaptureResult.CONTROL_AE_STATE);
773                if (aeStateMaybe != null) {
774                    int aeState = aeStateMaybe;
775
776                    switch (aeState) {
777                        case CaptureResult.CONTROL_AE_STATE_CONVERGED:
778                        case CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED:
779                        case CaptureResult.CONTROL_AE_STATE_LOCKED: {
780                            if (mOneshotCaptureCallback != null) {
781                                // A call to takePicture() was just made, and autoexposure converged
782                                // so it's time to initiate the capture!
783                                mCaptureReader.setOnImageAvailableListener(mOneshotCaptureCallback,
784                                        /*handler*/Camera2Handler.this);
785                                try {
786                                    mSession.capture(
787                                            mPersistentSettings.createRequest(mCamera,
788                                                    CameraDevice.TEMPLATE_STILL_CAPTURE,
789                                                    mCaptureReader.getSurface()),
790                                            /*listener*/mOneshotCaptureCallback,
791                                            /*handler*/Camera2Handler.this);
792                                } catch (CameraAccessException ex) {
793                                    Log.e(TAG, "Unable to initiate capture", ex);
794                                } finally {
795                                    mOneshotCaptureCallback = null;
796                                }
797                            }
798                            break;
799                        }
800                    }
801                }
802            }
803
804            @Override
805            public void onCaptureFailed(CameraCaptureSession session, CaptureRequest request,
806                                        CaptureFailure failure) {
807                Log.e(TAG, "Capture attempt failed with reason " + failure.getReason());
808            }};
809    }
810
811    private class AndroidCamera2ProxyImpl extends CameraAgent.CameraProxy {
812        private final int mCameraIndex;
813        private final CameraDevice mCamera;
814        private final CameraDeviceInfo.Characteristics mCharacteristics;
815        private final AndroidCamera2Capabilities mCapabilities;
816
817        public AndroidCamera2ProxyImpl(int cameraIndex, CameraDevice camera,
818                CameraDeviceInfo.Characteristics characteristics,
819                CameraCharacteristics properties) {
820            mCameraIndex = cameraIndex;
821            mCamera = camera;
822            mCharacteristics = characteristics;
823            mCapabilities = new AndroidCamera2Capabilities(properties);
824        }
825
826        // TODO: Implement
827        @Override
828        public android.hardware.Camera getCamera() { return null; }
829
830        @Override
831        public int getCameraId() {
832            return mCameraIndex;
833        }
834
835        @Override
836        public CameraDeviceInfo.Characteristics getCharacteristics() {
837            return mCharacteristics;
838        }
839
840        @Override
841        public CameraCapabilities getCapabilities() {
842            return mCapabilities;
843        }
844
845        private AndroidCamera2Capabilities getSpecializedCapabilities() {
846            return mCapabilities;
847        }
848
849        // TODO: Implement
850        @Override
851        public void setPreviewDataCallback(Handler handler, CameraPreviewDataCallback cb) {}
852
853        // TODO: Implement
854        @Override
855        public void setOneShotPreviewCallback(Handler handler, CameraPreviewDataCallback cb) {}
856
857        // TODO: Implement
858        @Override
859        public void setPreviewDataCallbackWithBuffer(Handler handler, CameraPreviewDataCallback cb)
860                {}
861
862        @Override
863        public void autoFocus(final Handler handler, final CameraAFCallback cb) {
864            mDispatchThread.runJob(new Runnable() {
865                @Override
866                public void run() {
867                    CameraAFCallback cbForward = null;
868                    if (cb != null) {
869                        cbForward = new CameraAFCallback() {
870                            @Override
871                            public void onAutoFocus(final boolean focused,
872                                                    final CameraProxy camera) {
873                                handler.post(new Runnable() {
874                                    @Override
875                                    public void run() {
876                                        cb.onAutoFocus(focused, camera);
877                                    }});
878                            }};
879                    }
880
881                    mCameraState.waitForStates(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE |
882                            AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED);
883                    mCameraHandler.obtainMessage(CameraActions.AUTO_FOCUS, cbForward)
884                            .sendToTarget();
885                }});
886        }
887
888        @TargetApi(Build.VERSION_CODES.JELLY_BEAN)
889        @Override
890        public void setAutoFocusMoveCallback(final Handler handler, final CameraAFMoveCallback cb) {
891            mDispatchThread.runJob(new Runnable() {
892                @Override
893                public void run() {
894                    CameraAFMoveCallback cbForward = null;
895                    if (cb != null) {
896                        cbForward = new CameraAFMoveCallback() {
897                            @Override
898                            public void onAutoFocusMoving(final boolean moving,
899                                                          final CameraProxy camera) {
900                                handler.post(new Runnable() {
901                                    @Override
902                                    public void run() {
903                                        cb.onAutoFocusMoving(moving, camera);
904                                    }});
905                                }};
906                    }
907
908                    mCameraHandler.obtainMessage(CameraActions.SET_AUTO_FOCUS_MOVE_CALLBACK,
909                            cbForward).sendToTarget();
910                }});
911        }
912
913        // TODO: Implement
914        @Override
915        public void takePicture(final Handler handler,
916                                final CameraShutterCallback shutter,
917                                CameraPictureCallback raw,
918                                CameraPictureCallback postview,
919                                final CameraPictureCallback jpeg) {
920            // TODO: We never call raw or postview
921            final CaptureAvailableListener picListener =
922                    new CaptureAvailableListener() {
923                @Override
924                public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request,
925                                             long timestamp) {
926                    if (shutter != null) {
927                        handler.post(new Runnable() {
928                            @Override
929                            public void run() {
930                                shutter.onShutter(AndroidCamera2ProxyImpl.this);
931                            }});
932                    }
933                }
934
935                @Override
936                public void onImageAvailable(ImageReader reader) {
937                    try (Image image = reader.acquireNextImage()) {
938                        if (jpeg != null) {
939                            ByteBuffer buffer = image.getPlanes()[0].getBuffer();
940                            final byte[] pixels = new byte[buffer.remaining()];
941                            buffer.get(pixels);
942                            handler.post(new Runnable() {
943                                @Override
944                                public void run() {
945                                    jpeg.onPictureTaken(pixels, AndroidCamera2ProxyImpl.this);
946                                }});
947                        }
948                    }
949                }};
950            mDispatchThread.runJob(new Runnable() {
951                @Override
952                public void run() {
953                    mCameraState.waitForStates(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE |
954                            AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED);
955                    mCameraHandler.obtainMessage(CameraActions.CAPTURE_PHOTO, picListener)
956                            .sendToTarget();
957                }});
958        }
959
960        // TODO: Implement
961        @Override
962        public void setZoomChangeListener(android.hardware.Camera.OnZoomChangeListener listener) {}
963
964        // TODO: Implement
965        @Override
966        public void setFaceDetectionCallback(Handler handler, CameraFaceDetectionCallback callback)
967                {}
968
969        // TODO: Remove this method override once we handle this message
970        @Override
971        public void startFaceDetection() {}
972
973        // TODO: Remove this method override once we handle this message
974        @Override
975        public void stopFaceDetection() {}
976
977        // TODO: Implement
978        @Override
979        public void setErrorCallback(Handler handler, CameraErrorCallback cb) {}
980
981        // TODO: Implement
982        @Override
983        public void setParameters(android.hardware.Camera.Parameters params) {}
984
985        // TODO: Implement
986        @Override
987        public android.hardware.Camera.Parameters getParameters() { return null; }
988
989        @Override
990        public CameraSettings getSettings() {
991            return mCameraHandler.buildSettings(mCapabilities);
992        }
993
994        @Override
995        public boolean applySettings(CameraSettings settings) {
996            if (settings == null) {
997                Log.w(TAG, "null parameters in applySettings()");
998                return false;
999            }
1000            if (!(settings instanceof AndroidCamera2Settings)) {
1001                Log.e(TAG, "Provided settings not compatible with the backing framework API");
1002                return false;
1003            }
1004
1005            return applySettingsHelper(settings, AndroidCamera2StateHolder.CAMERA_UNCONFIGURED |
1006                    AndroidCamera2StateHolder.CAMERA_CONFIGURED |
1007                    AndroidCamera2StateHolder.CAMERA_PREVIEW_READY);
1008        }
1009
1010        // TODO: Implement
1011        @Override
1012        public String dumpDeviceSettings() { return null; }
1013
1014        @Override
1015        public Handler getCameraHandler() {
1016            return AndroidCamera2AgentImpl.this.getCameraHandler();
1017        }
1018
1019        @Override
1020        public DispatchThread getDispatchThread() {
1021            return AndroidCamera2AgentImpl.this.getDispatchThread();
1022        }
1023
1024        @Override
1025        public CameraStateHolder getCameraState() {
1026            return mCameraState;
1027        }
1028    }
1029
1030    /** A linear state machine: each state entails all the states below it. */
1031    private static class AndroidCamera2StateHolder extends CameraStateHolder {
1032        // Usage flow: openCamera() -> applySettings() -> setPreviewTexture() -> startPreview() ->
1033        //             autoFocus() -> takePicture()
1034        /* Camera states */
1035        /** No camera device is opened. */
1036        public static final int CAMERA_UNOPENED = 1;
1037        /** A camera is opened, but no settings have been provided. */
1038        public static final int CAMERA_UNCONFIGURED = 2;
1039        /** The open camera has been configured by providing it with settings. */
1040        public static final int CAMERA_CONFIGURED = 3;
1041        /** A capture session is ready to stream a preview, but still has no repeating request. */
1042        public static final int CAMERA_PREVIEW_READY = 4;
1043        /** A preview is currently being streamed. */
1044        public static final int CAMERA_PREVIEW_ACTIVE = 5;
1045        /** The lens is locked on a particular region. */
1046        public static final int CAMERA_FOCUS_LOCKED = 6;
1047
1048        public AndroidCamera2StateHolder() {
1049            this(CAMERA_UNOPENED);
1050        }
1051
1052        public AndroidCamera2StateHolder(int state) {
1053            super(state);
1054        }
1055    }
1056
1057    private static class AndroidCamera2DeviceInfo implements CameraDeviceInfo {
1058        private final CameraManager mCameraManager;
1059        private final String[] mCameraIds;
1060        private final int mNumberOfCameras;
1061        private final int mFirstBackCameraId;
1062        private final int mFirstFrontCameraId;
1063
1064        public AndroidCamera2DeviceInfo(CameraManager cameraManager,
1065                                        String[] cameraIds, int numberOfCameras) {
1066            mCameraManager = cameraManager;
1067            mCameraIds = cameraIds;
1068            mNumberOfCameras = numberOfCameras;
1069
1070            int firstBackId = NO_DEVICE;
1071            int firstFrontId = NO_DEVICE;
1072            for (int id = 0; id < cameraIds.length; ++id) {
1073                try {
1074                    int lensDirection = cameraManager.getCameraCharacteristics(cameraIds[id])
1075                            .get(CameraCharacteristics.LENS_FACING);
1076                    if (firstBackId == NO_DEVICE &&
1077                            lensDirection == CameraCharacteristics.LENS_FACING_BACK) {
1078                        firstBackId = id;
1079                    }
1080                    if (firstFrontId == NO_DEVICE &&
1081                            lensDirection == CameraCharacteristics.LENS_FACING_FRONT) {
1082                        firstFrontId = id;
1083                    }
1084                } catch (CameraAccessException ex) {
1085                    Log.w(TAG, "Couldn't get characteristics of camera '" + id + "'", ex);
1086                }
1087            }
1088            mFirstBackCameraId = firstBackId;
1089            mFirstFrontCameraId = firstFrontId;
1090        }
1091
1092        @Override
1093        public Characteristics getCharacteristics(int cameraId) {
1094            String actualId = mCameraIds[cameraId];
1095            try {
1096                CameraCharacteristics info = mCameraManager.getCameraCharacteristics(actualId);
1097                return new AndroidCharacteristics2(info);
1098            } catch (CameraAccessException ex) {
1099                return null;
1100            }
1101        }
1102
1103        @Override
1104        public int getNumberOfCameras() {
1105            return mNumberOfCameras;
1106        }
1107
1108        @Override
1109        public int getFirstBackCameraId() {
1110            return mFirstBackCameraId;
1111        }
1112
1113        @Override
1114        public int getFirstFrontCameraId() {
1115            return mFirstFrontCameraId;
1116        }
1117
1118        private static class AndroidCharacteristics2 extends Characteristics {
1119            private CameraCharacteristics mCameraInfo;
1120
1121            AndroidCharacteristics2(CameraCharacteristics cameraInfo) {
1122                mCameraInfo = cameraInfo;
1123            }
1124
1125            @Override
1126            public boolean isFacingBack() {
1127                return mCameraInfo.get(CameraCharacteristics.LENS_FACING)
1128                        .equals(CameraCharacteristics.LENS_FACING_BACK);
1129            }
1130
1131            @Override
1132            public boolean isFacingFront() {
1133                return mCameraInfo.get(CameraCharacteristics.LENS_FACING)
1134                        .equals(CameraCharacteristics.LENS_FACING_FRONT);
1135            }
1136
1137            @Override
1138            public int getSensorOrientation() {
1139                return mCameraInfo.get(CameraCharacteristics.SENSOR_ORIENTATION);
1140            }
1141
1142            @Override
1143            public Matrix getPreviewTransform(int currentDisplayOrientation,
1144                                              RectF surfaceDimensions,
1145                                              RectF desiredBounds) {
1146                if (!orientationIsValid(currentDisplayOrientation)) {
1147                    return new Matrix();
1148                }
1149
1150                // The system transparently transforms the image to fill the surface
1151                // when the device is in its natural orientation. We rotate the
1152                // coordinates of the rectangle's corners to be relative to the
1153                // original image, instead of to the current screen orientation.
1154                float[] surfacePolygon = rotate(convertRectToPoly(surfaceDimensions),
1155                        2 * currentDisplayOrientation / 90);
1156                float[] desiredPolygon = convertRectToPoly(desiredBounds);
1157
1158                Matrix transform = new Matrix();
1159                // Use polygons instead of rectangles so that rotation will be
1160                // calculated, since that is not done by the new camera API.
1161                transform.setPolyToPoly(surfacePolygon, 0, desiredPolygon, 0, 4);
1162                return transform;
1163            }
1164
1165            @Override
1166            public boolean canDisableShutterSound() {
1167                // The new API doesn't support this operation, so don't encourage people to try it.
1168                // TODO: What kind of assumptions have callers made about this result's meaning?
1169                return false;
1170            }
1171
1172            private static float[] convertRectToPoly(RectF rf) {
1173                return new float[] {rf.left, rf.top, rf.right, rf.top,
1174                        rf.right, rf.bottom, rf.left, rf.bottom};
1175            }
1176
1177            private static float[] rotate(float[] arr, int times) {
1178                if (times < 0) {
1179                    times = times % arr.length + arr.length;
1180                }
1181
1182                float[] res = new float[arr.length];
1183                for (int offset = 0; offset < arr.length; ++offset) {
1184                    res[offset] = arr[(times + offset) % arr.length];
1185                }
1186                return res;
1187            }
1188        }
1189    }
1190
1191    private static final CameraExceptionCallback sCameraExceptionCallback =
1192            new CameraExceptionCallback() {
1193                @Override
1194                public synchronized void onCameraException(RuntimeException e) {
1195                    throw e;
1196                }
1197            };
1198}
1199