AndroidCamera2AgentImpl.java revision b2df32dd892ae8076714b7ebd33c715378c4c730 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.ex.camera2.portability;
18
19import android.annotation.TargetApi;
20import android.content.Context;
21import android.graphics.ImageFormat;
22import android.graphics.Rect;
23import android.graphics.SurfaceTexture;
24import android.hardware.camera2.CameraAccessException;
25import android.hardware.camera2.CameraCaptureSession;
26import android.hardware.camera2.CameraCharacteristics;
27import android.hardware.camera2.CameraDevice;
28import android.hardware.camera2.CameraManager;
29import android.hardware.camera2.CaptureFailure;
30import android.hardware.camera2.CaptureRequest;
31import android.hardware.camera2.CaptureResult;
32import android.hardware.camera2.TotalCaptureResult;
33import android.hardware.camera2.params.MeteringRectangle;
34import android.media.Image;
35import android.media.ImageReader;
36import android.os.Build;
37import android.os.Handler;
38import android.os.HandlerThread;
39import android.os.Looper;
40import android.os.Message;
41import android.view.Surface;
42
43import com.android.ex.camera2.portability.debug.Log;
44import com.android.ex.camera2.utils.Camera2RequestSettingsSet;
45
46import java.nio.ByteBuffer;
47import java.util.ArrayList;
48import java.util.Arrays;
49import java.util.HashSet;
50import java.util.List;
51import java.util.Set;
52
53/**
54 * A class to implement {@link CameraAgent} of the Android camera2 framework.
55 */
56class AndroidCamera2AgentImpl extends CameraAgent {
57    private static final Log.Tag TAG = new Log.Tag("AndCam2AgntImp");
58
59    private final Camera2Handler mCameraHandler;
60    private final HandlerThread mCameraHandlerThread;
61    private final CameraStateHolder mCameraState;
62    private final DispatchThread mDispatchThread;
63    private final CameraManager mCameraManager;
64
65    /**
66     * Number of camera devices.  The length of {@code mCameraDevices} does not reveal this
67     * information because that list may contain since-invalidated indices.
68     */
69    private int mNumCameraDevices;
70
71    /**
72     * Transformation between integral camera indices and the {@link java.lang.String} indices used
73     * by the underlying API.  Note that devices may disappear because they've been disconnected or
74     * have otherwise gone offline.  Because we need to keep the meanings of whatever indices we
75     * expose stable, we cannot simply remove them in such a case; instead, we insert {@code null}s
76     * to invalidate any such indices.  Whenever new devices appear, they are appended to the end of
77     * the list, and thereby assigned the lowest index that has never yet been used.
78     */
79    private final List<String> mCameraDevices;
80
81    AndroidCamera2AgentImpl(Context context) {
82        mCameraHandlerThread = new HandlerThread("Camera2 Handler Thread");
83        mCameraHandlerThread.start();
84        mCameraHandler = new Camera2Handler(mCameraHandlerThread.getLooper());
85        mCameraState = new AndroidCamera2StateHolder();
86        mDispatchThread = new DispatchThread(mCameraHandler, mCameraHandlerThread);
87        mDispatchThread.start();
88        mCameraManager = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE);
89
90        mNumCameraDevices = 0;
91        mCameraDevices = new ArrayList<String>();
92        updateCameraDevices();
93    }
94
95    /**
96     * Updates the camera device index assignments stored in {@link mCameraDevices}, without
97     * reappropriating any currently-assigned index.
98     * @return Whether the operation was successful
99     */
100    private boolean updateCameraDevices() {
101        try {
102            String[] currentCameraDevices = mCameraManager.getCameraIdList();
103            Set<String> currentSet = new HashSet<String>(Arrays.asList(currentCameraDevices));
104
105            // Invalidate the indices assigned to any camera devices that are no longer present
106            for (int index = 0; index < mCameraDevices.size(); ++index) {
107                if (!currentSet.contains(mCameraDevices.get(index))) {
108                    mCameraDevices.set(index, null);
109                    --mNumCameraDevices;
110                }
111            }
112
113            // Assign fresh indices to any new camera devices
114            currentSet.removeAll(mCameraDevices); // The devices we didn't know about
115            for (String device : currentCameraDevices) {
116                if (currentSet.contains(device)) {
117                    mCameraDevices.add(device);
118                    ++mNumCameraDevices;
119                }
120            }
121
122            return true;
123        } catch (CameraAccessException ex) {
124            Log.e(TAG, "Could not get device listing from camera subsystem", ex);
125            return false;
126        }
127    }
128
129    // TODO: Implement
130    @Override
131    public void setCameraDefaultExceptionCallback(CameraExceptionCallback callback,
132            Handler handler) {}
133
134    // TODO: Implement
135    @Override
136    public void recycle() {}
137
138    // TODO: Some indices may now be invalid; ensure everyone can handle that and update the docs
139    @Override
140    public CameraDeviceInfo getCameraDeviceInfo() {
141        updateCameraDevices();
142        return new AndroidCamera2DeviceInfo(mCameraManager, mCameraDevices.toArray(new String[0]),
143                mNumCameraDevices);
144    }
145
146    @Override
147    protected Handler getCameraHandler() {
148        return mCameraHandler;
149    }
150
151    @Override
152    protected DispatchThread getDispatchThread() {
153        return mDispatchThread;
154    }
155
156    private static abstract class CaptureAvailableListener
157            extends CameraCaptureSession.CaptureListener
158            implements ImageReader.OnImageAvailableListener {};
159
160    private class Camera2Handler extends HistoryHandler {
161        // Caller-provided when leaving CAMERA_UNOPENED state:
162        private CameraOpenCallback mOpenCallback;
163        private int mCameraIndex;
164        private String mCameraId;
165
166        // Available in CAMERA_UNCONFIGURED state and above:
167        private CameraDevice mCamera;
168        private AndroidCamera2ProxyImpl mCameraProxy;
169        private Camera2RequestSettingsSet mPersistentSettings;
170        private Rect mActiveArray;
171        private boolean mLegacyDevice;
172
173        // Available in CAMERA_CONFIGURED state and above:
174        private Size mPreviewSize;
175        private Size mPhotoSize;
176
177        // Available in PREVIEW_READY state and above:
178        private SurfaceTexture mPreviewTexture;
179        private Surface mPreviewSurface;
180        private CameraCaptureSession mSession;
181        private ImageReader mCaptureReader;
182
183        // Available from the beginning of PREVIEW_ACTIVE until the first preview frame arrives:
184        private CameraStartPreviewCallback mOneshotPreviewingCallback;
185
186        // Available in FOCUS_LOCKED between AF trigger receipt and whenever the lens stops moving:
187        private CameraAFCallback mOneshotAfCallback;
188
189        // Available when taking picture between AE trigger receipt and autoexposure convergence
190        private CaptureAvailableListener mOneshotCaptureCallback;
191
192        // Available whenever setAutoFocusMoveCallback() was last invoked with a non-null argument:
193        private CameraAFMoveCallback mPassiveAfCallback;
194
195        Camera2Handler(Looper looper) {
196            super(looper);
197        }
198
199        @Override
200        public void handleMessage(final Message msg) {
201            super.handleMessage(msg);
202            try {
203                switch(msg.what) {
204                    case CameraActions.OPEN_CAMERA:
205                    case CameraActions.RECONNECT: {
206                        CameraOpenCallback openCallback = (CameraOpenCallback) msg.obj;
207                        int cameraIndex = msg.arg1;
208
209                        if (mCameraState.getState() != AndroidCamera2StateHolder.CAMERA_UNOPENED) {
210                            openCallback.onDeviceOpenedAlready(cameraIndex,
211                                    generateHistoryString(cameraIndex));
212                            break;
213                        }
214
215                        mOpenCallback = openCallback;
216                        mCameraIndex = cameraIndex;
217                        mCameraId = mCameraDevices.get(mCameraIndex);
218                        Log.i(TAG, String.format("Opening camera index %d (id %s) with camera2 API",
219                                cameraIndex, mCameraId));
220
221                        if (mCameraId == null) {
222                            mOpenCallback.onCameraDisabled(msg.arg1);
223                            break;
224                        }
225                        mCameraManager.openCamera(mCameraId, mCameraDeviceStateListener, this);
226
227                        break;
228                    }
229
230                    case CameraActions.RELEASE: {
231                        if (mCameraState.getState() == AndroidCamera2StateHolder.CAMERA_UNOPENED) {
232                            Log.w(TAG, "Ignoring release at inappropriate time");
233                            break;
234                        }
235
236                        if (mSession != null) {
237                            closePreviewSession();
238                            mSession = null;
239                        }
240                        if (mCamera != null) {
241                            mCamera.close();
242                            mCamera = null;
243                        }
244                        mCameraProxy = null;
245                        mPersistentSettings = null;
246                        mActiveArray = null;
247                        if (mPreviewSurface != null) {
248                            mPreviewSurface.release();
249                            mPreviewSurface = null;
250                        }
251                        mPreviewTexture = null;
252                        if (mCaptureReader != null) {
253                            mCaptureReader.close();
254                            mCaptureReader = null;
255                        }
256                        mPreviewSize = null;
257                        mPhotoSize = null;
258                        mCameraIndex = 0;
259                        mCameraId = null;
260                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_UNOPENED);
261                        break;
262                    }
263
264                    /*case CameraActions.UNLOCK: {
265                        break;
266                    }
267
268                    case CameraActions.LOCK: {
269                        break;
270                    }*/
271
272                    case CameraActions.SET_PREVIEW_TEXTURE_ASYNC: {
273                        setPreviewTexture((SurfaceTexture) msg.obj);
274                        break;
275                    }
276
277                    case CameraActions.START_PREVIEW_ASYNC: {
278                        if (mCameraState.getState() !=
279                                        AndroidCamera2StateHolder.CAMERA_PREVIEW_READY) {
280                            // TODO: Provide better feedback here?
281                            Log.w(TAG, "Refusing to start preview at inappropriate time");
282                            break;
283                        }
284
285                        mOneshotPreviewingCallback = (CameraStartPreviewCallback) msg.obj;
286                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE);
287                        try {
288                            mSession.setRepeatingRequest(
289                                    mPersistentSettings.createRequest(mCamera,
290                                            CameraDevice.TEMPLATE_PREVIEW, mPreviewSurface),
291                                    /*listener*/mCameraFocusStateListener, /*handler*/this);
292                        } catch(CameraAccessException ex) {
293                            Log.w(TAG, "Unable to start preview", ex);
294                            mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_READY);
295                        }
296                        break;
297                    }
298
299                    case CameraActions.STOP_PREVIEW: {
300                        if (mCameraState.getState() <
301                                        AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
302                            Log.w(TAG, "Refusing to stop preview at inappropriate time");
303                            break;
304                        }
305
306                        mSession.stopRepeating();
307                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_READY);
308                        break;
309                    }
310
311                    /*case CameraActions.SET_PREVIEW_CALLBACK_WITH_BUFFER: {
312                        break;
313                    }
314
315                    case CameraActions.ADD_CALLBACK_BUFFER: {
316                        break;
317                    }
318
319                    case CameraActions.SET_PREVIEW_DISPLAY_ASYNC: {
320                        break;
321                    }
322
323                    case CameraActions.SET_PREVIEW_CALLBACK: {
324                        break;
325                    }
326
327                    case CameraActions.SET_ONE_SHOT_PREVIEW_CALLBACK: {
328                        break;
329                    }
330
331                    case CameraActions.SET_PARAMETERS: {
332                        break;
333                    }
334
335                    case CameraActions.GET_PARAMETERS: {
336                        break;
337                    }
338
339                    case CameraActions.REFRESH_PARAMETERS: {
340                        break;
341                    }*/
342
343                    case CameraActions.APPLY_SETTINGS: {
344                        AndroidCamera2Settings settings = (AndroidCamera2Settings) msg.obj;
345                        applyToRequest(settings);
346                        break;
347                    }
348
349                    case CameraActions.AUTO_FOCUS: {
350                        // We only support locking the focus while a preview is being displayed.
351                        // However, it can be requested multiple times in succession; the effect of
352                        // the subsequent invocations is determined by the focus mode defined in the
353                        // provided CameraSettings object. In passive (CONTINUOUS_*) mode, the
354                        // duplicate requests are no-ops and leave the lens locked at its current
355                        // position, but in active (AUTO) mode, they perform another scan and lock
356                        // once that is finished. In any manual focus mode, this call is a no-op,
357                        // and most notably, this is the only case where the callback isn't invoked.
358                        if (mCameraState.getState() <
359                                        AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
360                            Log.w(TAG, "Ignoring attempt to autofocus without preview");
361                            break;
362                        }
363
364                        // The earliest we can reliably tell whether the autofocus has locked in
365                        // response to our latest request is when our one-time capture completes.
366                        // However, it will probably take longer than that, so once that happens,
367                        // just start checking the repeating preview requests as they complete.
368                        final CameraAFCallback callback = (CameraAFCallback) msg.obj;
369                        CameraCaptureSession.CaptureListener deferredCallbackSetter =
370                                new CameraCaptureSession.CaptureListener() {
371                            @Override
372                            public void onCaptureCompleted(CameraCaptureSession session,
373                                                           CaptureRequest request,
374                                                           TotalCaptureResult result) {
375                                // Now our mCameraFocusStateListener will invoke the callback the
376                                // first time it finds the focus motor to be locked.
377                                mOneshotAfCallback = callback;
378                            }
379
380                            @Override
381                            public void onCaptureFailed(CameraCaptureSession session,
382                                                        CaptureRequest request,
383                                                        CaptureFailure failure) {
384                                Log.e(TAG, "Focusing failed with reason " + failure.getReason());
385                                callback.onAutoFocus(false, mCameraProxy);
386                            }};
387
388                        // Send a one-time capture to trigger the camera driver to lock focus.
389                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED);
390                        Camera2RequestSettingsSet trigger =
391                                new Camera2RequestSettingsSet(mPersistentSettings);
392                        trigger.set(CaptureRequest.CONTROL_AF_TRIGGER,
393                                CaptureRequest.CONTROL_AF_TRIGGER_START);
394                        try {
395                            mSession.capture(
396                                    trigger.createRequest(mCamera, CameraDevice.TEMPLATE_PREVIEW,
397                                            mPreviewSurface),
398                                    /*listener*/deferredCallbackSetter, /*handler*/ this);
399                        } catch(CameraAccessException ex) {
400                            Log.e(TAG, "Unable to lock autofocus", ex);
401                            mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE);
402                        }
403                        break;
404                    }
405
406                    case CameraActions.CANCEL_AUTO_FOCUS: {
407                        // Why would you want to unlock the lens if it isn't already locked?
408                        if (mCameraState.getState() <
409                                AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
410                            Log.w(TAG, "Ignoring attempt to release focus lock without preview");
411                            break;
412                        }
413
414                        // Send a one-time capture to trigger the camera driver to resume scanning.
415                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE);
416                        Camera2RequestSettingsSet cancel =
417                                new Camera2RequestSettingsSet(mPersistentSettings);
418                        cancel.set(CaptureRequest.CONTROL_AF_TRIGGER,
419                                CaptureRequest.CONTROL_AF_TRIGGER_CANCEL);
420                        try {
421                            mSession.capture(
422                                    cancel.createRequest(mCamera, CameraDevice.TEMPLATE_PREVIEW,
423                                            mPreviewSurface),
424                                    /*listener*/null, /*handler*/this);
425                        } catch(CameraAccessException ex) {
426                            Log.e(TAG, "Unable to cancel autofocus", ex);
427                            mCameraState.setState(
428                                    AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED);
429                        }
430                        break;
431                    }
432
433                    case CameraActions.SET_AUTO_FOCUS_MOVE_CALLBACK: {
434                        mPassiveAfCallback = (CameraAFMoveCallback) msg.obj;
435                        break;
436                    }
437
438                    /*case CameraActions.SET_ZOOM_CHANGE_LISTENER: {
439                        break;
440                    }
441
442                    case CameraActions.SET_FACE_DETECTION_LISTENER: {
443                        break;
444                    }
445
446                    case CameraActions.START_FACE_DETECTION: {
447                        break;
448                    }
449
450                    case CameraActions.STOP_FACE_DETECTION: {
451                        break;
452                    }
453
454                    case CameraActions.SET_ERROR_CALLBACK: {
455                        break;
456                    }
457
458                    case CameraActions.ENABLE_SHUTTER_SOUND: {
459                        break;
460                    }*/
461
462                    case CameraActions.SET_DISPLAY_ORIENTATION: {
463                        // TODO: Need to handle preview in addition to capture
464                        // Only set the JPEG capture orientation if requested to do so; otherwise,
465                        // capture in the sensor's physical orientation
466                        mPersistentSettings.set(CaptureRequest.JPEG_ORIENTATION, msg.arg2 > 0 ?
467                                mCameraProxy.getCharacteristics().getJpegOrientation(msg.arg1) : 0);
468                        break;
469                    }
470
471                    case CameraActions.CAPTURE_PHOTO: {
472                        if (mCameraState.getState() <
473                                        AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
474                            Log.e(TAG, "Photos may only be taken when a preview is active");
475                            break;
476                        }
477                        if (mCameraState.getState() !=
478                                AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED) {
479                            Log.w(TAG, "Taking a (likely blurry) photo without the lens locked");
480                        }
481
482                        final CaptureAvailableListener listener =
483                                (CaptureAvailableListener) msg.obj;
484                        if (mLegacyDevice) {
485                            // Just snap the shot
486                            mCaptureReader.setOnImageAvailableListener(listener, /*handler*/this);
487                            try {
488                                mSession.capture(
489                                        mPersistentSettings.createRequest(mCamera,
490                                                CameraDevice.TEMPLATE_STILL_CAPTURE,
491                                                mCaptureReader.getSurface()),
492                                        listener, /*handler*/this);
493                            } catch (CameraAccessException ex) {
494                                Log.e(TAG, "Unable to initiate legacy capture", ex);
495                            }
496                        } else {
497                            // Not a legacy device, so we need to let AE converge before capturing
498                            CameraCaptureSession.CaptureListener deferredCallbackSetter =
499                                    new CameraCaptureSession.CaptureListener() {
500                                @Override
501                                public void onCaptureCompleted(CameraCaptureSession session,
502                                                               CaptureRequest request,
503                                                               TotalCaptureResult result) {
504                                    mOneshotCaptureCallback = listener;
505                                }
506
507                                @Override
508                                public void onCaptureFailed(CameraCaptureSession session,
509                                                            CaptureRequest request,
510                                                            CaptureFailure failure) {
511                                    Log.e(TAG, "Autoexposure and capture failed with reason " +
512                                            failure.getReason());
513                                    // TODO: Make an error callback?
514                                }};
515
516                            // Set a one-time capture to trigger the camera driver's autoexposure:
517                            Camera2RequestSettingsSet expose =
518                                    new Camera2RequestSettingsSet(mPersistentSettings);
519                            expose.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER,
520                                    CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_START);
521                            try {
522                                mSession.capture(
523                                        expose.createRequest(mCamera, CameraDevice.TEMPLATE_PREVIEW,
524                                                mPreviewSurface),
525                                        /*listener*/deferredCallbackSetter, /*handler*/this);
526                            } catch (CameraAccessException ex) {
527                                Log.e(TAG, "Unable to run autoexposure and perform capture", ex);
528                            }
529                        }
530                        break;
531                    }
532
533                    default: {
534                        // TODO: Rephrase once everything has been implemented
535                        throw new RuntimeException("Unimplemented CameraProxy message=" + msg.what);
536                    }
537                }
538            } catch (final Exception ex) {
539                if (msg.what != CameraActions.RELEASE && mCamera != null) {
540                    // TODO: Handle this better
541                    mCamera.close();
542                    mCamera = null;
543                } else if (mCamera == null) {
544                    if (msg.what == CameraActions.OPEN_CAMERA) {
545                        if (mOpenCallback != null) {
546                            mOpenCallback.onDeviceOpenFailure(mCameraIndex,
547                                    generateHistoryString(mCameraIndex));
548                        }
549                    } else {
550                        Log.w(TAG, "Cannot handle message " + msg.what + ", mCamera is null");
551                    }
552                    return;
553                }
554
555                if (ex instanceof RuntimeException) {
556                    post(new Runnable() {
557                        @Override
558                        public void run() {
559                            sCameraExceptionCallback.onCameraException((RuntimeException) ex);
560                        }});
561                }
562            }
563        }
564
565        public CameraSettings buildSettings(AndroidCamera2Capabilities caps) {
566            try {
567                return new AndroidCamera2Settings(mCamera, CameraDevice.TEMPLATE_PREVIEW,
568                        mActiveArray, mPreviewSize, mPhotoSize);
569            } catch (CameraAccessException ex) {
570                Log.e(TAG, "Unable to query camera device to build settings representation");
571                return null;
572            }
573        }
574
575        /**
576         * Simply propagates settings from provided {@link CameraSettings}
577         * object to our {@link CaptureRequest.Builder} for use in captures.
578         * <p>Most conversions to match the API 2 formats are performed by
579         * {@link AndroidCamera2Capabilities.IntegralStringifier}; otherwise
580         * any final adjustments are done here before updating the builder.</p>
581         *
582         * @param settings The new/updated settings
583         */
584        private void applyToRequest(AndroidCamera2Settings settings) {
585            // TODO: If invoked when in PREVIEW_READY state, a new preview size will not take effect
586
587            mPersistentSettings.union(settings.getRequestSettings());
588            mPreviewSize = settings.getCurrentPreviewSize();
589            mPhotoSize = settings.getCurrentPhotoSize();
590
591            if (mCameraState.getState() >= AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
592                // If we're already previewing, reflect most settings immediately
593                try {
594                    mSession.setRepeatingRequest(
595                            mPersistentSettings.createRequest(mCamera,
596                                    CameraDevice.TEMPLATE_PREVIEW, mPreviewSurface),
597                            /*listener*/mCameraFocusStateListener, /*handler*/this);
598                } catch (CameraAccessException ex) {
599                    Log.e(TAG, "Failed to apply updated request settings", ex);
600                }
601            } else if (mCameraState.getState() < AndroidCamera2StateHolder.CAMERA_PREVIEW_READY) {
602                // If we're already ready to preview, this doesn't regress our state
603                mCameraState.setState(AndroidCamera2StateHolder.CAMERA_CONFIGURED);
604            }
605        }
606
607        private void setPreviewTexture(SurfaceTexture surfaceTexture) {
608            // TODO: Must be called after providing a .*Settings populated with sizes
609            // TODO: We don't technically offer a selection of sizes tailored to SurfaceTextures!
610
611            // TODO: Handle this error condition with a callback or exception
612            if (mCameraState.getState() < AndroidCamera2StateHolder.CAMERA_CONFIGURED) {
613                Log.w(TAG, "Ignoring texture setting at inappropriate time");
614                return;
615            }
616
617            // Avoid initializing another capture session unless we absolutely have to
618            if (surfaceTexture == mPreviewTexture) {
619                Log.i(TAG, "Optimizing out redundant preview texture setting");
620                return;
621            }
622
623            if (mSession != null) {
624                closePreviewSession();
625            }
626
627            mPreviewTexture = surfaceTexture;
628            surfaceTexture.setDefaultBufferSize(mPreviewSize.width(), mPreviewSize.height());
629
630            if (mPreviewSurface != null) {
631                mPreviewSurface.release();
632            }
633            mPreviewSurface = new Surface(surfaceTexture);
634
635            if (mCaptureReader != null) {
636                mCaptureReader.close();
637            }
638            mCaptureReader = ImageReader.newInstance(
639                    mPhotoSize.width(), mPhotoSize.height(), ImageFormat.JPEG, 1);
640
641            try {
642                mCamera.createCaptureSession(
643                        Arrays.asList(mPreviewSurface, mCaptureReader.getSurface()),
644                        mCameraPreviewStateListener, this);
645            } catch (CameraAccessException ex) {
646                Log.e(TAG, "Failed to create camera capture session", ex);
647            }
648        }
649
650        private void closePreviewSession() {
651            try {
652                mSession.abortCaptures();
653                mSession = null;
654            } catch (CameraAccessException ex) {
655                Log.e(TAG, "Failed to close existing camera capture session", ex);
656            }
657            mCameraState.setState(AndroidCamera2StateHolder.CAMERA_CONFIGURED);
658        }
659
660        // This listener monitors our connection to and disconnection from camera devices.
661        private CameraDevice.StateListener mCameraDeviceStateListener =
662                new CameraDevice.StateListener() {
663            @Override
664            public void onOpened(CameraDevice camera) {
665                mCamera = camera;
666                if (mOpenCallback != null) {
667                    try {
668                        CameraCharacteristics props =
669                                mCameraManager.getCameraCharacteristics(mCameraId);
670                        mCameraProxy = new AndroidCamera2ProxyImpl(mCameraIndex, mCamera,
671                                    getCameraDeviceInfo().getCharacteristics(mCameraIndex), props);
672                        mPersistentSettings = new Camera2RequestSettingsSet();
673                        mActiveArray =
674                                props.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE);
675                        mLegacyDevice =
676                                props.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL) ==
677                                        CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY;
678                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_UNCONFIGURED);
679                        mOpenCallback.onCameraOpened(mCameraProxy);
680                    } catch (CameraAccessException ex) {
681                        mOpenCallback.onDeviceOpenFailure(mCameraIndex,
682                                generateHistoryString(mCameraIndex));
683                    }
684                }
685            }
686
687            @Override
688            public void onDisconnected(CameraDevice camera) {
689                Log.w(TAG, "Camera device '" + mCameraIndex + "' was disconnected");
690            }
691
692            @Override
693            public void onError(CameraDevice camera, int error) {
694                Log.e(TAG, "Camera device '" + mCameraIndex + "' encountered error code '" +
695                        error + '\'');
696                if (mOpenCallback != null) {
697                    mOpenCallback.onDeviceOpenFailure(mCameraIndex,
698                            generateHistoryString(mCameraIndex));
699                }
700            }};
701
702        // This listener monitors our camera session (i.e. our transition into and out of preview).
703        private CameraCaptureSession.StateListener mCameraPreviewStateListener =
704                new CameraCaptureSession.StateListener() {
705            @Override
706            public void onConfigured(CameraCaptureSession session) {
707                mSession = session;
708                mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_READY);
709            }
710
711            @Override
712            public void onConfigureFailed(CameraCaptureSession session) {
713                // TODO: Invoke a callback
714                Log.e(TAG, "Failed to configure the camera for capture");
715            }
716
717            @Override
718            public void onActive(CameraCaptureSession session) {
719                if (mOneshotPreviewingCallback != null) {
720                    // The session is up and processing preview requests. Inform the caller.
721                    mOneshotPreviewingCallback.onPreviewStarted();
722                    mOneshotPreviewingCallback = null;
723                }
724            }};
725
726        // This listener monitors requested captures and notifies any relevant callbacks.
727        private CameraCaptureSession.CaptureListener mCameraFocusStateListener =
728                new CameraCaptureSession.CaptureListener() {
729            private int mLastAfState = -1;
730
731            @Override
732            public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request,
733                                           TotalCaptureResult result) {
734                Integer afStateMaybe = result.get(CaptureResult.CONTROL_AF_STATE);
735                if (afStateMaybe != null) {
736                    int afState = afStateMaybe;
737                    boolean afStateChanged = false;
738                    if (afState != mLastAfState) {
739                        mLastAfState = afState;
740                        afStateChanged = true;
741                    }
742
743                    switch (afState) {
744                        case CaptureResult.CONTROL_AF_STATE_PASSIVE_SCAN:
745                        case CaptureResult.CONTROL_AF_STATE_PASSIVE_FOCUSED:
746                        case CaptureResult.CONTROL_AF_STATE_PASSIVE_UNFOCUSED: {
747                            if (afStateChanged && mPassiveAfCallback != null) {
748                                // A CameraAFMoveCallback is attached. If we just started to scan,
749                                // the motor is moving; otherwise, it has settled.
750                                mPassiveAfCallback.onAutoFocusMoving(
751                                        afState == CaptureResult.CONTROL_AF_STATE_PASSIVE_SCAN,
752                                        mCameraProxy);
753                            }
754                            break;
755                        }
756
757                        case CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED:
758                        case CaptureResult.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED: {
759                            if (mOneshotAfCallback != null) {
760                                // A call to autoFocus() was just made to request a focus lock.
761                                // Notify the caller that the lens is now indefinitely fixed, and
762                                // report whether the image we're now stuck with is in focus.
763                                mOneshotAfCallback.onAutoFocus(
764                                        afState == CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED,
765                                        mCameraProxy);
766                                mOneshotAfCallback = null;
767                            }
768                            break;
769                        }
770                    }
771                }
772
773                Integer aeStateMaybe = result.get(CaptureResult.CONTROL_AE_STATE);
774                if (aeStateMaybe != null) {
775                    int aeState = aeStateMaybe;
776
777                    switch (aeState) {
778                        case CaptureResult.CONTROL_AE_STATE_CONVERGED:
779                        case CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED:
780                        case CaptureResult.CONTROL_AE_STATE_LOCKED: {
781                            if (mOneshotCaptureCallback != null) {
782                                // A call to takePicture() was just made, and autoexposure converged
783                                // so it's time to initiate the capture!
784                                mCaptureReader.setOnImageAvailableListener(mOneshotCaptureCallback,
785                                        /*handler*/Camera2Handler.this);
786                                try {
787                                    mSession.capture(
788                                            mPersistentSettings.createRequest(mCamera,
789                                                    CameraDevice.TEMPLATE_STILL_CAPTURE,
790                                                    mCaptureReader.getSurface()),
791                                            /*listener*/mOneshotCaptureCallback,
792                                            /*handler*/Camera2Handler.this);
793                                } catch (CameraAccessException ex) {
794                                    Log.e(TAG, "Unable to initiate capture", ex);
795                                } finally {
796                                    mOneshotCaptureCallback = null;
797                                }
798                            }
799                            break;
800                        }
801                    }
802                }
803            }
804
805            @Override
806            public void onCaptureFailed(CameraCaptureSession session, CaptureRequest request,
807                                        CaptureFailure failure) {
808                Log.e(TAG, "Capture attempt failed with reason " + failure.getReason());
809            }};
810    }
811
812    private class AndroidCamera2ProxyImpl extends CameraAgent.CameraProxy {
813        private final int mCameraIndex;
814        private final CameraDevice mCamera;
815        private final CameraDeviceInfo.Characteristics mCharacteristics;
816        private final AndroidCamera2Capabilities mCapabilities;
817
818        public AndroidCamera2ProxyImpl(int cameraIndex, CameraDevice camera,
819                CameraDeviceInfo.Characteristics characteristics,
820                CameraCharacteristics properties) {
821            mCameraIndex = cameraIndex;
822            mCamera = camera;
823            mCharacteristics = characteristics;
824            mCapabilities = new AndroidCamera2Capabilities(properties);
825        }
826
827        // TODO: Implement
828        @Override
829        public android.hardware.Camera getCamera() { return null; }
830
831        @Override
832        public int getCameraId() {
833            return mCameraIndex;
834        }
835
836        @Override
837        public CameraDeviceInfo.Characteristics getCharacteristics() {
838            return mCharacteristics;
839        }
840
841        @Override
842        public CameraCapabilities getCapabilities() {
843            return mCapabilities;
844        }
845
846        private AndroidCamera2Capabilities getSpecializedCapabilities() {
847            return mCapabilities;
848        }
849
850        // TODO: Implement
851        @Override
852        public void setPreviewDataCallback(Handler handler, CameraPreviewDataCallback cb) {}
853
854        // TODO: Implement
855        @Override
856        public void setOneShotPreviewCallback(Handler handler, CameraPreviewDataCallback cb) {}
857
858        // TODO: Implement
859        @Override
860        public void setPreviewDataCallbackWithBuffer(Handler handler, CameraPreviewDataCallback cb)
861                {}
862
863        // TODO: Implement
864        public void addCallbackBuffer(final byte[] callbackBuffer) {}
865
866        @Override
867        public void autoFocus(final Handler handler, final CameraAFCallback cb) {
868            mDispatchThread.runJob(new Runnable() {
869                @Override
870                public void run() {
871                    CameraAFCallback cbForward = null;
872                    if (cb != null) {
873                        cbForward = new CameraAFCallback() {
874                            @Override
875                            public void onAutoFocus(final boolean focused,
876                                                    final CameraProxy camera) {
877                                handler.post(new Runnable() {
878                                    @Override
879                                    public void run() {
880                                        cb.onAutoFocus(focused, camera);
881                                    }});
882                            }};
883                    }
884
885                    mCameraState.waitForStates(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE |
886                            AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED);
887                    mCameraHandler.obtainMessage(CameraActions.AUTO_FOCUS, cbForward)
888                            .sendToTarget();
889                }});
890        }
891
892        @TargetApi(Build.VERSION_CODES.JELLY_BEAN)
893        @Override
894        public void setAutoFocusMoveCallback(final Handler handler, final CameraAFMoveCallback cb) {
895            mDispatchThread.runJob(new Runnable() {
896                @Override
897                public void run() {
898                    CameraAFMoveCallback cbForward = null;
899                    if (cb != null) {
900                        cbForward = new CameraAFMoveCallback() {
901                            @Override
902                            public void onAutoFocusMoving(final boolean moving,
903                                                          final CameraProxy camera) {
904                                handler.post(new Runnable() {
905                                    @Override
906                                    public void run() {
907                                        cb.onAutoFocusMoving(moving, camera);
908                                    }});
909                                }};
910                    }
911
912                    mCameraHandler.obtainMessage(CameraActions.SET_AUTO_FOCUS_MOVE_CALLBACK,
913                            cbForward).sendToTarget();
914                }});
915        }
916
917        // TODO: Implement
918        @Override
919        public void takePicture(final Handler handler,
920                                final CameraShutterCallback shutter,
921                                CameraPictureCallback raw,
922                                CameraPictureCallback postview,
923                                final CameraPictureCallback jpeg) {
924            // TODO: We never call raw or postview
925            final CaptureAvailableListener picListener =
926                    new CaptureAvailableListener() {
927                @Override
928                public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request,
929                                             long timestamp) {
930                    if (shutter != null) {
931                        handler.post(new Runnable() {
932                            @Override
933                            public void run() {
934                                shutter.onShutter(AndroidCamera2ProxyImpl.this);
935                            }});
936                    }
937                }
938
939                @Override
940                public void onImageAvailable(ImageReader reader) {
941                    try (Image image = reader.acquireNextImage()) {
942                        if (jpeg != null) {
943                            ByteBuffer buffer = image.getPlanes()[0].getBuffer();
944                            final byte[] pixels = new byte[buffer.remaining()];
945                            buffer.get(pixels);
946                            handler.post(new Runnable() {
947                                @Override
948                                public void run() {
949                                    jpeg.onPictureTaken(pixels, AndroidCamera2ProxyImpl.this);
950                                }});
951                        }
952                    }
953                }};
954            mDispatchThread.runJob(new Runnable() {
955                @Override
956                public void run() {
957                    mCameraState.waitForStates(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE |
958                            AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED);
959                    mCameraHandler.obtainMessage(CameraActions.CAPTURE_PHOTO, picListener)
960                            .sendToTarget();
961                }});
962        }
963
964        // TODO: Implement
965        @Override
966        public void setZoomChangeListener(android.hardware.Camera.OnZoomChangeListener listener) {}
967
968        // TODO: Implement
969        @Override
970        public void setFaceDetectionCallback(Handler handler, CameraFaceDetectionCallback callback)
971                {}
972
973        // TODO: Remove this method override once we handle this message
974        @Override
975        public void startFaceDetection() {}
976
977        // TODO: Remove this method override once we handle this message
978        @Override
979        public void stopFaceDetection() {}
980
981        // TODO: Implement
982        @Override
983        public void setErrorCallback(Handler handler, CameraErrorCallback cb) {}
984
985        // TODO: Implement
986        @Override
987        public void setParameters(android.hardware.Camera.Parameters params) {}
988
989        // TODO: Implement
990        @Override
991        public android.hardware.Camera.Parameters getParameters() { return null; }
992
993        @Override
994        public CameraSettings getSettings() {
995            return mCameraHandler.buildSettings(mCapabilities);
996        }
997
998        @Override
999        public boolean applySettings(CameraSettings settings) {
1000            if (settings == null) {
1001                Log.w(TAG, "null parameters in applySettings()");
1002                return false;
1003            }
1004            if (!(settings instanceof AndroidCamera2Settings)) {
1005                Log.e(TAG, "Provided settings not compatible with the backing framework API");
1006                return false;
1007            }
1008
1009            return applySettingsHelper(settings, AndroidCamera2StateHolder.CAMERA_UNCONFIGURED |
1010                    AndroidCamera2StateHolder.CAMERA_CONFIGURED |
1011                    AndroidCamera2StateHolder.CAMERA_PREVIEW_READY);
1012        }
1013
1014        // TODO: Implement
1015        @Override
1016        public String dumpDeviceSettings() { return null; }
1017
1018        @Override
1019        public Handler getCameraHandler() {
1020            return AndroidCamera2AgentImpl.this.getCameraHandler();
1021        }
1022
1023        @Override
1024        public DispatchThread getDispatchThread() {
1025            return AndroidCamera2AgentImpl.this.getDispatchThread();
1026        }
1027
1028        @Override
1029        public CameraStateHolder getCameraState() {
1030            return mCameraState;
1031        }
1032    }
1033
1034    /** A linear state machine: each state entails all the states below it. */
1035    private static class AndroidCamera2StateHolder extends CameraStateHolder {
1036        // Usage flow: openCamera() -> applySettings() -> setPreviewTexture() -> startPreview() ->
1037        //             autoFocus() -> takePicture()
1038        /* Camera states */
1039        /** No camera device is opened. */
1040        public static final int CAMERA_UNOPENED = 1;
1041        /** A camera is opened, but no settings have been provided. */
1042        public static final int CAMERA_UNCONFIGURED = 2;
1043        /** The open camera has been configured by providing it with settings. */
1044        public static final int CAMERA_CONFIGURED = 3;
1045        /** A capture session is ready to stream a preview, but still has no repeating request. */
1046        public static final int CAMERA_PREVIEW_READY = 4;
1047        /** A preview is currently being streamed. */
1048        public static final int CAMERA_PREVIEW_ACTIVE = 5;
1049        /** The lens is locked on a particular region. */
1050        public static final int CAMERA_FOCUS_LOCKED = 6;
1051
1052        public AndroidCamera2StateHolder() {
1053            this(CAMERA_UNOPENED);
1054        }
1055
1056        public AndroidCamera2StateHolder(int state) {
1057            super(state);
1058        }
1059    }
1060
1061    private static class AndroidCamera2DeviceInfo implements CameraDeviceInfo {
1062        private final CameraManager mCameraManager;
1063        private final String[] mCameraIds;
1064        private final int mNumberOfCameras;
1065        private final int mFirstBackCameraId;
1066        private final int mFirstFrontCameraId;
1067
1068        public AndroidCamera2DeviceInfo(CameraManager cameraManager,
1069                                        String[] cameraIds, int numberOfCameras) {
1070            mCameraManager = cameraManager;
1071            mCameraIds = cameraIds;
1072            mNumberOfCameras = numberOfCameras;
1073
1074            int firstBackId = NO_DEVICE;
1075            int firstFrontId = NO_DEVICE;
1076            for (int id = 0; id < cameraIds.length; ++id) {
1077                try {
1078                    int lensDirection = cameraManager.getCameraCharacteristics(cameraIds[id])
1079                            .get(CameraCharacteristics.LENS_FACING);
1080                    if (firstBackId == NO_DEVICE &&
1081                            lensDirection == CameraCharacteristics.LENS_FACING_BACK) {
1082                        firstBackId = id;
1083                    }
1084                    if (firstFrontId == NO_DEVICE &&
1085                            lensDirection == CameraCharacteristics.LENS_FACING_FRONT) {
1086                        firstFrontId = id;
1087                    }
1088                } catch (CameraAccessException ex) {
1089                    Log.w(TAG, "Couldn't get characteristics of camera '" + id + "'", ex);
1090                }
1091            }
1092            mFirstBackCameraId = firstBackId;
1093            mFirstFrontCameraId = firstFrontId;
1094        }
1095
1096        @Override
1097        public Characteristics getCharacteristics(int cameraId) {
1098            String actualId = mCameraIds[cameraId];
1099            try {
1100                CameraCharacteristics info = mCameraManager.getCameraCharacteristics(actualId);
1101                return new AndroidCharacteristics2(info);
1102            } catch (CameraAccessException ex) {
1103                return null;
1104            }
1105        }
1106
1107        @Override
1108        public int getNumberOfCameras() {
1109            return mNumberOfCameras;
1110        }
1111
1112        @Override
1113        public int getFirstBackCameraId() {
1114            return mFirstBackCameraId;
1115        }
1116
1117        @Override
1118        public int getFirstFrontCameraId() {
1119            return mFirstFrontCameraId;
1120        }
1121
1122        private static class AndroidCharacteristics2 extends Characteristics {
1123            private CameraCharacteristics mCameraInfo;
1124
1125            AndroidCharacteristics2(CameraCharacteristics cameraInfo) {
1126                mCameraInfo = cameraInfo;
1127            }
1128
1129            @Override
1130            public boolean isFacingBack() {
1131                return mCameraInfo.get(CameraCharacteristics.LENS_FACING)
1132                        .equals(CameraCharacteristics.LENS_FACING_BACK);
1133            }
1134
1135            @Override
1136            public boolean isFacingFront() {
1137                return mCameraInfo.get(CameraCharacteristics.LENS_FACING)
1138                        .equals(CameraCharacteristics.LENS_FACING_FRONT);
1139            }
1140
1141            @Override
1142            public int getSensorOrientation() {
1143                return mCameraInfo.get(CameraCharacteristics.SENSOR_ORIENTATION);
1144            }
1145
1146            @Override
1147            public boolean canDisableShutterSound() {
1148                // The new API doesn't support this operation, so don't encourage people to try it.
1149                // TODO: What kind of assumptions have callers made about this result's meaning?
1150                return false;
1151            }
1152        }
1153    }
1154
1155    private static final CameraExceptionCallback sCameraExceptionCallback =
1156            new CameraExceptionCallback() {
1157                @Override
1158                public synchronized void onCameraException(RuntimeException e) {
1159                    throw e;
1160                }
1161            };
1162}
1163