AndroidCamera2AgentImpl.java revision 80cda8dae801f09c2ce928d0f3feb8a569939fc2 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.ex.camera2.portability;
18
19import android.annotation.TargetApi;
20import android.content.Context;
21import android.graphics.ImageFormat;
22import android.graphics.Matrix;
23import android.graphics.Rect;
24import android.graphics.RectF;
25import android.graphics.SurfaceTexture;
26import android.hardware.camera2.CameraAccessException;
27import android.hardware.camera2.CameraCaptureSession;
28import android.hardware.camera2.CameraCharacteristics;
29import android.hardware.camera2.CameraDevice;
30import android.hardware.camera2.CameraManager;
31import android.hardware.camera2.CaptureFailure;
32import android.hardware.camera2.CaptureRequest;
33import android.hardware.camera2.CaptureResult;
34import android.hardware.camera2.TotalCaptureResult;
35import android.hardware.camera2.params.MeteringRectangle;
36import android.media.Image;
37import android.media.ImageReader;
38import android.media.MediaActionSound;
39import android.os.Build;
40import android.os.Handler;
41import android.os.HandlerThread;
42import android.os.Looper;
43import android.os.Message;
44import android.view.Surface;
45
46import com.android.ex.camera2.portability.debug.Log;
47import com.android.ex.camera2.utils.Camera2RequestSettingsSet;
48
49import java.nio.ByteBuffer;
50import java.util.ArrayList;
51import java.util.Arrays;
52import java.util.HashSet;
53import java.util.List;
54import java.util.Set;
55
56/**
57 * A class to implement {@link CameraAgent} of the Android camera2 framework.
58 */
59class AndroidCamera2AgentImpl extends CameraAgent {
60    private static final Log.Tag TAG = new Log.Tag("AndCam2AgntImp");
61
62    private final Camera2Handler mCameraHandler;
63    private final HandlerThread mCameraHandlerThread;
64    private final CameraStateHolder mCameraState;
65    private final DispatchThread mDispatchThread;
66    private final CameraManager mCameraManager;
67    private final MediaActionSound mNoisemaker;
68
69    /**
70     * Number of camera devices.  The length of {@code mCameraDevices} does not reveal this
71     * information because that list may contain since-invalidated indices.
72     */
73    private int mNumCameraDevices;
74
75    /**
76     * Transformation between integral camera indices and the {@link java.lang.String} indices used
77     * by the underlying API.  Note that devices may disappear because they've been disconnected or
78     * have otherwise gone offline.  Because we need to keep the meanings of whatever indices we
79     * expose stable, we cannot simply remove them in such a case; instead, we insert {@code null}s
80     * to invalidate any such indices.  Whenever new devices appear, they are appended to the end of
81     * the list, and thereby assigned the lowest index that has never yet been used.
82     */
83    private final List<String> mCameraDevices;
84
85    AndroidCamera2AgentImpl(Context context) {
86        mCameraHandlerThread = new HandlerThread("Camera2 Handler Thread");
87        mCameraHandlerThread.start();
88        mCameraHandler = new Camera2Handler(mCameraHandlerThread.getLooper());
89        mCameraState = new AndroidCamera2StateHolder();
90        mDispatchThread = new DispatchThread(mCameraHandler, mCameraHandlerThread);
91        mDispatchThread.start();
92        mCameraManager = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE);
93        mNoisemaker = new MediaActionSound();
94        mNoisemaker.load(MediaActionSound.SHUTTER_CLICK);
95
96        mNumCameraDevices = 0;
97        mCameraDevices = new ArrayList<String>();
98        updateCameraDevices();
99    }
100
101    /**
102     * Updates the camera device index assignments stored in {@link mCameraDevices}, without
103     * reappropriating any currently-assigned index.
104     * @return Whether the operation was successful
105     */
106    private boolean updateCameraDevices() {
107        try {
108            String[] currentCameraDevices = mCameraManager.getCameraIdList();
109            Set<String> currentSet = new HashSet<String>(Arrays.asList(currentCameraDevices));
110
111            // Invalidate the indices assigned to any camera devices that are no longer present
112            for (int index = 0; index < mCameraDevices.size(); ++index) {
113                if (!currentSet.contains(mCameraDevices.get(index))) {
114                    mCameraDevices.set(index, null);
115                    --mNumCameraDevices;
116                }
117            }
118
119            // Assign fresh indices to any new camera devices
120            currentSet.removeAll(mCameraDevices); // The devices we didn't know about
121            for (String device : currentCameraDevices) {
122                if (currentSet.contains(device)) {
123                    mCameraDevices.add(device);
124                    ++mNumCameraDevices;
125                }
126            }
127
128            return true;
129        } catch (CameraAccessException ex) {
130            Log.e(TAG, "Could not get device listing from camera subsystem", ex);
131            return false;
132        }
133    }
134
135    // TODO: Implement
136    @Override
137    public void setCameraDefaultExceptionCallback(CameraExceptionCallback callback,
138            Handler handler) {}
139
140    // TODO: Implement
141    @Override
142    public void recycle() {}
143
144    // TODO: Some indices may now be invalid; ensure everyone can handle that and update the docs
145    @Override
146    public CameraDeviceInfo getCameraDeviceInfo() {
147        updateCameraDevices();
148        return new AndroidCamera2DeviceInfo(mCameraManager, mCameraDevices.toArray(new String[0]),
149                mNumCameraDevices);
150    }
151
152    @Override
153    protected Handler getCameraHandler() {
154        return mCameraHandler;
155    }
156
157    @Override
158    protected DispatchThread getDispatchThread() {
159        return mDispatchThread;
160    }
161
162    private static abstract class CaptureAvailableListener
163            extends CameraCaptureSession.CaptureListener
164            implements ImageReader.OnImageAvailableListener {};
165
166    private class Camera2Handler extends HistoryHandler {
167        // Caller-provided when leaving CAMERA_UNOPENED state:
168        private CameraOpenCallback mOpenCallback;
169        private int mCameraIndex;
170        private String mCameraId;
171
172        // Available in CAMERA_UNCONFIGURED state and above:
173        private CameraDevice mCamera;
174        private AndroidCamera2ProxyImpl mCameraProxy;
175        private Camera2RequestSettingsSet mPersistentSettings;
176        private Rect mActiveArray;
177        private boolean mLegacyDevice;
178
179        // Available in CAMERA_CONFIGURED state and above:
180        private Size mPreviewSize;
181        private Size mPhotoSize;
182
183        // Available in PREVIEW_READY state and above:
184        private SurfaceTexture mPreviewTexture;
185        private Surface mPreviewSurface;
186        private CameraCaptureSession mSession;
187        private ImageReader mCaptureReader;
188
189        // Available from the beginning of PREVIEW_ACTIVE until the first preview frame arrives:
190        private CameraStartPreviewCallback mOneshotPreviewingCallback;
191
192        // Available in FOCUS_LOCKED between AF trigger receipt and whenever the lens stops moving:
193        private CameraAFCallback mOneshotAfCallback;
194
195        // Available when taking picture between AE trigger receipt and autoexposure convergence
196        private CaptureAvailableListener mOneshotCaptureCallback;
197
198        // Available whenever setAutoFocusMoveCallback() was last invoked with a non-null argument:
199        private CameraAFMoveCallback mPassiveAfCallback;
200
201        Camera2Handler(Looper looper) {
202            super(looper);
203        }
204
205        @Override
206        public void handleMessage(final Message msg) {
207            super.handleMessage(msg);
208            try {
209                switch(msg.what) {
210                    case CameraActions.OPEN_CAMERA:
211                    case CameraActions.RECONNECT: {
212                        CameraOpenCallback openCallback = (CameraOpenCallback) msg.obj;
213                        int cameraIndex = msg.arg1;
214
215                        if (mCameraState.getState() != AndroidCamera2StateHolder.CAMERA_UNOPENED) {
216                            openCallback.onDeviceOpenedAlready(cameraIndex,
217                                    generateHistoryString(cameraIndex));
218                            break;
219                        }
220
221                        mOpenCallback = openCallback;
222                        mCameraIndex = cameraIndex;
223                        mCameraId = mCameraDevices.get(mCameraIndex);
224                        Log.i(TAG, String.format("Opening camera index %d (id %s) with camera2 API",
225                                cameraIndex, mCameraId));
226
227                        if (mCameraId == null) {
228                            mOpenCallback.onCameraDisabled(msg.arg1);
229                            break;
230                        }
231                        mCameraManager.openCamera(mCameraId, mCameraDeviceStateListener, this);
232
233                        break;
234                    }
235
236                    case CameraActions.RELEASE: {
237                        if (mCameraState.getState() == AndroidCamera2StateHolder.CAMERA_UNOPENED) {
238                            Log.w(TAG, "Ignoring release at inappropriate time");
239                            break;
240                        }
241
242                        if (mSession != null) {
243                            closePreviewSession();
244                            mSession = null;
245                        }
246                        if (mCamera != null) {
247                            mCamera.close();
248                            mCamera = null;
249                        }
250                        mCameraProxy = null;
251                        mPersistentSettings = null;
252                        mActiveArray = null;
253                        if (mPreviewSurface != null) {
254                            mPreviewSurface.release();
255                            mPreviewSurface = null;
256                        }
257                        mPreviewTexture = null;
258                        if (mCaptureReader != null) {
259                            mCaptureReader.close();
260                            mCaptureReader = null;
261                        }
262                        mPreviewSize = null;
263                        mPhotoSize = null;
264                        mCameraIndex = 0;
265                        mCameraId = null;
266                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_UNOPENED);
267                        break;
268                    }
269
270                    /*case CameraActions.UNLOCK: {
271                        break;
272                    }
273
274                    case CameraActions.LOCK: {
275                        break;
276                    }*/
277
278                    case CameraActions.SET_PREVIEW_TEXTURE_ASYNC: {
279                        setPreviewTexture((SurfaceTexture) msg.obj);
280                        break;
281                    }
282
283                    case CameraActions.START_PREVIEW_ASYNC: {
284                        if (mCameraState.getState() !=
285                                        AndroidCamera2StateHolder.CAMERA_PREVIEW_READY) {
286                            // TODO: Provide better feedback here?
287                            Log.w(TAG, "Refusing to start preview at inappropriate time");
288                            break;
289                        }
290
291                        mOneshotPreviewingCallback = (CameraStartPreviewCallback) msg.obj;
292                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE);
293                        try {
294                            mSession.setRepeatingRequest(
295                                    mPersistentSettings.createRequest(mCamera,
296                                            CameraDevice.TEMPLATE_PREVIEW, mPreviewSurface),
297                                    /*listener*/mCameraFocusStateListener, /*handler*/this);
298                        } catch(CameraAccessException ex) {
299                            Log.w(TAG, "Unable to start preview", ex);
300                            mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_READY);
301                        }
302                        break;
303                    }
304
305                    case CameraActions.STOP_PREVIEW: {
306                        if (mCameraState.getState() <
307                                        AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
308                            Log.w(TAG, "Refusing to stop preview at inappropriate time");
309                            break;
310                        }
311
312                        mSession.stopRepeating();
313                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_READY);
314                        break;
315                    }
316
317                    /*case CameraActions.SET_PREVIEW_CALLBACK_WITH_BUFFER: {
318                        break;
319                    }
320
321                    case CameraActions.ADD_CALLBACK_BUFFER: {
322                        break;
323                    }
324
325                    case CameraActions.SET_PREVIEW_DISPLAY_ASYNC: {
326                        break;
327                    }
328
329                    case CameraActions.SET_PREVIEW_CALLBACK: {
330                        break;
331                    }
332
333                    case CameraActions.SET_ONE_SHOT_PREVIEW_CALLBACK: {
334                        break;
335                    }
336
337                    case CameraActions.SET_PARAMETERS: {
338                        break;
339                    }
340
341                    case CameraActions.GET_PARAMETERS: {
342                        break;
343                    }
344
345                    case CameraActions.REFRESH_PARAMETERS: {
346                        break;
347                    }*/
348
349                    case CameraActions.APPLY_SETTINGS: {
350                        AndroidCamera2Settings settings = (AndroidCamera2Settings) msg.obj;
351                        applyToRequest(settings);
352                        break;
353                    }
354
355                    case CameraActions.AUTO_FOCUS: {
356                        // We only support locking the focus while a preview is being displayed.
357                        // However, it can be requested multiple times in succession; the effect of
358                        // the subsequent invocations is determined by the focus mode defined in the
359                        // provided CameraSettings object. In passive (CONTINUOUS_*) mode, the
360                        // duplicate requests are no-ops and leave the lens locked at its current
361                        // position, but in active (AUTO) mode, they perform another scan and lock
362                        // once that is finished. In any manual focus mode, this call is a no-op,
363                        // and most notably, this is the only case where the callback isn't invoked.
364                        if (mCameraState.getState() <
365                                        AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
366                            Log.w(TAG, "Ignoring attempt to autofocus without preview");
367                            break;
368                        }
369
370                        // The earliest we can reliably tell whether the autofocus has locked in
371                        // response to our latest request is when our one-time capture completes.
372                        // However, it will probably take longer than that, so once that happens,
373                        // just start checking the repeating preview requests as they complete.
374                        final CameraAFCallback callback = (CameraAFCallback) msg.obj;
375                        CameraCaptureSession.CaptureListener deferredCallbackSetter =
376                                new CameraCaptureSession.CaptureListener() {
377                            @Override
378                            public void onCaptureCompleted(CameraCaptureSession session,
379                                                           CaptureRequest request,
380                                                           TotalCaptureResult result) {
381                                // Now our mCameraFocusStateListener will invoke the callback the
382                                // first time it finds the focus motor to be locked.
383                                mOneshotAfCallback = callback;
384                            }
385
386                            @Override
387                            public void onCaptureFailed(CameraCaptureSession session,
388                                                        CaptureRequest request,
389                                                        CaptureFailure failure) {
390                                Log.e(TAG, "Focusing failed with reason " + failure.getReason());
391                                callback.onAutoFocus(false, mCameraProxy);
392                            }};
393
394                        // Send a one-time capture to trigger the camera driver to lock focus.
395                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED);
396                        Camera2RequestSettingsSet trigger =
397                                new Camera2RequestSettingsSet(mPersistentSettings);
398                        trigger.set(CaptureRequest.CONTROL_AF_TRIGGER,
399                                CaptureRequest.CONTROL_AF_TRIGGER_START);
400                        try {
401                            mSession.capture(
402                                    trigger.createRequest(mCamera, CameraDevice.TEMPLATE_PREVIEW,
403                                            mPreviewSurface),
404                                    /*listener*/deferredCallbackSetter, /*handler*/ this);
405                        } catch(CameraAccessException ex) {
406                            Log.e(TAG, "Unable to lock autofocus", ex);
407                            mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE);
408                        }
409                        break;
410                    }
411
412                    case CameraActions.CANCEL_AUTO_FOCUS: {
413                        // Why would you want to unlock the lens if it isn't already locked?
414                        if (mCameraState.getState() <
415                                AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
416                            Log.w(TAG, "Ignoring attempt to release focus lock without preview");
417                            break;
418                        }
419
420                        // Send a one-time capture to trigger the camera driver to resume scanning.
421                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE);
422                        Camera2RequestSettingsSet cancel =
423                                new Camera2RequestSettingsSet(mPersistentSettings);
424                        cancel.set(CaptureRequest.CONTROL_AF_TRIGGER,
425                                CaptureRequest.CONTROL_AF_TRIGGER_CANCEL);
426                        try {
427                            mSession.capture(
428                                    cancel.createRequest(mCamera, CameraDevice.TEMPLATE_PREVIEW,
429                                            mPreviewSurface),
430                                    /*listener*/null, /*handler*/this);
431                        } catch(CameraAccessException ex) {
432                            Log.e(TAG, "Unable to cancel autofocus", ex);
433                            mCameraState.setState(
434                                    AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED);
435                        }
436                        break;
437                    }
438
439                    case CameraActions.SET_AUTO_FOCUS_MOVE_CALLBACK: {
440                        mPassiveAfCallback = (CameraAFMoveCallback) msg.obj;
441                        break;
442                    }
443
444                    /*case CameraActions.SET_ZOOM_CHANGE_LISTENER: {
445                        break;
446                    }
447
448                    case CameraActions.SET_FACE_DETECTION_LISTENER: {
449                        break;
450                    }
451
452                    case CameraActions.START_FACE_DETECTION: {
453                        break;
454                    }
455
456                    case CameraActions.STOP_FACE_DETECTION: {
457                        break;
458                    }
459
460                    case CameraActions.SET_ERROR_CALLBACK: {
461                        break;
462                    }
463
464                    case CameraActions.ENABLE_SHUTTER_SOUND: {
465                        break;
466                    }*/
467
468                    case CameraActions.SET_DISPLAY_ORIENTATION: {
469                        // Only set the JPEG capture orientation if requested to do so; otherwise,
470                        // capture in the sensor's physical orientation
471                        mPersistentSettings.set(CaptureRequest.JPEG_ORIENTATION, msg.arg2 > 0 ?
472                                mCameraProxy.getCharacteristics().getJpegOrientation(msg.arg1) : 0);
473                        break;
474                    }
475
476                    case CameraActions.CAPTURE_PHOTO: {
477                        if (mCameraState.getState() <
478                                        AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
479                            Log.e(TAG, "Photos may only be taken when a preview is active");
480                            break;
481                        }
482                        if (mCameraState.getState() !=
483                                AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED) {
484                            Log.w(TAG, "Taking a (likely blurry) photo without the lens locked");
485                        }
486
487                        final CaptureAvailableListener listener =
488                                (CaptureAvailableListener) msg.obj;
489                        if (mLegacyDevice) {
490                            // Just snap the shot
491                            mCaptureReader.setOnImageAvailableListener(listener, /*handler*/this);
492                            try {
493                                mSession.capture(
494                                        mPersistentSettings.createRequest(mCamera,
495                                                CameraDevice.TEMPLATE_STILL_CAPTURE,
496                                                mCaptureReader.getSurface()),
497                                        listener, /*handler*/this);
498                            } catch (CameraAccessException ex) {
499                                Log.e(TAG, "Unable to initiate legacy capture", ex);
500                            }
501                        } else {
502                            // Not a legacy device, so we need to let AE converge before capturing
503                            CameraCaptureSession.CaptureListener deferredCallbackSetter =
504                                    new CameraCaptureSession.CaptureListener() {
505                                @Override
506                                public void onCaptureCompleted(CameraCaptureSession session,
507                                                               CaptureRequest request,
508                                                               TotalCaptureResult result) {
509                                    mOneshotCaptureCallback = listener;
510                                }
511
512                                @Override
513                                public void onCaptureFailed(CameraCaptureSession session,
514                                                            CaptureRequest request,
515                                                            CaptureFailure failure) {
516                                    Log.e(TAG, "Autoexposure and capture failed with reason " +
517                                            failure.getReason());
518                                    // TODO: Make an error callback?
519                                }};
520
521                            // Set a one-time capture to trigger the camera driver's autoexposure:
522                            Camera2RequestSettingsSet expose =
523                                    new Camera2RequestSettingsSet(mPersistentSettings);
524                            expose.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER,
525                                    CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_START);
526                            try {
527                                mSession.capture(
528                                        expose.createRequest(mCamera, CameraDevice.TEMPLATE_PREVIEW,
529                                                mPreviewSurface),
530                                        /*listener*/deferredCallbackSetter, /*handler*/this);
531                            } catch (CameraAccessException ex) {
532                                Log.e(TAG, "Unable to run autoexposure and perform capture", ex);
533                            }
534                        }
535                        break;
536                    }
537
538                    default: {
539                        // TODO: Rephrase once everything has been implemented
540                        throw new RuntimeException("Unimplemented CameraProxy message=" + msg.what);
541                    }
542                }
543            } catch (final Exception ex) {
544                if (msg.what != CameraActions.RELEASE && mCamera != null) {
545                    // TODO: Handle this better
546                    mCamera.close();
547                    mCamera = null;
548                } else if (mCamera == null) {
549                    if (msg.what == CameraActions.OPEN_CAMERA) {
550                        if (mOpenCallback != null) {
551                            mOpenCallback.onDeviceOpenFailure(mCameraIndex,
552                                    generateHistoryString(mCameraIndex));
553                        }
554                    } else {
555                        Log.w(TAG, "Cannot handle message " + msg.what + ", mCamera is null");
556                    }
557                    return;
558                }
559
560                if (ex instanceof RuntimeException) {
561                    post(new Runnable() {
562                        @Override
563                        public void run() {
564                            sCameraExceptionCallback.onCameraException((RuntimeException) ex);
565                        }});
566                }
567            }
568        }
569
570        public CameraSettings buildSettings(AndroidCamera2Capabilities caps) {
571            try {
572                return new AndroidCamera2Settings(mCamera, CameraDevice.TEMPLATE_PREVIEW,
573                        mActiveArray, mPreviewSize, mPhotoSize);
574            } catch (CameraAccessException ex) {
575                Log.e(TAG, "Unable to query camera device to build settings representation");
576                return null;
577            }
578        }
579
580        /**
581         * Simply propagates settings from provided {@link CameraSettings}
582         * object to our {@link CaptureRequest.Builder} for use in captures.
583         * <p>Most conversions to match the API 2 formats are performed by
584         * {@link AndroidCamera2Capabilities.IntegralStringifier}; otherwise
585         * any final adjustments are done here before updating the builder.</p>
586         *
587         * @param settings The new/updated settings
588         */
589        private void applyToRequest(AndroidCamera2Settings settings) {
590            // TODO: If invoked when in PREVIEW_READY state, a new preview size will not take effect
591
592            mPersistentSettings.union(settings.getRequestSettings());
593            mPreviewSize = settings.getCurrentPreviewSize();
594            mPhotoSize = settings.getCurrentPhotoSize();
595
596            if (mCameraState.getState() >= AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE) {
597                // If we're already previewing, reflect most settings immediately
598                try {
599                    mSession.setRepeatingRequest(
600                            mPersistentSettings.createRequest(mCamera,
601                                    CameraDevice.TEMPLATE_PREVIEW, mPreviewSurface),
602                            /*listener*/mCameraFocusStateListener, /*handler*/this);
603                } catch (CameraAccessException ex) {
604                    Log.e(TAG, "Failed to apply updated request settings", ex);
605                }
606            } else if (mCameraState.getState() < AndroidCamera2StateHolder.CAMERA_PREVIEW_READY) {
607                // If we're already ready to preview, this doesn't regress our state
608                mCameraState.setState(AndroidCamera2StateHolder.CAMERA_CONFIGURED);
609            }
610        }
611
612        private void setPreviewTexture(SurfaceTexture surfaceTexture) {
613            // TODO: Must be called after providing a .*Settings populated with sizes
614            // TODO: We don't technically offer a selection of sizes tailored to SurfaceTextures!
615
616            // TODO: Handle this error condition with a callback or exception
617            if (mCameraState.getState() < AndroidCamera2StateHolder.CAMERA_CONFIGURED) {
618                Log.w(TAG, "Ignoring texture setting at inappropriate time");
619                return;
620            }
621
622            // Avoid initializing another capture session unless we absolutely have to
623            if (surfaceTexture == mPreviewTexture) {
624                Log.i(TAG, "Optimizing out redundant preview texture setting");
625                return;
626            }
627
628            if (mSession != null) {
629                closePreviewSession();
630            }
631
632            mPreviewTexture = surfaceTexture;
633            surfaceTexture.setDefaultBufferSize(mPreviewSize.width(), mPreviewSize.height());
634
635            if (mPreviewSurface != null) {
636                mPreviewSurface.release();
637            }
638            mPreviewSurface = new Surface(surfaceTexture);
639
640            if (mCaptureReader != null) {
641                mCaptureReader.close();
642            }
643            mCaptureReader = ImageReader.newInstance(
644                    mPhotoSize.width(), mPhotoSize.height(), ImageFormat.JPEG, 1);
645
646            try {
647                mCamera.createCaptureSession(
648                        Arrays.asList(mPreviewSurface, mCaptureReader.getSurface()),
649                        mCameraPreviewStateListener, this);
650            } catch (CameraAccessException ex) {
651                Log.e(TAG, "Failed to create camera capture session", ex);
652            }
653        }
654
655        private void closePreviewSession() {
656            try {
657                mSession.abortCaptures();
658                mSession = null;
659            } catch (CameraAccessException ex) {
660                Log.e(TAG, "Failed to close existing camera capture session", ex);
661            }
662            mCameraState.setState(AndroidCamera2StateHolder.CAMERA_CONFIGURED);
663        }
664
665        // This listener monitors our connection to and disconnection from camera devices.
666        private CameraDevice.StateListener mCameraDeviceStateListener =
667                new CameraDevice.StateListener() {
668            @Override
669            public void onOpened(CameraDevice camera) {
670                mCamera = camera;
671                if (mOpenCallback != null) {
672                    try {
673                        CameraCharacteristics props =
674                                mCameraManager.getCameraCharacteristics(mCameraId);
675                        mCameraProxy = new AndroidCamera2ProxyImpl(mCameraIndex, mCamera,
676                                    getCameraDeviceInfo().getCharacteristics(mCameraIndex), props);
677                        mPersistentSettings = new Camera2RequestSettingsSet();
678                        mActiveArray =
679                                props.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE);
680                        mLegacyDevice =
681                                props.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL) ==
682                                        CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY;
683                        mCameraState.setState(AndroidCamera2StateHolder.CAMERA_UNCONFIGURED);
684                        mOpenCallback.onCameraOpened(mCameraProxy);
685                    } catch (CameraAccessException ex) {
686                        mOpenCallback.onDeviceOpenFailure(mCameraIndex,
687                                generateHistoryString(mCameraIndex));
688                    }
689                }
690            }
691
692            @Override
693            public void onDisconnected(CameraDevice camera) {
694                Log.w(TAG, "Camera device '" + mCameraIndex + "' was disconnected");
695            }
696
697            @Override
698            public void onError(CameraDevice camera, int error) {
699                Log.e(TAG, "Camera device '" + mCameraIndex + "' encountered error code '" +
700                        error + '\'');
701                if (mOpenCallback != null) {
702                    mOpenCallback.onDeviceOpenFailure(mCameraIndex,
703                            generateHistoryString(mCameraIndex));
704                }
705            }};
706
707        // This listener monitors our camera session (i.e. our transition into and out of preview).
708        private CameraCaptureSession.StateListener mCameraPreviewStateListener =
709                new CameraCaptureSession.StateListener() {
710            @Override
711            public void onConfigured(CameraCaptureSession session) {
712                mSession = session;
713                mCameraState.setState(AndroidCamera2StateHolder.CAMERA_PREVIEW_READY);
714            }
715
716            @Override
717            public void onConfigureFailed(CameraCaptureSession session) {
718                // TODO: Invoke a callback
719                Log.e(TAG, "Failed to configure the camera for capture");
720            }
721
722            @Override
723            public void onActive(CameraCaptureSession session) {
724                if (mOneshotPreviewingCallback != null) {
725                    // The session is up and processing preview requests. Inform the caller.
726                    mOneshotPreviewingCallback.onPreviewStarted();
727                    mOneshotPreviewingCallback = null;
728                }
729            }};
730
731        // This listener monitors requested captures and notifies any relevant callbacks.
732        private CameraCaptureSession.CaptureListener mCameraFocusStateListener =
733                new CameraCaptureSession.CaptureListener() {
734            private int mLastAfState = -1;
735
736            @Override
737            public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request,
738                                           TotalCaptureResult result) {
739                Integer afStateMaybe = result.get(CaptureResult.CONTROL_AF_STATE);
740                if (afStateMaybe != null) {
741                    int afState = afStateMaybe;
742                    boolean afStateChanged = false;
743                    if (afState != mLastAfState) {
744                        mLastAfState = afState;
745                        afStateChanged = true;
746                    }
747
748                    switch (afState) {
749                        case CaptureResult.CONTROL_AF_STATE_PASSIVE_SCAN:
750                        case CaptureResult.CONTROL_AF_STATE_PASSIVE_FOCUSED:
751                        case CaptureResult.CONTROL_AF_STATE_PASSIVE_UNFOCUSED: {
752                            if (afStateChanged && mPassiveAfCallback != null) {
753                                // A CameraAFMoveCallback is attached. If we just started to scan,
754                                // the motor is moving; otherwise, it has settled.
755                                mPassiveAfCallback.onAutoFocusMoving(
756                                        afState == CaptureResult.CONTROL_AF_STATE_PASSIVE_SCAN,
757                                        mCameraProxy);
758                            }
759                            break;
760                        }
761
762                        case CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED:
763                        case CaptureResult.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED: {
764                            if (mOneshotAfCallback != null) {
765                                // A call to autoFocus() was just made to request a focus lock.
766                                // Notify the caller that the lens is now indefinitely fixed, and
767                                // report whether the image we're now stuck with is in focus.
768                                mOneshotAfCallback.onAutoFocus(
769                                        afState == CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED,
770                                        mCameraProxy);
771                                mOneshotAfCallback = null;
772                            }
773                            break;
774                        }
775                    }
776                }
777
778                Integer aeStateMaybe = result.get(CaptureResult.CONTROL_AE_STATE);
779                if (aeStateMaybe != null) {
780                    int aeState = aeStateMaybe;
781
782                    switch (aeState) {
783                        case CaptureResult.CONTROL_AE_STATE_CONVERGED:
784                        case CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED:
785                        case CaptureResult.CONTROL_AE_STATE_LOCKED: {
786                            if (mOneshotCaptureCallback != null) {
787                                // A call to takePicture() was just made, and autoexposure converged
788                                // so it's time to initiate the capture!
789                                mCaptureReader.setOnImageAvailableListener(mOneshotCaptureCallback,
790                                        /*handler*/Camera2Handler.this);
791                                try {
792                                    mSession.capture(
793                                            mPersistentSettings.createRequest(mCamera,
794                                                    CameraDevice.TEMPLATE_STILL_CAPTURE,
795                                                    mCaptureReader.getSurface()),
796                                            /*listener*/mOneshotCaptureCallback,
797                                            /*handler*/Camera2Handler.this);
798                                } catch (CameraAccessException ex) {
799                                    Log.e(TAG, "Unable to initiate capture", ex);
800                                } finally {
801                                    mOneshotCaptureCallback = null;
802                                }
803                            }
804                            break;
805                        }
806                    }
807                }
808            }
809
810            @Override
811            public void onCaptureFailed(CameraCaptureSession session, CaptureRequest request,
812                                        CaptureFailure failure) {
813                Log.e(TAG, "Capture attempt failed with reason " + failure.getReason());
814            }};
815    }
816
817    private class AndroidCamera2ProxyImpl extends CameraAgent.CameraProxy {
818        private final int mCameraIndex;
819        private final CameraDevice mCamera;
820        private final CameraDeviceInfo.Characteristics mCharacteristics;
821        private final AndroidCamera2Capabilities mCapabilities;
822        private CameraSettings mLastSettings;
823
824        public AndroidCamera2ProxyImpl(int cameraIndex, CameraDevice camera,
825                CameraDeviceInfo.Characteristics characteristics,
826                CameraCharacteristics properties) {
827            mCameraIndex = cameraIndex;
828            mCamera = camera;
829            mCharacteristics = characteristics;
830            mCapabilities = new AndroidCamera2Capabilities(properties);
831            mLastSettings = null;
832        }
833
834        // TODO: Implement
835        @Override
836        public android.hardware.Camera getCamera() { return null; }
837
838        @Override
839        public int getCameraId() {
840            return mCameraIndex;
841        }
842
843        @Override
844        public CameraDeviceInfo.Characteristics getCharacteristics() {
845            return mCharacteristics;
846        }
847
848        @Override
849        public CameraCapabilities getCapabilities() {
850            return mCapabilities;
851        }
852
853        private AndroidCamera2Capabilities getSpecializedCapabilities() {
854            return mCapabilities;
855        }
856
857        // TODO: Implement
858        @Override
859        public void setPreviewDataCallback(Handler handler, CameraPreviewDataCallback cb) {}
860
861        // TODO: Implement
862        @Override
863        public void setOneShotPreviewCallback(Handler handler, CameraPreviewDataCallback cb) {}
864
865        // TODO: Implement
866        @Override
867        public void setPreviewDataCallbackWithBuffer(Handler handler, CameraPreviewDataCallback cb)
868                {}
869
870        // TODO: Implement
871        public void addCallbackBuffer(final byte[] callbackBuffer) {}
872
873        @Override
874        public void autoFocus(final Handler handler, final CameraAFCallback cb) {
875            mDispatchThread.runJob(new Runnable() {
876                @Override
877                public void run() {
878                    CameraAFCallback cbForward = null;
879                    if (cb != null) {
880                        cbForward = new CameraAFCallback() {
881                            @Override
882                            public void onAutoFocus(final boolean focused,
883                                                    final CameraProxy camera) {
884                                handler.post(new Runnable() {
885                                    @Override
886                                    public void run() {
887                                        cb.onAutoFocus(focused, camera);
888                                    }});
889                            }};
890                    }
891
892                    mCameraState.waitForStates(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE |
893                            AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED);
894                    mCameraHandler.obtainMessage(CameraActions.AUTO_FOCUS, cbForward)
895                            .sendToTarget();
896                }});
897        }
898
899        @TargetApi(Build.VERSION_CODES.JELLY_BEAN)
900        @Override
901        public void setAutoFocusMoveCallback(final Handler handler, final CameraAFMoveCallback cb) {
902            mDispatchThread.runJob(new Runnable() {
903                @Override
904                public void run() {
905                    CameraAFMoveCallback cbForward = null;
906                    if (cb != null) {
907                        cbForward = new CameraAFMoveCallback() {
908                            @Override
909                            public void onAutoFocusMoving(final boolean moving,
910                                                          final CameraProxy camera) {
911                                handler.post(new Runnable() {
912                                    @Override
913                                    public void run() {
914                                        cb.onAutoFocusMoving(moving, camera);
915                                    }});
916                                }};
917                    }
918
919                    mCameraHandler.obtainMessage(CameraActions.SET_AUTO_FOCUS_MOVE_CALLBACK,
920                            cbForward).sendToTarget();
921                }});
922        }
923
924        @Override
925        public void takePicture(final Handler handler,
926                                final CameraShutterCallback shutter,
927                                CameraPictureCallback raw,
928                                CameraPictureCallback postview,
929                                final CameraPictureCallback jpeg) {
930            // TODO: We never call raw or postview
931            final CaptureAvailableListener picListener =
932                    new CaptureAvailableListener() {
933                @Override
934                public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request,
935                                             long timestamp) {
936                    if (shutter != null) {
937                        handler.post(new Runnable() {
938                            @Override
939                            public void run() {
940                                mNoisemaker.play(MediaActionSound.SHUTTER_CLICK);
941                                shutter.onShutter(AndroidCamera2ProxyImpl.this);
942                            }});
943                    }
944                }
945
946                @Override
947                public void onImageAvailable(ImageReader reader) {
948                    try (Image image = reader.acquireNextImage()) {
949                        if (jpeg != null) {
950                            ByteBuffer buffer = image.getPlanes()[0].getBuffer();
951                            final byte[] pixels = new byte[buffer.remaining()];
952                            buffer.get(pixels);
953                            handler.post(new Runnable() {
954                                @Override
955                                public void run() {
956                                    jpeg.onPictureTaken(pixels, AndroidCamera2ProxyImpl.this);
957                                }});
958                        }
959                    }
960                }};
961            mDispatchThread.runJob(new Runnable() {
962                @Override
963                public void run() {
964                    mCameraState.waitForStates(AndroidCamera2StateHolder.CAMERA_PREVIEW_ACTIVE |
965                            AndroidCamera2StateHolder.CAMERA_FOCUS_LOCKED);
966                    mCameraHandler.obtainMessage(CameraActions.CAPTURE_PHOTO, picListener)
967                            .sendToTarget();
968                }});
969        }
970
971        // TODO: Implement
972        @Override
973        public void setZoomChangeListener(android.hardware.Camera.OnZoomChangeListener listener) {}
974
975        // TODO: Implement
976        @Override
977        public void setFaceDetectionCallback(Handler handler, CameraFaceDetectionCallback callback)
978                {}
979
980        // TODO: Remove this method override once we handle this message
981        @Override
982        public void startFaceDetection() {}
983
984        // TODO: Remove this method override once we handle this message
985        @Override
986        public void stopFaceDetection() {}
987
988        // TODO: Implement
989        @Override
990        public void setErrorCallback(Handler handler, CameraErrorCallback cb) {}
991
992        // TODO: Implement
993        @Override
994        public void setParameters(android.hardware.Camera.Parameters params) {}
995
996        // TODO: Implement
997        @Override
998        public android.hardware.Camera.Parameters getParameters() { return null; }
999
1000        @Override
1001        public CameraSettings getSettings() {
1002            if (mLastSettings == null) {
1003                mLastSettings = mCameraHandler.buildSettings(mCapabilities);
1004            }
1005            return mLastSettings;
1006        }
1007
1008        @Override
1009        public boolean applySettings(CameraSettings settings) {
1010            if (settings == null) {
1011                Log.w(TAG, "null parameters in applySettings()");
1012                return false;
1013            }
1014            if (!(settings instanceof AndroidCamera2Settings)) {
1015                Log.e(TAG, "Provided settings not compatible with the backing framework API");
1016                return false;
1017            }
1018
1019            if (applySettingsHelper(settings, AndroidCamera2StateHolder.CAMERA_UNCONFIGURED |
1020                    AndroidCamera2StateHolder.CAMERA_CONFIGURED |
1021                    AndroidCamera2StateHolder.CAMERA_PREVIEW_READY)) {
1022                mLastSettings = settings;
1023                return true;
1024            }
1025            return false;
1026        }
1027
1028        // TODO: Implement
1029        @Override
1030        public String dumpDeviceSettings() { return null; }
1031
1032        @Override
1033        public Handler getCameraHandler() {
1034            return AndroidCamera2AgentImpl.this.getCameraHandler();
1035        }
1036
1037        @Override
1038        public DispatchThread getDispatchThread() {
1039            return AndroidCamera2AgentImpl.this.getDispatchThread();
1040        }
1041
1042        @Override
1043        public CameraStateHolder getCameraState() {
1044            return mCameraState;
1045        }
1046    }
1047
1048    /** A linear state machine: each state entails all the states below it. */
1049    private static class AndroidCamera2StateHolder extends CameraStateHolder {
1050        // Usage flow: openCamera() -> applySettings() -> setPreviewTexture() -> startPreview() ->
1051        //             autoFocus() -> takePicture()
1052        /* Camera states */
1053        /** No camera device is opened. */
1054        public static final int CAMERA_UNOPENED = 1;
1055        /** A camera is opened, but no settings have been provided. */
1056        public static final int CAMERA_UNCONFIGURED = 2;
1057        /** The open camera has been configured by providing it with settings. */
1058        public static final int CAMERA_CONFIGURED = 3;
1059        /** A capture session is ready to stream a preview, but still has no repeating request. */
1060        public static final int CAMERA_PREVIEW_READY = 4;
1061        /** A preview is currently being streamed. */
1062        public static final int CAMERA_PREVIEW_ACTIVE = 5;
1063        /** The lens is locked on a particular region. */
1064        public static final int CAMERA_FOCUS_LOCKED = 6;
1065
1066        public AndroidCamera2StateHolder() {
1067            this(CAMERA_UNOPENED);
1068        }
1069
1070        public AndroidCamera2StateHolder(int state) {
1071            super(state);
1072        }
1073    }
1074
1075    private static class AndroidCamera2DeviceInfo implements CameraDeviceInfo {
1076        private final CameraManager mCameraManager;
1077        private final String[] mCameraIds;
1078        private final int mNumberOfCameras;
1079        private final int mFirstBackCameraId;
1080        private final int mFirstFrontCameraId;
1081
1082        public AndroidCamera2DeviceInfo(CameraManager cameraManager,
1083                                        String[] cameraIds, int numberOfCameras) {
1084            mCameraManager = cameraManager;
1085            mCameraIds = cameraIds;
1086            mNumberOfCameras = numberOfCameras;
1087
1088            int firstBackId = NO_DEVICE;
1089            int firstFrontId = NO_DEVICE;
1090            for (int id = 0; id < cameraIds.length; ++id) {
1091                try {
1092                    int lensDirection = cameraManager.getCameraCharacteristics(cameraIds[id])
1093                            .get(CameraCharacteristics.LENS_FACING);
1094                    if (firstBackId == NO_DEVICE &&
1095                            lensDirection == CameraCharacteristics.LENS_FACING_BACK) {
1096                        firstBackId = id;
1097                    }
1098                    if (firstFrontId == NO_DEVICE &&
1099                            lensDirection == CameraCharacteristics.LENS_FACING_FRONT) {
1100                        firstFrontId = id;
1101                    }
1102                } catch (CameraAccessException ex) {
1103                    Log.w(TAG, "Couldn't get characteristics of camera '" + id + "'", ex);
1104                }
1105            }
1106            mFirstBackCameraId = firstBackId;
1107            mFirstFrontCameraId = firstFrontId;
1108        }
1109
1110        @Override
1111        public Characteristics getCharacteristics(int cameraId) {
1112            String actualId = mCameraIds[cameraId];
1113            try {
1114                CameraCharacteristics info = mCameraManager.getCameraCharacteristics(actualId);
1115                return new AndroidCharacteristics2(info);
1116            } catch (CameraAccessException ex) {
1117                return null;
1118            }
1119        }
1120
1121        @Override
1122        public int getNumberOfCameras() {
1123            return mNumberOfCameras;
1124        }
1125
1126        @Override
1127        public int getFirstBackCameraId() {
1128            return mFirstBackCameraId;
1129        }
1130
1131        @Override
1132        public int getFirstFrontCameraId() {
1133            return mFirstFrontCameraId;
1134        }
1135
1136        private static class AndroidCharacteristics2 extends Characteristics {
1137            private CameraCharacteristics mCameraInfo;
1138
1139            AndroidCharacteristics2(CameraCharacteristics cameraInfo) {
1140                mCameraInfo = cameraInfo;
1141            }
1142
1143            @Override
1144            public boolean isFacingBack() {
1145                return mCameraInfo.get(CameraCharacteristics.LENS_FACING)
1146                        .equals(CameraCharacteristics.LENS_FACING_BACK);
1147            }
1148
1149            @Override
1150            public boolean isFacingFront() {
1151                return mCameraInfo.get(CameraCharacteristics.LENS_FACING)
1152                        .equals(CameraCharacteristics.LENS_FACING_FRONT);
1153            }
1154
1155            @Override
1156            public int getSensorOrientation() {
1157                return mCameraInfo.get(CameraCharacteristics.SENSOR_ORIENTATION);
1158            }
1159
1160            @Override
1161            public Matrix getPreviewTransform(int currentDisplayOrientation,
1162                                              RectF surfaceDimensions,
1163                                              RectF desiredBounds) {
1164                if (!orientationIsValid(currentDisplayOrientation)) {
1165                    return new Matrix();
1166                }
1167
1168                // The system transparently transforms the image to fill the surface
1169                // when the device is in its natural orientation. We rotate the
1170                // coordinates of the rectangle's corners to be relative to the
1171                // original image, instead of to the current screen orientation.
1172                float[] surfacePolygon = rotate(convertRectToPoly(surfaceDimensions),
1173                        2 * currentDisplayOrientation / 90);
1174                float[] desiredPolygon = convertRectToPoly(desiredBounds);
1175
1176                Matrix transform = new Matrix();
1177                // Use polygons instead of rectangles so that rotation will be
1178                // calculated, since that is not done by the new camera API.
1179                transform.setPolyToPoly(surfacePolygon, 0, desiredPolygon, 0, 4);
1180                return transform;
1181            }
1182
1183            @Override
1184            public boolean canDisableShutterSound() {
1185                // The new API doesn't support this operation, so don't encourage people to try it.
1186                // TODO: What kind of assumptions have callers made about this result's meaning?
1187                return false;
1188            }
1189
1190            private static float[] convertRectToPoly(RectF rf) {
1191                return new float[] {rf.left, rf.top, rf.right, rf.top,
1192                        rf.right, rf.bottom, rf.left, rf.bottom};
1193            }
1194
1195            private static float[] rotate(float[] arr, int times) {
1196                if (times < 0) {
1197                    times = times % arr.length + arr.length;
1198                }
1199
1200                float[] res = new float[arr.length];
1201                for (int offset = 0; offset < arr.length; ++offset) {
1202                    res[offset] = arr[(times + offset) % arr.length];
1203                }
1204                return res;
1205            }
1206        }
1207    }
1208
1209    private static final CameraExceptionCallback sCameraExceptionCallback =
1210            new CameraExceptionCallback() {
1211                @Override
1212                public synchronized void onCameraException(RuntimeException e) {
1213                    throw e;
1214                }
1215            };
1216}
1217