xref: /packages/apps/DevCamera/src/com/android/devcamera/Api2Camera.java

/*
 * Copyright (C) 2016 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.android.devcamera;

import android.content.Context;
import android.graphics.ImageFormat;
import android.graphics.SurfaceTexture;
import android.hardware.camera2.CameraAccessException;
import android.hardware.camera2.CameraCaptureSession;
import android.hardware.camera2.CameraCharacteristics;
import android.hardware.camera2.CameraDevice;
import android.hardware.camera2.CameraManager;
import android.hardware.camera2.CameraMetadata;
import android.hardware.camera2.CaptureRequest;
import android.hardware.camera2.CaptureResult;
import android.hardware.camera2.TotalCaptureResult;
import android.hardware.camera2.params.Face;
import android.hardware.camera2.params.InputConfiguration;
import android.media.Image;
import android.media.ImageReader;
import android.media.ImageWriter;
import android.media.MediaActionSound;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.SystemClock;
import android.util.Log;
import android.util.Size;
import android.view.Surface;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

import javax.microedition.khronos.opengles.GL10;


/**
 * Api2Camera : a camera2 implementation
 *
 * The goal here is to make the simplest possible API2 camera,
 * where individual streams and capture options (e.g. edge enhancement,
 * noise reduction, face detection) can be toggled on and off.
 *
 */

public class Api2Camera implements CameraInterface, SurfaceTexture.OnFrameAvailableListener {
    private static final String TAG = "DevCamera_API2";

    // Nth frame to log; put 10^6 if you don't want logging.
    private static int LOG_NTH_FRAME = 30;
    // Log dropped frames. There are a log on Angler MDA32.
    private static boolean LOG_DROPPED_FRAMES = true;

    // IMPORTANT: Only one of these can be true:
    private static boolean SECOND_YUV_IMAGEREADER_STREAM = true;
    private static boolean SECOND_SURFACE_TEXTURE_STREAM = false;

    // Enable raw stream if available.
    private static boolean RAW_STREAM_ENABLE = true;
    // Use JPEG ImageReader and YUV ImageWriter if reprocessing is available
    private static final boolean USE_REPROCESSING_IF_AVAIL = true;

    // Whether we are continuously taking pictures, or not.
    boolean mIsBursting = false;
    // Last total capture result
    TotalCaptureResult mLastTotalCaptureResult;

    // ImageReader/Writer buffer sizes.
    private static final int YUV1_IMAGEREADER_SIZE = 8;
    private static final int YUV2_IMAGEREADER_SIZE = 8;
    private static final int RAW_IMAGEREADER_SIZE = 8;
    private static final int IMAGEWRITER_SIZE = 2;

    private CameraInfoCache mCameraInfoCache;
    private CameraManager mCameraManager;
    private CameraCaptureSession mCurrentCaptureSession;
    private MediaActionSound mMediaActionSound = new MediaActionSound();

    MyCameraCallback mMyCameraCallback;

    // Generally everything running on this thread & this module is *not thread safe*.
    private HandlerThread mOpsThread;
    private Handler mOpsHandler;
    private HandlerThread mInitThread;
    private Handler mInitHandler;
    private HandlerThread mJpegListenerThread;
    private Handler mJpegListenerHandler;

    Context mContext;
    boolean mCameraIsFront;
    SurfaceTexture mSurfaceTexture;
    Surface mSurfaceTextureSurface;

    private boolean mFirstFrameArrived;
    private ImageReader mYuv1ImageReader;
    private int mYuv1ImageCounter;
    // Handle to last received Image: allows ZSL to be implemented.
    private Image mYuv1LastReceivedImage = null;
    // Time at which reprocessing request went in (right now we are doing one at a time).
    private long mReprocessingRequestNanoTime;

    private ImageReader mJpegImageReader;
    private ImageReader mYuv2ImageReader;
    private int mYuv2ImageCounter;
    private ImageReader mRawImageReader;
    private int mRawImageCounter;

    // Starting the preview requires each of these 3 to be true/non-null:
    volatile private Surface mPreviewSurface;
    volatile private CameraDevice mCameraDevice;
    volatile boolean mAllThingsInitialized = false;

    /**
     * Constructor.
     */
    public Api2Camera(Context context, boolean useFrontCamera) {
        mContext = context;
        mCameraIsFront = useFrontCamera;
        mCameraManager = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE);
        mCameraInfoCache = new CameraInfoCache(mCameraManager, useFrontCamera);

        // Create thread and handler for camera operations.
        mOpsThread = new HandlerThread("CameraOpsThread");
        mOpsThread.start();
        mOpsHandler = new Handler(mOpsThread.getLooper());

        // Create thread and handler for slow initialization operations.
        // Don't want to use camera operations thread because we want to time camera open carefully.
        mInitThread = new HandlerThread("CameraInitThread");
        mInitThread.start();
        mInitHandler = new Handler(mInitThread.getLooper());
        mInitHandler.post(new Runnable() {
            @Override
            public void run() {
                InitializeAllTheThings();
                mAllThingsInitialized = true;
                Log.v(TAG, "STARTUP_REQUIREMENT ImageReader initialization done.");
                tryToStartCaptureSession();
            }
        });

        // Set initial Noise and Edge modes.
        if (mCameraInfoCache.isHardwareLevelAtLeast(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_3)) {
            // YUV streams.
            if (mCameraInfoCache.supportedModesContains(mCameraInfoCache.noiseModes,
                    CameraCharacteristics.NOISE_REDUCTION_MODE_ZERO_SHUTTER_LAG)) {
                mCaptureNoiseMode = CameraCharacteristics.NOISE_REDUCTION_MODE_ZERO_SHUTTER_LAG;
            } else {
                mCaptureNoiseMode = CameraCharacteristics.NOISE_REDUCTION_MODE_FAST;
            }
            if (mCameraInfoCache.supportedModesContains(mCameraInfoCache.edgeModes,
                    CameraCharacteristics.EDGE_MODE_ZERO_SHUTTER_LAG)) {
                mCaptureEdgeMode = CameraCharacteristics.EDGE_MODE_ZERO_SHUTTER_LAG;
            } else {
                mCaptureEdgeMode = CameraCharacteristics.EDGE_MODE_FAST;
            }

            // Reprocessing.
            mReprocessingNoiseMode = CameraCharacteristics.NOISE_REDUCTION_MODE_HIGH_QUALITY;
            mReprocessingEdgeMode = CameraCharacteristics.EDGE_MODE_HIGH_QUALITY;
        }
    }

    // Ugh, why is this stuff so slow?
    private void InitializeAllTheThings() {

        // Thread to handle returned JPEGs.
        mJpegListenerThread = new HandlerThread("CameraJpegThread");
        mJpegListenerThread.start();
        mJpegListenerHandler = new Handler(mJpegListenerThread.getLooper());

        // Create ImageReader to receive JPEG image buffers via reprocessing.
        mJpegImageReader = ImageReader.newInstance(
                mCameraInfoCache.getYuvStream1Size().getWidth(),
                mCameraInfoCache.getYuvStream1Size().getHeight(),
                ImageFormat.JPEG,
                2);
        mJpegImageReader.setOnImageAvailableListener(mJpegImageListener, mJpegListenerHandler);

        // Create ImageReader to receive YUV image buffers.
        mYuv1ImageReader = ImageReader.newInstance(
                mCameraInfoCache.getYuvStream1Size().getWidth(),
                mCameraInfoCache.getYuvStream1Size().getHeight(),
                ImageFormat.YUV_420_888,
                YUV1_IMAGEREADER_SIZE);
        mYuv1ImageReader.setOnImageAvailableListener(mYuv1ImageListener, mOpsHandler);

        if (SECOND_YUV_IMAGEREADER_STREAM) {
            // Create ImageReader to receive YUV image buffers.
            mYuv2ImageReader = ImageReader.newInstance(
                    mCameraInfoCache.getYuvStream2Size().getWidth(),
                    mCameraInfoCache.getYuvStream2Size().getHeight(),
                    ImageFormat.YUV_420_888,
                    YUV2_IMAGEREADER_SIZE);
            mYuv2ImageReader.setOnImageAvailableListener(mYuv2ImageListener, mOpsHandler);
        }

        if (SECOND_SURFACE_TEXTURE_STREAM) {
            int[] textures = new int[1];
            // generate one texture pointer and bind it as an external texture.
            GLES20.glGenTextures(1, textures, 0);
            GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textures[0]);
            // No mip-mapping with camera source.
            GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
                    GL10.GL_TEXTURE_MIN_FILTER,
                    GL10.GL_LINEAR);
            GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
                    GL10.GL_TEXTURE_MAG_FILTER, GL10.GL_LINEAR);
            // Clamp to edge is only option.
            GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
                    GL10.GL_TEXTURE_WRAP_S, GL10.GL_CLAMP_TO_EDGE);
            GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
                    GL10.GL_TEXTURE_WRAP_T, GL10.GL_CLAMP_TO_EDGE);

            int texture_id = textures[0];
            mSurfaceTexture = new SurfaceTexture(texture_id);
            mSurfaceTexture.setDefaultBufferSize(320, 240);
            mSurfaceTexture.setOnFrameAvailableListener(this);
            mSurfaceTextureSurface = new Surface(mSurfaceTexture);
        }

        if (RAW_STREAM_ENABLE && mCameraInfoCache.rawAvailable()) {
            // Create ImageReader to receive thumbnail sized YUV image buffers.
            mRawImageReader = ImageReader.newInstance(
                    mCameraInfoCache.getRawStreamSize().getWidth(),
                    mCameraInfoCache.getRawStreamSize().getHeight(),
                    mCameraInfoCache.getRawFormat(),
                    RAW_IMAGEREADER_SIZE);
            mRawImageReader.setOnImageAvailableListener(mRawImageListener, mOpsHandler);
        }

        // Load click sound.
        mMediaActionSound.load(MediaActionSound.SHUTTER_CLICK);

    }

    public void setCallback(MyCameraCallback callback) {
        mMyCameraCallback = callback;
    }

    public void triggerAFScan() {
        Log.v(TAG, "AF trigger");
        issuePreviewCaptureRequest(true);
    }

    public void setCAF() {
        Log.v(TAG, "run CAF");
        issuePreviewCaptureRequest(false);
    }

    public void takePicture() {
        mMediaActionSound.play(MediaActionSound.SHUTTER_CLICK);
        mOpsHandler.post(new Runnable() {
            @Override
            public void run() {
                runReprocessing();
            }
        });
    }

    public void onFrameAvailable (SurfaceTexture surfaceTexture) {
        Log.v(TAG, " onFrameAvailable(SurfaceTexture)");
    }

    public void setBurst(boolean go) {
        // if false to true transition.
        if (go && !mIsBursting) {
            takePicture();
        }
        mIsBursting = go;
    }

    public boolean isRawAvailable() {
        return mCameraInfoCache.rawAvailable();
    }

    public boolean isReprocessingAvailable() {
        return mCameraInfoCache.isYuvReprocessingAvailable();
    }

    @Override
    public Size getPreviewSize() {
        return mCameraInfoCache.getPreviewSize();
    }

    @Override
    public float[] getFieldOfView() {
        return mCameraInfoCache.getFieldOfView();
    }

    @Override
    public int getOrientation() {
        return mCameraInfoCache.sensorOrientation();
    }

    @Override
    public void openCamera() {
        // If API2 FULL mode is not available, display toast
        if (!mCameraInfoCache.isCamera2FullModeAvailable()) {
            mMyCameraCallback.noCamera2Full();
        }

        Log.v(TAG, "Opening camera " + mCameraInfoCache.getCameraId());
        mOpsHandler.post(new Runnable() {
            @Override
            public void run() {
                CameraTimer.t_open_start = SystemClock.elapsedRealtime();
                try {
                    mCameraManager.openCamera(mCameraInfoCache.getCameraId(), mCameraStateCallback, null);
                } catch (CameraAccessException e) {
                    Log.e(TAG, "Unable to openCamera().");
                }
            }
        });
    }

    @Override
    public void closeCamera() {
        // TODO: We are stalling main thread now which is bad.
        Log.v(TAG, "Closing camera " + mCameraInfoCache.getCameraId());
        if (mCameraDevice != null) {
            try {
                mCurrentCaptureSession.abortCaptures();
            } catch (CameraAccessException e) {
                Log.e(TAG, "Could not abortCaptures().");
            }
            mCameraDevice.close();
        }
        mCurrentCaptureSession = null;
        Log.v(TAG, "Done closing camera " + mCameraInfoCache.getCameraId());
    }

    public void startPreview(final Surface surface) {
        Log.v(TAG, "STARTUP_REQUIREMENT preview Surface ready.");
        mPreviewSurface = surface;
        tryToStartCaptureSession();
    }

    private CameraDevice.StateCallback mCameraStateCallback = new LoggingCallbacks.DeviceStateCallback() {
        @Override
        public void onOpened(CameraDevice camera) {
            CameraTimer.t_open_end = SystemClock.elapsedRealtime();
            mCameraDevice = camera;
            Log.v(TAG, "STARTUP_REQUIREMENT Done opening camera " + mCameraInfoCache.getCameraId() +
                    ". HAL open took: (" + (CameraTimer.t_open_end - CameraTimer.t_open_start) + " ms)");

            super.onOpened(camera);
            tryToStartCaptureSession();
        }
    };

    private void tryToStartCaptureSession() {
        if (mCameraDevice != null && mAllThingsInitialized && mPreviewSurface != null) {
            mOpsHandler.post(new Runnable() {
                @Override
                public void run() {
                    // It used to be: this needed to be posted on a Handler.
                    startCaptureSession();
                }
            });
        }
    }

    // Create CameraCaptureSession. Callback will start repeating request with current parameters.
    private void startCaptureSession() {
        CameraTimer.t_session_go = SystemClock.elapsedRealtime();

        Log.v(TAG, "Configuring session..");
        List<Surface> outputSurfaces = new ArrayList<Surface>(3);

        outputSurfaces.add(mPreviewSurface);
        Log.v(TAG, "  .. added SurfaceView " + mCameraInfoCache.getPreviewSize().getWidth() +
                " x " + mCameraInfoCache.getPreviewSize().getHeight());

        outputSurfaces.add(mYuv1ImageReader.getSurface());
        Log.v(TAG, "  .. added YUV ImageReader " + mCameraInfoCache.getYuvStream1Size().getWidth() +
                " x " + mCameraInfoCache.getYuvStream1Size().getHeight());

        if (SECOND_YUV_IMAGEREADER_STREAM) {
            outputSurfaces.add(mYuv2ImageReader.getSurface());
            Log.v(TAG, "  .. added YUV ImageReader " + mCameraInfoCache.getYuvStream2Size().getWidth() +
                    " x " + mCameraInfoCache.getYuvStream2Size().getHeight());
        }

        if (SECOND_SURFACE_TEXTURE_STREAM) {
            outputSurfaces.add(mSurfaceTextureSurface);
            Log.v(TAG, "  .. added SurfaceTexture");
        }

        if (RAW_STREAM_ENABLE && mCameraInfoCache.rawAvailable()) {
            outputSurfaces.add(mRawImageReader.getSurface());
            Log.v(TAG, "  .. added Raw ImageReader " + mCameraInfoCache.getRawStreamSize().getWidth() +
                    " x " + mCameraInfoCache.getRawStreamSize().getHeight());
        }

        if (USE_REPROCESSING_IF_AVAIL && mCameraInfoCache.isYuvReprocessingAvailable()) {
            outputSurfaces.add(mJpegImageReader.getSurface());
            Log.v(TAG, "  .. added JPEG ImageReader " + mCameraInfoCache.getJpegStreamSize().getWidth() +
                    " x " + mCameraInfoCache.getJpegStreamSize().getHeight());
        }

        try {
            if (USE_REPROCESSING_IF_AVAIL && mCameraInfoCache.isYuvReprocessingAvailable()) {
                InputConfiguration inputConfig = new InputConfiguration(mCameraInfoCache.getYuvStream1Size().getWidth(),
                        mCameraInfoCache.getYuvStream1Size().getHeight(), ImageFormat.YUV_420_888);
                mCameraDevice.createReprocessableCaptureSession(inputConfig, outputSurfaces,
                        mSessionStateCallback, null);
                Log.v(TAG, "  Call to createReprocessableCaptureSession complete.");
            } else {
                mCameraDevice.createCaptureSession(outputSurfaces, mSessionStateCallback, null);
                Log.v(TAG, "  Call to createCaptureSession complete.");
            }

        } catch (CameraAccessException e) {
            Log.e(TAG, "Error configuring ISP.");
        }
    }

    ImageWriter mImageWriter;

    private CameraCaptureSession.StateCallback mSessionStateCallback = new LoggingCallbacks.SessionStateCallback() {
        @Override
        public void onReady(CameraCaptureSession session) {
            Log.v(TAG, "capture session onReady().  HAL capture session took: (" + (SystemClock.elapsedRealtime() - CameraTimer.t_session_go) + " ms)");
            mCurrentCaptureSession = session;
            issuePreviewCaptureRequest(false);

            if (session.isReprocessable()) {
                mImageWriter = ImageWriter.newInstance(session.getInputSurface(), IMAGEWRITER_SIZE);
                mImageWriter.setOnImageReleasedListener(
                        new ImageWriter.OnImageReleasedListener() {
                            @Override
                            public void onImageReleased(ImageWriter writer) {
                                Log.v(TAG, "ImageWriter.OnImageReleasedListener onImageReleased()");
                            }
                        }, null);
                Log.v(TAG, "Created ImageWriter.");
            }
            super.onReady(session);
        }
    };

    // Variables to hold capture flow state.
    private boolean mCaptureYuv1 = false;
    private boolean mCaptureYuv2 = false;
    private boolean mCaptureRaw = false;
    private int mCaptureNoiseMode = CaptureRequest.NOISE_REDUCTION_MODE_FAST;
    private int mCaptureEdgeMode = CaptureRequest.EDGE_MODE_FAST;
    private boolean mCaptureFace = false;
    // Variables to hold reprocessing state.
    private int mReprocessingNoiseMode = CaptureRequest.NOISE_REDUCTION_MODE_HIGH_QUALITY;
    private int mReprocessingEdgeMode = CaptureRequest.EDGE_MODE_HIGH_QUALITY;

    public void setCaptureFlow(Boolean yuv1, Boolean yuv2, Boolean raw10, Boolean nr, Boolean edge, Boolean face) {
        if (yuv1 != null) mCaptureYuv1 = yuv1;
        if (yuv2 != null) mCaptureYuv2 = yuv2;
        if (raw10 != null) mCaptureRaw = raw10 && RAW_STREAM_ENABLE;
        if (nr) {
            mCaptureNoiseMode = getNextMode(mCaptureNoiseMode, mCameraInfoCache.noiseModes);
        }
        if (edge) {
            mCaptureEdgeMode = getNextMode(mCaptureEdgeMode, mCameraInfoCache.edgeModes);
        }
        if (face != null) mCaptureFace = face;
        mMyCameraCallback.setNoiseEdgeText(
                "NR " + noiseModeToString(mCaptureNoiseMode),
                "Edge " + edgeModeToString(mCaptureEdgeMode)
        );

        if (mCurrentCaptureSession != null) {
            issuePreviewCaptureRequest(false);
        }
    }

    public void setReprocessingFlow(Boolean nr, Boolean edge) {
        if (nr) {
            mReprocessingNoiseMode = getNextMode(mReprocessingNoiseMode, mCameraInfoCache.noiseModes);
        }
        if (edge) {
            mReprocessingEdgeMode = getNextMode(mReprocessingEdgeMode, mCameraInfoCache.edgeModes);
        }
        mMyCameraCallback.setNoiseEdgeTextForReprocessing(
                "NR " + noiseModeToString(mReprocessingNoiseMode),
                "Edge " + edgeModeToString(mReprocessingEdgeMode)
        );
    }

    public void issuePreviewCaptureRequest(boolean AFtrigger) {
        CameraTimer.t_burst = SystemClock.elapsedRealtime();
        Log.v(TAG, "issuePreviewCaptureRequest...");
        try {
            CaptureRequest.Builder b1 = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
            b1.set(CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_USE_SCENE_MODE);
            b1.set(CaptureRequest.CONTROL_SCENE_MODE, CameraMetadata.CONTROL_SCENE_MODE_FACE_PRIORITY);
            if (AFtrigger) {
                b1.set(CaptureRequest.CONTROL_AF_MODE, CameraMetadata.CONTROL_AF_MODE_AUTO);
            } else {
                b1.set(CaptureRequest.CONTROL_AF_MODE, CameraMetadata.CONTROL_AF_MODE_CONTINUOUS_PICTURE);
            }

            b1.set(CaptureRequest.NOISE_REDUCTION_MODE, mCaptureNoiseMode);
            b1.set(CaptureRequest.EDGE_MODE, mCaptureEdgeMode);
            b1.set(CaptureRequest.STATISTICS_FACE_DETECT_MODE, mCaptureFace ? mCameraInfoCache.bestFaceDetectionMode() : CaptureRequest.STATISTICS_FACE_DETECT_MODE_OFF);

            Log.v(TAG, "  .. NR=" + mCaptureNoiseMode + "  Edge=" + mCaptureEdgeMode + "  Face=" + mCaptureFace);

            if (mCaptureYuv1) {
                b1.addTarget(mYuv1ImageReader.getSurface());
                Log.v(TAG, "  .. YUV1 on");
            }

            if (mCaptureRaw) {
                b1.addTarget(mRawImageReader.getSurface());
            }

            b1.addTarget(mPreviewSurface);

            if (mCaptureYuv2) {
                if (SECOND_SURFACE_TEXTURE_STREAM) {
                    b1.addTarget(mSurfaceTextureSurface);
                }
                if (SECOND_YUV_IMAGEREADER_STREAM) {
                    b1.addTarget(mYuv2ImageReader.getSurface());
                }
                Log.v(TAG, "  .. YUV2 on");
            }

            if (AFtrigger) {
                b1.set(CaptureRequest.CONTROL_AF_TRIGGER, CameraMetadata.CONTROL_AF_TRIGGER_START);
                mCurrentCaptureSession.capture(b1.build(), mCaptureCallback, mOpsHandler);
                b1.set(CaptureRequest.CONTROL_AF_TRIGGER, CameraMetadata.CONTROL_AF_TRIGGER_IDLE);
            }
            mCurrentCaptureSession.setRepeatingRequest(b1.build(), mCaptureCallback, mOpsHandler);
        } catch (CameraAccessException e) {
            Log.e(TAG, "Could not access camera for issuePreviewCaptureRequest.");
        }
    }

    void runReprocessing() {
        if (mYuv1LastReceivedImage == null) {
            Log.e(TAG, "No YUV Image available.");
            return;
        }
        mImageWriter.queueInputImage(mYuv1LastReceivedImage);
        Log.v(TAG, "  Sent YUV1 image to ImageWriter.queueInputImage()");
        try {
            CaptureRequest.Builder b1 = mCameraDevice.createReprocessCaptureRequest(mLastTotalCaptureResult);
            // Todo: Read current orientation instead of just assuming device is in native orientation
            b1.set(CaptureRequest.JPEG_ORIENTATION, mCameraInfoCache.sensorOrientation());
            b1.set(CaptureRequest.JPEG_QUALITY, (byte) 95);
            b1.set(CaptureRequest.NOISE_REDUCTION_MODE, mReprocessingNoiseMode);
            b1.set(CaptureRequest.EDGE_MODE, mReprocessingEdgeMode);
            b1.addTarget(mJpegImageReader.getSurface());
            mCurrentCaptureSession.capture(b1.build(), mReprocessingCaptureCallback, mOpsHandler);
            mReprocessingRequestNanoTime = System.nanoTime();
        } catch (CameraAccessException e) {
            Log.e(TAG, "Could not access camera for issuePreviewCaptureRequest.");
        }
        mYuv1LastReceivedImage = null;
        Log.v(TAG, "  Reprocessing request submitted.");
    }


    /*********************************
     * onImageAvailable() processing *
     *********************************/

    ImageReader.OnImageAvailableListener mYuv1ImageListener =
            new ImageReader.OnImageAvailableListener() {
                @Override
                public void onImageAvailable(ImageReader reader) {
                    Image img = reader.acquireLatestImage();
                    if (img == null) {
                        Log.e(TAG, "Null image returned YUV1");
                        return;
                    }
                    if (mYuv1LastReceivedImage != null) {
                        mYuv1LastReceivedImage.close();
                    }
                    mYuv1LastReceivedImage = img;
                    if (++mYuv1ImageCounter % LOG_NTH_FRAME == 0) {
                        Log.v(TAG, "YUV1 buffer available, Frame #=" + mYuv1ImageCounter + " w=" + img.getWidth() + " h=" + img.getHeight() + " time=" + img.getTimestamp());
                    }

                }
            };


    ImageReader.OnImageAvailableListener mJpegImageListener =
            new ImageReader.OnImageAvailableListener() {
                @Override
                public void onImageAvailable(ImageReader reader) {
                    Image img = reader.acquireLatestImage();
                    if (img == null) {
                        Log.e(TAG, "Null image returned JPEG");
                        return;
                    }
                    Image.Plane plane0 = img.getPlanes()[0];
                    final ByteBuffer buffer = plane0.getBuffer();
                    long dt = System.nanoTime() - mReprocessingRequestNanoTime;
                    Log.v(TAG, String.format("JPEG buffer available, w=%d h=%d time=%d size=%d dt=%.1f ms  ISO=%d",
                            img.getWidth(), img.getHeight(), img.getTimestamp(), buffer.capacity(), 0.000001 * dt, mLastIso));
                    // Save JPEG on the utility thread,
                    final byte[] jpegBuf;
                    if (buffer.hasArray()) {
                        jpegBuf = buffer.array();
                    } else {
                        jpegBuf = new byte[buffer.capacity()];
                        buffer.get(jpegBuf);
                    }
                    mMyCameraCallback.jpegAvailable(jpegBuf, img.getWidth(), img.getHeight());
                    img.close();

                    // take (reprocess) another picture right away if bursting.
                    if (mIsBursting) {
                        takePicture();
                    }
                }
            };


    ImageReader.OnImageAvailableListener mYuv2ImageListener =
            new ImageReader.OnImageAvailableListener() {
                @Override
                public void onImageAvailable(ImageReader reader) {
                    Image img = reader.acquireLatestImage();
                    if (img == null) {
                        Log.e(TAG, "Null image returned YUV2");
                    } else {
                        if (++mYuv2ImageCounter % LOG_NTH_FRAME == 0) {
                            Log.v(TAG, "YUV2 buffer available, Frame #=" + mYuv2ImageCounter + " w=" + img.getWidth() + " h=" + img.getHeight() + " time=" + img.getTimestamp());
                        }
                        img.close();
                    }
                }
            };


    ImageReader.OnImageAvailableListener mRawImageListener =
            new ImageReader.OnImageAvailableListener() {
                @Override
                public void onImageAvailable(ImageReader reader) {
                    final Image img = reader.acquireLatestImage();
                    if (img == null) {
                        Log.e(TAG, "Null image returned RAW");
                    } else {
                        if (++mRawImageCounter % LOG_NTH_FRAME == 0) {
                            Image.Plane plane0 = img.getPlanes()[0];
                            final ByteBuffer buffer = plane0.getBuffer();
                            Log.v(TAG, "Raw buffer available, Frame #=" + mRawImageCounter + "w=" + img.getWidth()
                                    + " h=" + img.getHeight()
                                    + " format=" + CameraDeviceReport.getFormatName(img.getFormat())
                                    + " time=" + img.getTimestamp()
                                    + " size=" + buffer.capacity()
                                    + " getRowStride()=" + plane0.getRowStride());
                        }
                        img.close();
                    }
                }
            };

    /*************************************
     * CaptureResult metadata processing *
     *************************************/

    private CameraCaptureSession.CaptureCallback mCaptureCallback = new LoggingCallbacks.SessionCaptureCallback() {
        @Override
        public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, TotalCaptureResult result) {
            if (!mFirstFrameArrived) {
                mFirstFrameArrived = true;
                long now = SystemClock.elapsedRealtime();
                long dt = now - CameraTimer.t0;
                long camera_dt = now - CameraTimer.t_session_go + CameraTimer.t_open_end - CameraTimer.t_open_start;
                long repeating_req_dt = now - CameraTimer.t_burst;
                Log.v(TAG, "App control to first frame: (" + dt + " ms)");
                Log.v(TAG, "HAL request to first frame: (" + repeating_req_dt + " ms) " + " Total HAL wait: (" + camera_dt + " ms)");
                mMyCameraCallback.receivedFirstFrame();
                mMyCameraCallback.performanceDataAvailable((int) dt, (int) camera_dt, null);
            }
            publishFrameData(result);
            // Used for reprocessing.
            mLastTotalCaptureResult = result;
            super.onCaptureCompleted(session, request, result);
        }
    };

    // Reprocessing capture completed.
    private CameraCaptureSession.CaptureCallback mReprocessingCaptureCallback = new LoggingCallbacks.SessionCaptureCallback() {
        @Override
        public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, TotalCaptureResult result) {
            Log.v(TAG, "Reprocessing onCaptureCompleted()");
        }
    };

    private static double SHORT_LOG_EXPOSURE = Math.log10(1000000000 / 10000); // 1/10000 second
    private static double LONG_LOG_EXPOSURE = Math.log10(1000000000 / 10); // 1/10 second
    public int FPS_CALC_LOOKBACK = 15;
    private LinkedList<Long> mFrameTimes = new LinkedList<Long>();

    private void publishFrameData(TotalCaptureResult result) {
        // Faces.
        final Face[] faces = result.get(CaptureResult.STATISTICS_FACES);
        NormalizedFace[] newFaces = new NormalizedFace[faces.length];
        if (faces.length > 0) {
            int offX = mCameraInfoCache.faceOffsetX();
            int offY = mCameraInfoCache.faceOffsetY();
            int dX = mCameraInfoCache.activeAreaWidth() - 2 * offX;
            int dY = mCameraInfoCache.activeAreaHeight() - 2 * offY;
            if (mCameraInfoCache.IS_NEXUS_6 && mCameraIsFront) {
                // Front camera on Nexus 6 is currently 16 x 9 cropped to 4 x 3.
                // TODO: Generalize this.
                int cropOffset = dX / 8;
                dX -= 2 * cropOffset;
                offX += cropOffset;
            }
            int orientation = mCameraInfoCache.sensorOrientation();
            for (int i = 0; i < faces.length; ++i) {
                newFaces[i] = new NormalizedFace(faces[i], dX, dY, offX, offY);
                if (mCameraIsFront && orientation == 90) {
                    newFaces[i].mirrorInY();
                }
                if (mCameraIsFront && orientation == 270) {
                    newFaces[i].mirrorInX();
                }
                if (!mCameraIsFront && orientation == 270) {
                    newFaces[i].mirrorInX();
                    newFaces[i].mirrorInY();
                }
            }
        }

        // Normalized lens and exposure coordinates.
        double rm = Math.log10(result.get(CaptureResult.SENSOR_EXPOSURE_TIME));
        float normExposure = (float) ((rm - SHORT_LOG_EXPOSURE) / (LONG_LOG_EXPOSURE - SHORT_LOG_EXPOSURE));
        float normLensPos = (mCameraInfoCache.getDiopterHi() - result.get(CaptureResult.LENS_FOCUS_DISTANCE)) / (mCameraInfoCache.getDiopterHi() - mCameraInfoCache.getDiopterLow());
        mLastIso = result.get(CaptureResult.SENSOR_SENSITIVITY);

        // Update frame arrival history.
        mFrameTimes.add(result.get(CaptureResult.SENSOR_TIMESTAMP));
        if (mFrameTimes.size() > FPS_CALC_LOOKBACK) {
            mFrameTimes.removeFirst();
        }

        // Frame drop detector
        {
            float frameDuration = result.get(CaptureResult.SENSOR_FRAME_DURATION);
            if (mFrameTimes.size() > 1) {
                long dt = result.get(CaptureResult.SENSOR_TIMESTAMP) - mFrameTimes.get(mFrameTimes.size()-2);
                if (dt > 3 * frameDuration / 2 && LOG_DROPPED_FRAMES) {
                    float drops = (dt * 1f / frameDuration) - 1f;
                    Log.e(TAG, String.format("dropped %.2f frames", drops));
                    mMyCameraCallback.performanceDataAvailable(null, null, drops);
                }
            }
        }

        // FPS calc.
        float fps = 0;
        if (mFrameTimes.size() > 1) {
            long dt = mFrameTimes.getLast() - mFrameTimes.getFirst();
            fps = (mFrameTimes.size() - 1) * 1000000000f / dt;
            fps = (float) Math.floor(fps + 0.1); // round to nearest whole number, ish.
        }

        // Do callback.
        if (mMyCameraCallback != null) {
            mMyCameraCallback.frameDataAvailable(newFaces, normExposure, normLensPos, fps,
                    (int) mLastIso, result.get(CaptureResult.CONTROL_AF_STATE), result.get(CaptureResult.CONTROL_AE_STATE), result.get(CaptureResult.CONTROL_AWB_STATE));
        } else {
            Log.v(TAG, "mMyCameraCallbacks is null!!.");
        }
    }

    long mLastIso = 0;

    /*********************
     * UTILITY FUNCTIONS *
     *********************/

    /**
     * Return the next mode after currentMode in supportedModes, wrapping to
     * start of mode list if currentMode is last.  Returns currentMode if it is not found in
     * supportedModes.
     *
     * @param currentMode
     * @param supportedModes
     * @return next mode after currentMode in supportedModes
     */
    private int getNextMode(int currentMode, int[] supportedModes) {
        boolean getNext = false;
        for (int m : supportedModes) {
            if (getNext) {
                return m;
            }
            if (m == currentMode) {
                getNext = true;
            }
        }
        if (getNext) {
            return supportedModes[0];
        }
        // Can't find mode in list
        return currentMode;
    }

    private static String edgeModeToString(int mode) {
        switch (mode) {
            case CaptureRequest.EDGE_MODE_OFF:
                return "OFF";
            case CaptureRequest.EDGE_MODE_FAST:
                return "FAST";
            case CaptureRequest.EDGE_MODE_HIGH_QUALITY:
                return "HiQ";
            case CaptureRequest.EDGE_MODE_ZERO_SHUTTER_LAG:
                return "ZSL";
        }
        return Integer.toString(mode);
    }

    private static String noiseModeToString(int mode) {
        switch (mode) {
            case CaptureRequest.NOISE_REDUCTION_MODE_OFF:
                return "OFF";
            case CaptureRequest.NOISE_REDUCTION_MODE_FAST:
                return "FAST";
            case CaptureRequest.NOISE_REDUCTION_MODE_HIGH_QUALITY:
                return "HiQ";
            case CaptureRequest.NOISE_REDUCTION_MODE_MINIMAL:
                return "MIN";
            case CaptureRequest.NOISE_REDUCTION_MODE_ZERO_SHUTTER_LAG:
                return "ZSL";
        }
        return Integer.toString(mode);
    }
}