one/v2/ImageCaptureManager.java

/*
 * Copyright (C) 2014 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.camera.one.v2;

import android.annotation.TargetApi;
import android.hardware.camera2.CameraCaptureSession;
import android.hardware.camera2.CaptureRequest;
import android.hardware.camera2.CaptureResult;
import android.hardware.camera2.CaptureResult.Key;
import android.hardware.camera2.TotalCaptureResult;
import android.media.Image;
import android.media.ImageReader;
import android.os.Build;
import android.os.Handler;
import android.os.SystemClock;
import android.util.Pair;

import com.android.camera.debug.Log;
import com.android.camera.debug.Log.Tag;
import com.android.camera.util.ConcurrentSharedRingBuffer;
import com.android.camera.util.ConcurrentSharedRingBuffer.PinStateListener;
import com.android.camera.util.ConcurrentSharedRingBuffer.Selector;
import com.android.camera.util.ConcurrentSharedRingBuffer.SwapTask;
import com.android.camera.util.Task;

import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * Implements {@link android.media.ImageReader.OnImageAvailableListener} and
 * {@link android.hardware.camera2.CameraCaptureSession.CaptureListener} to
 * store the results of capture requests (both {@link Image}s and
 * {@link TotalCaptureResult}s in a ring-buffer from which they may be saved.
 * <br>
 * This also manages the lifecycle of {@link Image}s within the application as
 * they are passed in from the lower-level camera2 API.
 */
@TargetApi(Build.VERSION_CODES.LOLLIPOP)
public class ImageCaptureManager extends CameraCaptureSession.CaptureCallback implements
        ImageReader.OnImageAvailableListener {
    /**
     * Callback to listen for changes to the ability to capture an existing
     * image from the internal ring-buffer.
     */
    public interface CaptureReadyListener {
        /**
         * Called whenever the ability to capture an existing image from the
         * ring-buffer changes. Calls to {@link #tryCaptureExistingImage} are
         * more likely to succeed or fail depending on the value passed in to
         * this function.
         *
         * @param capturePossible true if capture is more-likely to be possible,
         *            false if capture is less-likely to be possible.
         */
        public void onReadyStateChange(boolean capturePossible);
    }

    /**
     * Callback for listening to changes to individual metadata values.
     */
    public static interface MetadataChangeListener {
        /**
         * This will be called whenever a metadata value changes.
         * Implementations should not take too much time to execute since this
         * will be called faster than the camera's frame rate.
         *
         * @param key the {@link CaptureResult} key this listener listens for.
         * @param second the previous value, or null if no such value existed.
         *            The type will be that associated with the
         *            {@link android.hardware.camera2.CaptureResult.Key} this
         *            listener is bound to.
         * @param newValue the new value. The type will be that associated with
         *            the {@link android.hardware.camera2.CaptureResult.Key}
         *            this listener is bound to.
         * @param result the CaptureResult containing the new value
         */
        public void onImageMetadataChange(Key<?> key, Object second, Object newValue,
                CaptureResult result);
    }

    /**
     * Callback for saving an image.
     */
    public interface ImageCaptureListener {
         /**
         * Called with the {@link Image} and associated
         * {@link TotalCaptureResult}. A typical implementation would save this
         * to disk.
         * <p>
         * Note: Implementations must be thread-safe and must not close the
         * image.
         * </p>
         */
        public void onImageCaptured(Image image, TotalCaptureResult captureResult);
    }

    /**
     * Callback for placing constraints on which images to capture. See
     * {@link #tryCaptureExistingImage} and {@link #captureNextImage}.
     */
    public static interface CapturedImageConstraint {
        /**
         * Implementations should return true if the provided
         * TotalCaptureResults satisfies constraints necessary for the intended
         * image capture. For example, a constraint may return false if
         * {@captureResult} indicates that the lens was moving during image
         * capture.
         *
         * @param captureResult The metadata associated with the image.
         * @return true if this image satisfies the constraint and can be
         *         captured, false otherwise.
         */
        boolean satisfiesConstraint(TotalCaptureResult captureResult);
    }

    /**
     * Holds an {@link Image} and {@link TotalCaptureResult} pair which may be
     * added asynchronously.
     */
    private class CapturedImage {
        /**
         * The Image and TotalCaptureResult may be received at different times
         * (via the onImageAvailableListener and onCaptureProgressed callbacks,
         * respectively).
         */
        private Image mImage = null;
        private TotalCaptureResult mMetadata = null;

        /**
         * Resets the object, closing and removing any existing image and
         * metadata.
         */
        public void reset() {
            if (mImage != null) {
                mImage.close();
                int numOpenImages = mNumOpenImages.decrementAndGet();
                if (DEBUG_PRINT_OPEN_IMAGE_COUNT) {
                    Log.v(TAG, "Closed an image. Number of open images = " + numOpenImages);
                }
            }

            mImage = null;

            mMetadata = null;
        }

        /**
         * @return true if both the image and metadata are present, false
         *         otherwise.
         */
        public boolean isComplete() {
            return mImage != null && mMetadata != null;
        }

        /**
         * Adds the image. Note that this can only be called once before a
         * {@link #reset()} is necessary.
         *
         * @param image the {@Link Image} to add.
         */
        public void addImage(Image image) {
            if (mImage != null) {
                throw new IllegalArgumentException(
                        "Unable to add an Image when one already exists.");
            }
            mImage = image;
        }

        /**
         * Retrieves the {@link Image} if it has been added, returns null if it
         * is not available yet.
         */
        public Image tryGetImage() {
            return mImage;
        }

        /**
         * Adds the metadata. Note that this can only be called once before a
         * {@link #reset()} is necessary.
         *
         * @param metadata the {@Link TotalCaptureResult} to add.
         */
        public void addMetadata(TotalCaptureResult metadata) {
            if (mMetadata != null) {
                throw new IllegalArgumentException(
                        "Unable to add a TotalCaptureResult when one already exists.");
            }
            mMetadata = metadata;
        }

        /**
         * Retrieves the {@link TotalCaptureResult} if it has been added,
         * returns null if it is not available yet.
         */
        public TotalCaptureResult tryGetMetadata() {
            return mMetadata;
        }
    }

    private static final Tag TAG = new Tag("ZSLImageListener");

    /**
     * If true, the number of open images will be printed to LogCat every time
     * an image is opened or closed.
     */
    private static final boolean DEBUG_PRINT_OPEN_IMAGE_COUNT = false;

    /**
     * The maximum duration for an onImageAvailable() callback before debugging
     * output is printed. This is a little under 1/30th of a second to enable
     * detecting jank in the preview stream caused by {@link #onImageAvailable}
     * taking too long to return.
     */
    private static final long DEBUG_MAX_IMAGE_CALLBACK_DUR = 25;

    /**
     * If spacing between onCaptureCompleted() callbacks is lower than this
     * value, camera operations at the Java level have stalled, and are now
     * catching up. In milliseconds.
     */
    private static final long DEBUG_INTERFRAME_STALL_WARNING = 5;

    /**
     * Last called to onCaptureCompleted() in SystemClock.uptimeMillis().
     */
    private long mDebugLastOnCaptureCompletedMillis = 0;

    /**
     * Number of frames in a row exceeding DEBUG_INTERFRAME_STALL_WARNING.
     */
    private long mDebugStalledFrameCount = 0;

    /**
     * Stores the ring-buffer of captured images.<br>
     * Note that this takes care of thread-safe reference counting of images to
     * ensure that they are never leaked by the app.
     */
    private final ConcurrentSharedRingBuffer<CapturedImage> mCapturedImageBuffer;

    /** Track the number of open images for debugging purposes. */
    private final AtomicInteger mNumOpenImages = new AtomicInteger(0);

    /**
     * The handler used to invoke light-weight listeners:
     * {@link CaptureReadyListener} and {@link MetadataChangeListener}.
     */
    private final Handler mListenerHandler;

    /**
     * The executor used to invoke {@link ImageCaptureListener}. Note that this
     * is different from mListenerHandler because a typical ImageCaptureListener
     * will compress the image to jpeg, and we may wish to execute these tasks
     * on multiple threads.
     */
    private final Executor mImageCaptureListenerExecutor;

    /**
     * The set of constraints which must be satisfied for a newly acquired image
     * to be captured and sent to {@link #mPendingImageCaptureCallback}. null if
     * there is no pending capture request.
     */
    private List<ImageCaptureManager.CapturedImageConstraint> mPendingImageCaptureConstraints;

    /**
     * The callback to be invoked upon successfully capturing a newly-acquired
     * image which satisfies {@link #mPendingImageCaptureConstraints}. null if
     * there is no pending capture request.
     */
    private ImageCaptureManager.ImageCaptureListener mPendingImageCaptureCallback;

    /**
     * Map from CaptureResult key to the frame number of the capture result
     * containing the most recent value for this key and the most recent value
     * of the key.
     */
    private final Map<Key<?>, Pair<Long, Object>>
            mMetadata = new ConcurrentHashMap<CaptureResult.Key<?>, Pair<Long, Object>>();

    /**
     * The set of callbacks to be invoked when an entry in {@link #mMetadata} is
     * changed.
     */
    private final Map<Key<?>, Set<MetadataChangeListener>>
            mMetadataChangeListeners = new ConcurrentHashMap<Key<?>, Set<MetadataChangeListener>>();

    /**
     * @param maxImages the maximum number of images provided by the
     *            {@link ImageReader}. This must be greater than 2.
     * @param listenerHandler the handler on which to invoke listeners. Note
     *            that this should probably be on a different thread than the
     *            one used for camera operations, such as capture requests and
     *            OnImageAvailable listeners, to avoid stalling the preview.
     * @param imageCaptureListenerExecutor the executor on which to invoke image
     *            capture listeners, {@link ImageCaptureListener}.
     */
    ImageCaptureManager(int maxImages, Handler listenerHandler,
            Executor imageCaptureListenerExecutor) {
        // Ensure that there are always 2 images available for the framework to
        // continue processing frames.
        // TODO Could we make this tighter?
        mCapturedImageBuffer = new ConcurrentSharedRingBuffer<ImageCaptureManager.CapturedImage>(
                maxImages - 2);

        mListenerHandler = listenerHandler;
        mImageCaptureListenerExecutor = imageCaptureListenerExecutor;
    }

    /**
     * See {@link CaptureReadyListener}.
     */
    public void setCaptureReadyListener(final CaptureReadyListener listener) {
        mCapturedImageBuffer.setListener(mListenerHandler,
                new PinStateListener() {
                @Override
                    public void onPinStateChange(boolean pinsAvailable) {
                        listener.onReadyStateChange(pinsAvailable);
                    }
                });
    }

    /**
     * Adds a metadata stream listener associated with the given key.
     *
     * @param key the key of the metadata to track.
     * @param listener the listener to be invoked when the value associated with
     *            key changes.
     */
    public <T> void addMetadataChangeListener(Key<T> key, MetadataChangeListener listener) {
        if (!mMetadataChangeListeners.containsKey(key)) {
            // Listeners may be added to this set from a different thread than
            // that which must iterate over this set to invoke the listeners.
            // Therefore, we need a thread save hash set.
            mMetadataChangeListeners.put(key,
                    Collections.newSetFromMap(new ConcurrentHashMap<
                            ImageCaptureManager.MetadataChangeListener, Boolean>()));
        }
        mMetadataChangeListeners.get(key).add(listener);
    }

    /**
     * Removes the metadata stream listener associated with the given key.
     *
     * @param key the key associated with the metadata to track.
     * @param listener the listener to be invoked when the value associated with
     *            key changes.
     * @return true if the listener was removed, false if no such listener had
     *         been added.
     */
    public <T> boolean removeMetadataChangeListener(Key<T> key, MetadataChangeListener listener) {
        if (!mMetadataChangeListeners.containsKey(key)) {
            return false;
        } else {
            return mMetadataChangeListeners.get(key).remove(listener);
        }
    }

    @Override
    public void onCaptureProgressed(CameraCaptureSession session, CaptureRequest request,
            final CaptureResult partialResult) {
        long frameNumber = partialResult.getFrameNumber();

        // Update mMetadata for whichever keys are present, if this frame is
        // supplying newer values.
        for (final Key<?> key : partialResult.getKeys()) {
            Pair<Long, Object> oldEntry = mMetadata.get(key);
            final Object oldValue = (oldEntry != null) ? oldEntry.second : null;

            boolean newerValueAlreadyExists = oldEntry != null
                    && frameNumber < oldEntry.first;
            if (newerValueAlreadyExists) {
                continue;
            }

            final Object newValue = partialResult.get(key);
            mMetadata.put(key, new Pair<Long, Object>(frameNumber, newValue));

            // If the value has changed, call the appropriate listeners, if
            // any exist.
            if (oldValue == newValue || !mMetadataChangeListeners.containsKey(key)) {
                continue;
            }

            for (final MetadataChangeListener listener :
                    mMetadataChangeListeners.get(key)) {
                Log.v(TAG, "Dispatching to metadata change listener for key: "
                        + key.toString());
                mListenerHandler.post(new Runnable() {
                        @Override
                    public void run() {
                        listener.onImageMetadataChange(key, oldValue, newValue,
                                partialResult);
                    }
                });
            }
        }
    }

    @Override
    public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request,
            final TotalCaptureResult result) {
        final long timestamp = result.get(TotalCaptureResult.SENSOR_TIMESTAMP);

        // Detect camera thread stall.
        long now = SystemClock.uptimeMillis();
        if (now - mDebugLastOnCaptureCompletedMillis < DEBUG_INTERFRAME_STALL_WARNING) {
            Log.e(TAG, "Camera thread has stalled for " + ++mDebugStalledFrameCount +
                    " frames at # " + result.getFrameNumber() + ".");
        } else {
            mDebugStalledFrameCount = 0;
        }
        mDebugLastOnCaptureCompletedMillis = now;

        // Find the CapturedImage in the ring-buffer and attach the
        // TotalCaptureResult to it.
        // See documentation for swapLeast() for details.
        boolean swapSuccess = mCapturedImageBuffer.swapLeast(timestamp,
                new SwapTask<CapturedImage>() {
                @Override
                    public CapturedImage create() {
                        CapturedImage image = new CapturedImage();
                        image.addMetadata(result);
                        return image;
                    }

                @Override
                    public CapturedImage swap(CapturedImage oldElement) {
                        oldElement.reset();
                        oldElement.addMetadata(result);
                        return oldElement;
                    }

                @Override
                    public void update(CapturedImage existingElement) {
                        existingElement.addMetadata(result);
                    }
                });

        if (!swapSuccess) {
            // Do nothing on failure to swap in.
            Log.v(TAG, "Unable to add new image metadata to ring-buffer.");
        }

        tryExecutePendingCaptureRequest(timestamp);
    }

    @Override
    public void onImageAvailable(ImageReader reader) {
        long startTime = SystemClock.currentThreadTimeMillis();

        final Image img = reader.acquireLatestImage();

        if (img != null) {
            int numOpenImages = mNumOpenImages.incrementAndGet();
            if (DEBUG_PRINT_OPEN_IMAGE_COUNT) {
                Log.v(TAG, "Acquired an image. Number of open images = " + numOpenImages);
            }

            // Try to place the newly-acquired image into the ring buffer.
            boolean swapSuccess = mCapturedImageBuffer.swapLeast(
                    img.getTimestamp(), new SwapTask<CapturedImage>() {
                            @Override
                        public CapturedImage create() {
                            CapturedImage image = new CapturedImage();
                            image.addImage(img);
                            return image;
                        }

                            @Override
                        public CapturedImage swap(CapturedImage oldElement) {
                            oldElement.reset();
                            oldElement.addImage(img);
                            return oldElement;
                        }

                            @Override
                        public void update(CapturedImage existingElement) {
                            existingElement.addImage(img);
                        }
                    });

            if (!swapSuccess) {
                // If we were unable to save the image to the ring buffer, we
                // must close it now.
                // We should only get here if the ring buffer is closed.
                img.close();
                numOpenImages = mNumOpenImages.decrementAndGet();
                if (DEBUG_PRINT_OPEN_IMAGE_COUNT) {
                    Log.v(TAG, "Closed an image. Number of open images = " + numOpenImages);
                }
            }

            tryExecutePendingCaptureRequest(img.getTimestamp());

            long endTime = SystemClock.currentThreadTimeMillis();
            long totTime = endTime - startTime;
            if (totTime > DEBUG_MAX_IMAGE_CALLBACK_DUR) {
                // If it takes too long to swap elements, we will start skipping
                // preview frames, resulting in visible jank.
                Log.v(TAG, "onImageAvailable() took " + totTime + "ms");
            }
        }
    }

    /**
     * Closes the listener, eventually freeing all currently-held {@link Image}
     * s.
     */
    public void close() {
        try {
            mCapturedImageBuffer.close(new Task<CapturedImage>() {
                    @Override
                public void run(CapturedImage e) {
                    e.reset();
                }
            });
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

    /**
     * Sets the pending image capture request, overriding any previous calls to
     * {@link #captureNextImage} which have not yet been resolved. When the next
     * available image which satisfies the given constraints can be captured,
     * onImageCaptured will be invoked.
     *
     * @param onImageCaptured the callback which will be invoked with the
     *            captured image.
     * @param constraints the set of constraints which must be satisfied in
     *            order for the image to be captured.
     */
    public void captureNextImage(final ImageCaptureListener onImageCaptured,
            final List<CapturedImageConstraint> constraints) {
        mPendingImageCaptureCallback = onImageCaptured;
        mPendingImageCaptureConstraints = constraints;
    }

    /**
     * Tries to resolve any pending image capture requests.
     *
     * @param newImageTimestamp the timestamp of a newly-acquired image which
     *            should be captured if appropriate and possible.
     */
    private void tryExecutePendingCaptureRequest(long newImageTimestamp) {
        if (mPendingImageCaptureCallback != null) {
            final Pair<Long, CapturedImage> pinnedImage = mCapturedImageBuffer.tryPin(
                    newImageTimestamp);
            if (pinnedImage != null) {
                CapturedImage image = pinnedImage.second;

                if (!image.isComplete()) {
                    mCapturedImageBuffer.release(pinnedImage.first);
                    return;
                }

                // Check to see if the image satisfies all constraints.
                TotalCaptureResult captureResult = image.tryGetMetadata();

                if (mPendingImageCaptureConstraints != null) {
                    for (CapturedImageConstraint constraint : mPendingImageCaptureConstraints) {
                        if (!constraint.satisfiesConstraint(captureResult)) {
                            mCapturedImageBuffer.release(pinnedImage.first);
                            return;
                        }
                    }
                }

                // If we get here, the image satisfies all the necessary
                // constraints.

                if (tryExecuteCaptureOrRelease(pinnedImage, mPendingImageCaptureCallback)) {
                    // If we successfully handed the image off to the callback,
                    // remove the pending
                    // capture request.
                    mPendingImageCaptureCallback = null;
                    mPendingImageCaptureConstraints = null;
                }
            }
        }
    }

    /**
     * Tries to capture an existing image from the ring-buffer, if one exists
     * that satisfies the given constraint and can be pinned.
     *
     * @return true if the image could be captured, false otherwise.
     */
    public boolean tryCaptureExistingImage(final ImageCaptureListener onImageCaptured,
            final List<CapturedImageConstraint> constraints) {
        // The selector to use in choosing the image to capture.
        Selector<ImageCaptureManager.CapturedImage> selector;

        if (constraints == null || constraints.isEmpty()) {
            // If there are no constraints, use a trivial Selector.
            selector = new Selector<ImageCaptureManager.CapturedImage>() {
                    @Override
                public boolean select(CapturedImage image) {
                    return true;
                }
            };
        } else {
            // If there are constraints, create a Selector which will return
            // true if all constraints
            // are satisfied.
            selector = new Selector<ImageCaptureManager.CapturedImage>() {
                    @Override
                public boolean select(CapturedImage e) {
                    // If this image already has metadata associated with it,
                    // then use it.
                    // Otherwise, we can't block until it's available, so assume
                    // it doesn't
                    // satisfy the required constraints.
                    TotalCaptureResult captureResult = e.tryGetMetadata();

                    if (captureResult == null || e.tryGetImage() == null) {
                        return false;
                    }

                    for (CapturedImageConstraint constraint : constraints) {
                        if (!constraint.satisfiesConstraint(captureResult)) {
                            return false;
                        }
                    }
                    return true;
                }
            };
        }

        // Acquire a lock (pin) on the most recent (greatest-timestamp) image in
        // the ring buffer which satisfies our constraints.
        // Note that this must be released as soon as we are done with it.
        final Pair<Long, CapturedImage> toCapture = mCapturedImageBuffer.tryPinGreatestSelected(
                selector);

        return tryExecuteCaptureOrRelease(toCapture, onImageCaptured);
    }

    /**
     * Tries to execute the image capture callback with the pinned CapturedImage
     * provided.
     *
     * @param toCapture The pinned CapturedImage to pass to the callback, or
     *            release on failure.
     * @param callback The callback to execute.
     * @return true upon success, false upon failure and the release of the
     *         pinned image.
     */
    private boolean tryExecuteCaptureOrRelease(final Pair<Long, CapturedImage> toCapture,
            final ImageCaptureListener callback) {
        if (toCapture == null) {
            return false;
        } else {
            try {
                mImageCaptureListenerExecutor.execute(new Runnable() {
                        @Override
                    public void run() {
                        try {
                            CapturedImage img = toCapture.second;
                            callback.onImageCaptured(img.tryGetImage(),
                                    img.tryGetMetadata());
                        } finally {
                            mCapturedImageBuffer.release(toCapture.first);
                        }
                    }
                });
            } catch (RejectedExecutionException e) {
                // We may get here if the thread pool has been closed.
                mCapturedImageBuffer.release(toCapture.first);
                return false;
            }

            return true;
        }
    }
}