1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.camera.one.v2;
18
19import android.annotation.TargetApi;
20import android.hardware.camera2.CameraCaptureSession;
21import android.hardware.camera2.CaptureRequest;
22import android.hardware.camera2.CaptureResult;
23import android.hardware.camera2.CaptureResult.Key;
24import android.hardware.camera2.TotalCaptureResult;
25import android.media.Image;
26import android.media.ImageReader;
27import android.os.Build;
28import android.os.Handler;
29import android.os.SystemClock;
30import android.util.Pair;
31
32import com.android.camera.debug.Log;
33import com.android.camera.debug.Log.Tag;
34import com.android.camera.util.ConcurrentSharedRingBuffer;
35import com.android.camera.util.ConcurrentSharedRingBuffer.PinStateListener;
36import com.android.camera.util.ConcurrentSharedRingBuffer.Selector;
37import com.android.camera.util.ConcurrentSharedRingBuffer.SwapTask;
38import com.android.camera.util.Task;
39
40import java.util.Collections;
41import java.util.List;
42import java.util.Map;
43import java.util.Set;
44import java.util.concurrent.ConcurrentHashMap;
45import java.util.concurrent.Executor;
46import java.util.concurrent.RejectedExecutionException;
47import java.util.concurrent.atomic.AtomicInteger;
48
49/**
50 * Implements {@link android.media.ImageReader.OnImageAvailableListener} and
51 * {@link android.hardware.camera2.CameraCaptureSession.CaptureListener} to
52 * store the results of capture requests (both {@link Image}s and
53 * {@link TotalCaptureResult}s in a ring-buffer from which they may be saved.
54 * <br>
55 * This also manages the lifecycle of {@link Image}s within the application as
56 * they are passed in from the lower-level camera2 API.
57 */
58@TargetApi(Build.VERSION_CODES.LOLLIPOP)
59public class ImageCaptureManager extends CameraCaptureSession.CaptureCallback implements
60        ImageReader.OnImageAvailableListener {
61    /**
62     * Callback to listen for changes to the ability to capture an existing
63     * image from the internal ring-buffer.
64     */
65    public interface CaptureReadyListener {
66        /**
67         * Called whenever the ability to capture an existing image from the
68         * ring-buffer changes. Calls to {@link #tryCaptureExistingImage} are
69         * more likely to succeed or fail depending on the value passed in to
70         * this function.
71         *
72         * @param capturePossible true if capture is more-likely to be possible,
73         *            false if capture is less-likely to be possible.
74         */
75        public void onReadyStateChange(boolean capturePossible);
76    }
77
78    /**
79     * Callback for listening to changes to individual metadata values.
80     */
81    public static interface MetadataChangeListener {
82        /**
83         * This will be called whenever a metadata value changes.
84         * Implementations should not take too much time to execute since this
85         * will be called faster than the camera's frame rate.
86         *
87         * @param key the {@link CaptureResult} key this listener listens for.
88         * @param second the previous value, or null if no such value existed.
89         *            The type will be that associated with the
90         *            {@link android.hardware.camera2.CaptureResult.Key} this
91         *            listener is bound to.
92         * @param newValue the new value. The type will be that associated with
93         *            the {@link android.hardware.camera2.CaptureResult.Key}
94         *            this listener is bound to.
95         * @param result the CaptureResult containing the new value
96         */
97        public void onImageMetadataChange(Key<?> key, Object second, Object newValue,
98                CaptureResult result);
99    }
100
101    /**
102     * Callback for saving an image.
103     */
104    public interface ImageCaptureListener {
105         /**
106         * Called with the {@link Image} and associated
107         * {@link TotalCaptureResult}. A typical implementation would save this
108         * to disk.
109         * <p>
110         * Note: Implementations must be thread-safe and must not close the
111         * image.
112         * </p>
113         */
114        public void onImageCaptured(Image image, TotalCaptureResult captureResult);
115    }
116
117    /**
118     * Callback for placing constraints on which images to capture. See
119     * {@link #tryCaptureExistingImage} and {@link #captureNextImage}.
120     */
121    public static interface CapturedImageConstraint {
122        /**
123         * Implementations should return true if the provided
124         * TotalCaptureResults satisfies constraints necessary for the intended
125         * image capture. For example, a constraint may return false if
126         * {@captureResult} indicates that the lens was moving during image
127         * capture.
128         *
129         * @param captureResult The metadata associated with the image.
130         * @return true if this image satisfies the constraint and can be
131         *         captured, false otherwise.
132         */
133        boolean satisfiesConstraint(TotalCaptureResult captureResult);
134    }
135
136    /**
137     * Holds an {@link Image} and {@link TotalCaptureResult} pair which may be
138     * added asynchronously.
139     */
140    private class CapturedImage {
141        /**
142         * The Image and TotalCaptureResult may be received at different times
143         * (via the onImageAvailableListener and onCaptureProgressed callbacks,
144         * respectively).
145         */
146        private Image mImage = null;
147        private TotalCaptureResult mMetadata = null;
148
149        /**
150         * Resets the object, closing and removing any existing image and
151         * metadata.
152         */
153        public void reset() {
154            if (mImage != null) {
155                mImage.close();
156                int numOpenImages = mNumOpenImages.decrementAndGet();
157                if (DEBUG_PRINT_OPEN_IMAGE_COUNT) {
158                    Log.v(TAG, "Closed an image. Number of open images = " + numOpenImages);
159                }
160            }
161
162            mImage = null;
163
164            mMetadata = null;
165        }
166
167        /**
168         * @return true if both the image and metadata are present, false
169         *         otherwise.
170         */
171        public boolean isComplete() {
172            return mImage != null && mMetadata != null;
173        }
174
175        /**
176         * Adds the image. Note that this can only be called once before a
177         * {@link #reset()} is necessary.
178         *
179         * @param image the {@Link Image} to add.
180         */
181        public void addImage(Image image) {
182            if (mImage != null) {
183                throw new IllegalArgumentException(
184                        "Unable to add an Image when one already exists.");
185            }
186            mImage = image;
187        }
188
189        /**
190         * Retrieves the {@link Image} if it has been added, returns null if it
191         * is not available yet.
192         */
193        public Image tryGetImage() {
194            return mImage;
195        }
196
197        /**
198         * Adds the metadata. Note that this can only be called once before a
199         * {@link #reset()} is necessary.
200         *
201         * @param metadata the {@Link TotalCaptureResult} to add.
202         */
203        public void addMetadata(TotalCaptureResult metadata) {
204            if (mMetadata != null) {
205                throw new IllegalArgumentException(
206                        "Unable to add a TotalCaptureResult when one already exists.");
207            }
208            mMetadata = metadata;
209        }
210
211        /**
212         * Retrieves the {@link TotalCaptureResult} if it has been added,
213         * returns null if it is not available yet.
214         */
215        public TotalCaptureResult tryGetMetadata() {
216            return mMetadata;
217        }
218    }
219
220    private static final Tag TAG = new Tag("ZSLImageListener");
221
222    /**
223     * If true, the number of open images will be printed to LogCat every time
224     * an image is opened or closed.
225     */
226    private static final boolean DEBUG_PRINT_OPEN_IMAGE_COUNT = false;
227
228    /**
229     * The maximum duration for an onImageAvailable() callback before debugging
230     * output is printed. This is a little under 1/30th of a second to enable
231     * detecting jank in the preview stream caused by {@link #onImageAvailable}
232     * taking too long to return.
233     */
234    private static final long DEBUG_MAX_IMAGE_CALLBACK_DUR = 25;
235
236    /**
237     * If spacing between onCaptureCompleted() callbacks is lower than this
238     * value, camera operations at the Java level have stalled, and are now
239     * catching up. In milliseconds.
240     */
241    private static final long DEBUG_INTERFRAME_STALL_WARNING = 5;
242
243    /**
244     * Last called to onCaptureCompleted() in SystemClock.uptimeMillis().
245     */
246    private long mDebugLastOnCaptureCompletedMillis = 0;
247
248    /**
249     * Number of frames in a row exceeding DEBUG_INTERFRAME_STALL_WARNING.
250     */
251    private long mDebugStalledFrameCount = 0;
252
253    /**
254     * Stores the ring-buffer of captured images.<br>
255     * Note that this takes care of thread-safe reference counting of images to
256     * ensure that they are never leaked by the app.
257     */
258    private final ConcurrentSharedRingBuffer<CapturedImage> mCapturedImageBuffer;
259
260    /** Track the number of open images for debugging purposes. */
261    private final AtomicInteger mNumOpenImages = new AtomicInteger(0);
262
263    /**
264     * The handler used to invoke light-weight listeners:
265     * {@link CaptureReadyListener} and {@link MetadataChangeListener}.
266     */
267    private final Handler mListenerHandler;
268
269    /**
270     * The executor used to invoke {@link ImageCaptureListener}. Note that this
271     * is different from mListenerHandler because a typical ImageCaptureListener
272     * will compress the image to jpeg, and we may wish to execute these tasks
273     * on multiple threads.
274     */
275    private final Executor mImageCaptureListenerExecutor;
276
277    /**
278     * The set of constraints which must be satisfied for a newly acquired image
279     * to be captured and sent to {@link #mPendingImageCaptureCallback}. null if
280     * there is no pending capture request.
281     */
282    private List<ImageCaptureManager.CapturedImageConstraint> mPendingImageCaptureConstraints;
283
284    /**
285     * The callback to be invoked upon successfully capturing a newly-acquired
286     * image which satisfies {@link #mPendingImageCaptureConstraints}. null if
287     * there is no pending capture request.
288     */
289    private ImageCaptureManager.ImageCaptureListener mPendingImageCaptureCallback;
290
291    /**
292     * Map from CaptureResult key to the frame number of the capture result
293     * containing the most recent value for this key and the most recent value
294     * of the key.
295     */
296    private final Map<Key<?>, Pair<Long, Object>>
297            mMetadata = new ConcurrentHashMap<CaptureResult.Key<?>, Pair<Long, Object>>();
298
299    /**
300     * The set of callbacks to be invoked when an entry in {@link #mMetadata} is
301     * changed.
302     */
303    private final Map<Key<?>, Set<MetadataChangeListener>>
304            mMetadataChangeListeners = new ConcurrentHashMap<Key<?>, Set<MetadataChangeListener>>();
305
306    /**
307     * @param maxImages the maximum number of images provided by the
308     *            {@link ImageReader}. This must be greater than 2.
309     * @param listenerHandler the handler on which to invoke listeners. Note
310     *            that this should probably be on a different thread than the
311     *            one used for camera operations, such as capture requests and
312     *            OnImageAvailable listeners, to avoid stalling the preview.
313     * @param imageCaptureListenerExecutor the executor on which to invoke image
314     *            capture listeners, {@link ImageCaptureListener}.
315     */
316    ImageCaptureManager(int maxImages, Handler listenerHandler,
317            Executor imageCaptureListenerExecutor) {
318        // Ensure that there are always 2 images available for the framework to
319        // continue processing frames.
320        // TODO Could we make this tighter?
321        mCapturedImageBuffer = new ConcurrentSharedRingBuffer<ImageCaptureManager.CapturedImage>(
322                maxImages - 2);
323
324        mListenerHandler = listenerHandler;
325        mImageCaptureListenerExecutor = imageCaptureListenerExecutor;
326    }
327
328    /**
329     * See {@link CaptureReadyListener}.
330     */
331    public void setCaptureReadyListener(final CaptureReadyListener listener) {
332        mCapturedImageBuffer.setListener(mListenerHandler,
333                new PinStateListener() {
334                @Override
335                    public void onPinStateChange(boolean pinsAvailable) {
336                        listener.onReadyStateChange(pinsAvailable);
337                    }
338                });
339    }
340
341    /**
342     * Adds a metadata stream listener associated with the given key.
343     *
344     * @param key the key of the metadata to track.
345     * @param listener the listener to be invoked when the value associated with
346     *            key changes.
347     */
348    public <T> void addMetadataChangeListener(Key<T> key, MetadataChangeListener listener) {
349        if (!mMetadataChangeListeners.containsKey(key)) {
350            // Listeners may be added to this set from a different thread than
351            // that which must iterate over this set to invoke the listeners.
352            // Therefore, we need a thread save hash set.
353            mMetadataChangeListeners.put(key,
354                    Collections.newSetFromMap(new ConcurrentHashMap<
355                            ImageCaptureManager.MetadataChangeListener, Boolean>()));
356        }
357        mMetadataChangeListeners.get(key).add(listener);
358    }
359
360    /**
361     * Removes the metadata stream listener associated with the given key.
362     *
363     * @param key the key associated with the metadata to track.
364     * @param listener the listener to be invoked when the value associated with
365     *            key changes.
366     * @return true if the listener was removed, false if no such listener had
367     *         been added.
368     */
369    public <T> boolean removeMetadataChangeListener(Key<T> key, MetadataChangeListener listener) {
370        if (!mMetadataChangeListeners.containsKey(key)) {
371            return false;
372        } else {
373            return mMetadataChangeListeners.get(key).remove(listener);
374        }
375    }
376
377    @Override
378    public void onCaptureProgressed(CameraCaptureSession session, CaptureRequest request,
379            final CaptureResult partialResult) {
380        long frameNumber = partialResult.getFrameNumber();
381
382        // Update mMetadata for whichever keys are present, if this frame is
383        // supplying newer values.
384        for (final Key<?> key : partialResult.getKeys()) {
385            Pair<Long, Object> oldEntry = mMetadata.get(key);
386            final Object oldValue = (oldEntry != null) ? oldEntry.second : null;
387
388            boolean newerValueAlreadyExists = oldEntry != null
389                    && frameNumber < oldEntry.first;
390            if (newerValueAlreadyExists) {
391                continue;
392            }
393
394            final Object newValue = partialResult.get(key);
395            mMetadata.put(key, new Pair<Long, Object>(frameNumber, newValue));
396
397            // If the value has changed, call the appropriate listeners, if
398            // any exist.
399            if (oldValue == newValue || !mMetadataChangeListeners.containsKey(key)) {
400                continue;
401            }
402
403            for (final MetadataChangeListener listener :
404                    mMetadataChangeListeners.get(key)) {
405                Log.v(TAG, "Dispatching to metadata change listener for key: "
406                        + key.toString());
407                mListenerHandler.post(new Runnable() {
408                        @Override
409                    public void run() {
410                        listener.onImageMetadataChange(key, oldValue, newValue,
411                                partialResult);
412                    }
413                });
414            }
415        }
416    }
417
418    @Override
419    public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request,
420            final TotalCaptureResult result) {
421        final long timestamp = result.get(TotalCaptureResult.SENSOR_TIMESTAMP);
422
423        // Detect camera thread stall.
424        long now = SystemClock.uptimeMillis();
425        if (now - mDebugLastOnCaptureCompletedMillis < DEBUG_INTERFRAME_STALL_WARNING) {
426            Log.e(TAG, "Camera thread has stalled for " + ++mDebugStalledFrameCount +
427                    " frames at # " + result.getFrameNumber() + ".");
428        } else {
429            mDebugStalledFrameCount = 0;
430        }
431        mDebugLastOnCaptureCompletedMillis = now;
432
433        // Find the CapturedImage in the ring-buffer and attach the
434        // TotalCaptureResult to it.
435        // See documentation for swapLeast() for details.
436        boolean swapSuccess = mCapturedImageBuffer.swapLeast(timestamp,
437                new SwapTask<CapturedImage>() {
438                @Override
439                    public CapturedImage create() {
440                        CapturedImage image = new CapturedImage();
441                        image.addMetadata(result);
442                        return image;
443                    }
444
445                @Override
446                    public CapturedImage swap(CapturedImage oldElement) {
447                        oldElement.reset();
448                        oldElement.addMetadata(result);
449                        return oldElement;
450                    }
451
452                @Override
453                    public void update(CapturedImage existingElement) {
454                        existingElement.addMetadata(result);
455                    }
456                });
457
458        if (!swapSuccess) {
459            // Do nothing on failure to swap in.
460            Log.v(TAG, "Unable to add new image metadata to ring-buffer.");
461        }
462
463        tryExecutePendingCaptureRequest(timestamp);
464    }
465
466    @Override
467    public void onImageAvailable(ImageReader reader) {
468        long startTime = SystemClock.currentThreadTimeMillis();
469
470        final Image img = reader.acquireLatestImage();
471
472        if (img != null) {
473            int numOpenImages = mNumOpenImages.incrementAndGet();
474            if (DEBUG_PRINT_OPEN_IMAGE_COUNT) {
475                Log.v(TAG, "Acquired an image. Number of open images = " + numOpenImages);
476            }
477
478            // Try to place the newly-acquired image into the ring buffer.
479            boolean swapSuccess = mCapturedImageBuffer.swapLeast(
480                    img.getTimestamp(), new SwapTask<CapturedImage>() {
481                            @Override
482                        public CapturedImage create() {
483                            CapturedImage image = new CapturedImage();
484                            image.addImage(img);
485                            return image;
486                        }
487
488                            @Override
489                        public CapturedImage swap(CapturedImage oldElement) {
490                            oldElement.reset();
491                            oldElement.addImage(img);
492                            return oldElement;
493                        }
494
495                            @Override
496                        public void update(CapturedImage existingElement) {
497                            existingElement.addImage(img);
498                        }
499                    });
500
501            if (!swapSuccess) {
502                // If we were unable to save the image to the ring buffer, we
503                // must close it now.
504                // We should only get here if the ring buffer is closed.
505                img.close();
506                numOpenImages = mNumOpenImages.decrementAndGet();
507                if (DEBUG_PRINT_OPEN_IMAGE_COUNT) {
508                    Log.v(TAG, "Closed an image. Number of open images = " + numOpenImages);
509                }
510            }
511
512            tryExecutePendingCaptureRequest(img.getTimestamp());
513
514            long endTime = SystemClock.currentThreadTimeMillis();
515            long totTime = endTime - startTime;
516            if (totTime > DEBUG_MAX_IMAGE_CALLBACK_DUR) {
517                // If it takes too long to swap elements, we will start skipping
518                // preview frames, resulting in visible jank.
519                Log.v(TAG, "onImageAvailable() took " + totTime + "ms");
520            }
521        }
522    }
523
524    /**
525     * Closes the listener, eventually freeing all currently-held {@link Image}
526     * s.
527     */
528    public void close() {
529        try {
530            mCapturedImageBuffer.close(new Task<CapturedImage>() {
531                    @Override
532                public void run(CapturedImage e) {
533                    e.reset();
534                }
535            });
536        } catch (InterruptedException e) {
537            e.printStackTrace();
538        }
539    }
540
541    /**
542     * Sets the pending image capture request, overriding any previous calls to
543     * {@link #captureNextImage} which have not yet been resolved. When the next
544     * available image which satisfies the given constraints can be captured,
545     * onImageCaptured will be invoked.
546     *
547     * @param onImageCaptured the callback which will be invoked with the
548     *            captured image.
549     * @param constraints the set of constraints which must be satisfied in
550     *            order for the image to be captured.
551     */
552    public void captureNextImage(final ImageCaptureListener onImageCaptured,
553            final List<CapturedImageConstraint> constraints) {
554        mPendingImageCaptureCallback = onImageCaptured;
555        mPendingImageCaptureConstraints = constraints;
556    }
557
558    /**
559     * Tries to resolve any pending image capture requests.
560     *
561     * @param newImageTimestamp the timestamp of a newly-acquired image which
562     *            should be captured if appropriate and possible.
563     */
564    private void tryExecutePendingCaptureRequest(long newImageTimestamp) {
565        if (mPendingImageCaptureCallback != null) {
566            final Pair<Long, CapturedImage> pinnedImage = mCapturedImageBuffer.tryPin(
567                    newImageTimestamp);
568            if (pinnedImage != null) {
569                CapturedImage image = pinnedImage.second;
570
571                if (!image.isComplete()) {
572                    mCapturedImageBuffer.release(pinnedImage.first);
573                    return;
574                }
575
576                // Check to see if the image satisfies all constraints.
577                TotalCaptureResult captureResult = image.tryGetMetadata();
578
579                if (mPendingImageCaptureConstraints != null) {
580                    for (CapturedImageConstraint constraint : mPendingImageCaptureConstraints) {
581                        if (!constraint.satisfiesConstraint(captureResult)) {
582                            mCapturedImageBuffer.release(pinnedImage.first);
583                            return;
584                        }
585                    }
586                }
587
588                // If we get here, the image satisfies all the necessary
589                // constraints.
590
591                if (tryExecuteCaptureOrRelease(pinnedImage, mPendingImageCaptureCallback)) {
592                    // If we successfully handed the image off to the callback,
593                    // remove the pending
594                    // capture request.
595                    mPendingImageCaptureCallback = null;
596                    mPendingImageCaptureConstraints = null;
597                }
598            }
599        }
600    }
601
602    /**
603     * Tries to capture an existing image from the ring-buffer, if one exists
604     * that satisfies the given constraint and can be pinned.
605     *
606     * @return true if the image could be captured, false otherwise.
607     */
608    public boolean tryCaptureExistingImage(final ImageCaptureListener onImageCaptured,
609            final List<CapturedImageConstraint> constraints) {
610        // The selector to use in choosing the image to capture.
611        Selector<ImageCaptureManager.CapturedImage> selector;
612
613        if (constraints == null || constraints.isEmpty()) {
614            // If there are no constraints, use a trivial Selector.
615            selector = new Selector<ImageCaptureManager.CapturedImage>() {
616                    @Override
617                public boolean select(CapturedImage image) {
618                    return true;
619                }
620            };
621        } else {
622            // If there are constraints, create a Selector which will return
623            // true if all constraints
624            // are satisfied.
625            selector = new Selector<ImageCaptureManager.CapturedImage>() {
626                    @Override
627                public boolean select(CapturedImage e) {
628                    // If this image already has metadata associated with it,
629                    // then use it.
630                    // Otherwise, we can't block until it's available, so assume
631                    // it doesn't
632                    // satisfy the required constraints.
633                    TotalCaptureResult captureResult = e.tryGetMetadata();
634
635                    if (captureResult == null || e.tryGetImage() == null) {
636                        return false;
637                    }
638
639                    for (CapturedImageConstraint constraint : constraints) {
640                        if (!constraint.satisfiesConstraint(captureResult)) {
641                            return false;
642                        }
643                    }
644                    return true;
645                }
646            };
647        }
648
649        // Acquire a lock (pin) on the most recent (greatest-timestamp) image in
650        // the ring buffer which satisfies our constraints.
651        // Note that this must be released as soon as we are done with it.
652        final Pair<Long, CapturedImage> toCapture = mCapturedImageBuffer.tryPinGreatestSelected(
653                selector);
654
655        return tryExecuteCaptureOrRelease(toCapture, onImageCaptured);
656    }
657
658    /**
659     * Tries to execute the image capture callback with the pinned CapturedImage
660     * provided.
661     *
662     * @param toCapture The pinned CapturedImage to pass to the callback, or
663     *            release on failure.
664     * @param callback The callback to execute.
665     * @return true upon success, false upon failure and the release of the
666     *         pinned image.
667     */
668    private boolean tryExecuteCaptureOrRelease(final Pair<Long, CapturedImage> toCapture,
669            final ImageCaptureListener callback) {
670        if (toCapture == null) {
671            return false;
672        } else {
673            try {
674                mImageCaptureListenerExecutor.execute(new Runnable() {
675                        @Override
676                    public void run() {
677                        try {
678                            CapturedImage img = toCapture.second;
679                            callback.onImageCaptured(img.tryGetImage(),
680                                    img.tryGetMetadata());
681                        } finally {
682                            mCapturedImageBuffer.release(toCapture.first);
683                        }
684                    }
685                });
686            } catch (RejectedExecutionException e) {
687                // We may get here if the thread pool has been closed.
688                mCapturedImageBuffer.release(toCapture.first);
689                return false;
690            }
691
692            return true;
693        }
694    }
695}
696