1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17
18package android.filterpacks.videosrc;
19
20import android.content.Context;
21import android.filterfw.core.Filter;
22import android.filterfw.core.FilterContext;
23import android.filterfw.core.Frame;
24import android.filterfw.core.FrameFormat;
25import android.filterfw.core.FrameManager;
26import android.filterfw.core.GenerateFieldPort;
27import android.filterfw.core.GenerateFinalPort;
28import android.filterfw.core.GLFrame;
29import android.filterfw.core.KeyValueMap;
30import android.filterfw.core.MutableFrameFormat;
31import android.filterfw.core.NativeFrame;
32import android.filterfw.core.Program;
33import android.filterfw.core.ShaderProgram;
34import android.filterfw.format.ImageFormat;
35import android.graphics.SurfaceTexture;
36import android.hardware.Camera;
37import android.os.ConditionVariable;
38import android.opengl.Matrix;
39
40import java.io.IOException;
41import java.util.List;
42import java.util.Set;
43
44import android.util.Log;
45
46/**
47 * @hide
48 */
49public class CameraSource extends Filter {
50
51    /** User-visible parameters */
52
53    /** Camera ID to use for input. Defaults to 0. */
54    @GenerateFieldPort(name = "id", hasDefault = true)
55    private int mCameraId = 0;
56
57    /** Frame width to request from camera. Actual size may not match requested. */
58    @GenerateFieldPort(name = "width", hasDefault = true)
59    private int mWidth = 320;
60
61    /** Frame height to request from camera. Actual size may not match requested. */
62    @GenerateFieldPort(name = "height", hasDefault = true)
63    private int mHeight = 240;
64
65    /** Stream framerate to request from camera. Actual frame rate may not match requested. */
66    @GenerateFieldPort(name = "framerate", hasDefault = true)
67    private int mFps = 30;
68
69    /** Whether the filter should always wait for a new frame from the camera
70     * before providing output.  If set to false, the filter will keep
71     * outputting the last frame it received from the camera if multiple process
72     * calls are received before the next update from the Camera. Defaults to true.
73     */
74    @GenerateFinalPort(name = "waitForNewFrame", hasDefault = true)
75    private boolean mWaitForNewFrame = true;
76
77    private Camera mCamera;
78    private GLFrame mCameraFrame;
79    private SurfaceTexture mSurfaceTexture;
80    private ShaderProgram mFrameExtractor;
81    private MutableFrameFormat mOutputFormat;
82
83    private float[] mCameraTransform;
84    private float[] mMappedCoords;
85    // These default source coordinates perform the necessary flip
86    // for converting from OpenGL origin to MFF/Bitmap origin.
87    private static final float[] mSourceCoords = { 0, 1, 0, 1,
88                                                   1, 1, 0, 1,
89                                                   0, 0, 0, 1,
90                                                   1, 0, 0, 1 };
91
92    private static final int NEWFRAME_TIMEOUT = 100; //ms
93    private static final int NEWFRAME_TIMEOUT_REPEAT = 10;
94
95    private boolean mNewFrameAvailable;
96
97    private Camera.Parameters mCameraParameters;
98
99    private static final String mFrameShader =
100            "#extension GL_OES_EGL_image_external : require\n" +
101            "precision mediump float;\n" +
102            "uniform samplerExternalOES tex_sampler_0;\n" +
103            "varying vec2 v_texcoord;\n" +
104            "void main() {\n" +
105            "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
106            "}\n";
107
108    private final boolean mLogVerbose;
109    private static final String TAG = "CameraSource";
110
111    public CameraSource(String name) {
112        super(name);
113        mCameraTransform = new float[16];
114        mMappedCoords = new float[16];
115
116        mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
117    }
118
119    @Override
120    public void setupPorts() {
121        // Add input port
122        addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
123                                                  FrameFormat.TARGET_GPU));
124    }
125
126    private void createFormats() {
127        mOutputFormat = ImageFormat.create(mWidth, mHeight,
128                                           ImageFormat.COLORSPACE_RGBA,
129                                           FrameFormat.TARGET_GPU);
130    }
131
132    @Override
133    public void prepare(FilterContext context) {
134        if (mLogVerbose) Log.v(TAG, "Preparing");
135        // Compile shader TODO: Move to onGLEnvSomething?
136        mFrameExtractor = new ShaderProgram(context, mFrameShader);
137    }
138
139    @Override
140    public void open(FilterContext context) {
141        if (mLogVerbose) Log.v(TAG, "Opening");
142        // Open camera
143        mCamera = Camera.open(mCameraId);
144
145        // Set parameters
146        getCameraParameters();
147        mCamera.setParameters(mCameraParameters);
148
149        // Create frame formats
150        createFormats();
151
152        // Bind it to our camera frame
153        mCameraFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat,
154                                                                        GLFrame.EXTERNAL_TEXTURE,
155                                                                        0);
156        mSurfaceTexture = new SurfaceTexture(mCameraFrame.getTextureId());
157        try {
158            mCamera.setPreviewTexture(mSurfaceTexture);
159        } catch (IOException e) {
160            throw new RuntimeException("Could not bind camera surface texture: " +
161                                       e.getMessage() + "!");
162        }
163
164        // Connect SurfaceTexture to callback
165        mSurfaceTexture.setOnFrameAvailableListener(onCameraFrameAvailableListener);
166        // Start the preview
167        mNewFrameAvailable = false;
168        mCamera.startPreview();
169    }
170
171    @Override
172    public void process(FilterContext context) {
173        if (mLogVerbose) Log.v(TAG, "Processing new frame");
174
175        if (mWaitForNewFrame) {
176            int waitCount = 0;
177            while (!mNewFrameAvailable) {
178                if (waitCount == NEWFRAME_TIMEOUT_REPEAT) {
179                    throw new RuntimeException("Timeout waiting for new frame");
180                }
181                try {
182                    this.wait(NEWFRAME_TIMEOUT);
183                } catch (InterruptedException e) {
184                    if (mLogVerbose) Log.v(TAG, "Interrupted while waiting for new frame");
185                }
186            }
187            mNewFrameAvailable = false;
188            if (mLogVerbose) Log.v(TAG, "Got new frame");
189        }
190
191        mSurfaceTexture.updateTexImage();
192
193        if (mLogVerbose) Log.v(TAG, "Using frame extractor in thread: " + Thread.currentThread());
194        mSurfaceTexture.getTransformMatrix(mCameraTransform);
195        Matrix.multiplyMM(mMappedCoords, 0,
196                          mCameraTransform, 0,
197                          mSourceCoords, 0);
198        mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1],
199                                        mMappedCoords[4], mMappedCoords[5],
200                                        mMappedCoords[8], mMappedCoords[9],
201                                        mMappedCoords[12], mMappedCoords[13]);
202
203        Frame output = context.getFrameManager().newFrame(mOutputFormat);
204        mFrameExtractor.process(mCameraFrame, output);
205
206        long timestamp = mSurfaceTexture.getTimestamp();
207        if (mLogVerbose) Log.v(TAG, "Timestamp: " + (timestamp / 1000000000.0) + " s");
208        output.setTimestamp(timestamp);
209
210        pushOutput("video", output);
211
212        // Release pushed frame
213        output.release();
214
215        if (mLogVerbose) Log.v(TAG, "Done processing new frame");
216    }
217
218    @Override
219    public void close(FilterContext context) {
220        if (mLogVerbose) Log.v(TAG, "Closing");
221
222        mCamera.release();
223        mCamera = null;
224        mSurfaceTexture.release();
225        mSurfaceTexture = null;
226    }
227
228    @Override
229    public void tearDown(FilterContext context) {
230        if (mCameraFrame != null) {
231            mCameraFrame.release();
232        }
233    }
234
235    @Override
236    public void fieldPortValueUpdated(String name, FilterContext context) {
237        if (name.equals("framerate")) {
238            getCameraParameters();
239            int closestRange[] = findClosestFpsRange(mFps, mCameraParameters);
240            mCameraParameters.setPreviewFpsRange(closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX],
241                                                 closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]);
242            mCamera.setParameters(mCameraParameters);
243        }
244    }
245
246    synchronized public Camera.Parameters getCameraParameters() {
247        boolean closeCamera = false;
248        if (mCameraParameters == null) {
249            if (mCamera == null) {
250                mCamera = Camera.open(mCameraId);
251                closeCamera = true;
252            }
253            mCameraParameters = mCamera.getParameters();
254
255            if (closeCamera) {
256                mCamera.release();
257                mCamera = null;
258            }
259        }
260
261        int closestSize[] = findClosestSize(mWidth, mHeight, mCameraParameters);
262        mWidth = closestSize[0];
263        mHeight = closestSize[1];
264        mCameraParameters.setPreviewSize(mWidth, mHeight);
265
266        int closestRange[] = findClosestFpsRange(mFps, mCameraParameters);
267
268        mCameraParameters.setPreviewFpsRange(closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX],
269                                             closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]);
270
271        return mCameraParameters;
272    }
273
274    /** Update camera parameters. Image resolution cannot be changed. */
275    synchronized public void setCameraParameters(Camera.Parameters params) {
276        params.setPreviewSize(mWidth, mHeight);
277        mCameraParameters = params;
278        if (isOpen()) {
279            mCamera.setParameters(mCameraParameters);
280        }
281    }
282
283    private int[] findClosestSize(int width, int height, Camera.Parameters parameters) {
284        List<Camera.Size> previewSizes = parameters.getSupportedPreviewSizes();
285        int closestWidth = -1;
286        int closestHeight = -1;
287        int smallestWidth = previewSizes.get(0).width;
288        int smallestHeight =  previewSizes.get(0).height;
289        for (Camera.Size size : previewSizes) {
290            // Best match defined as not being larger in either dimension than
291            // the requested size, but as close as possible. The below isn't a
292            // stable selection (reording the size list can give different
293            // results), but since this is a fallback nicety, that's acceptable.
294            if ( size.width <= width &&
295                 size.height <= height &&
296                 size.width >= closestWidth &&
297                 size.height >= closestHeight) {
298                closestWidth = size.width;
299                closestHeight = size.height;
300            }
301            if ( size.width < smallestWidth &&
302                 size.height < smallestHeight) {
303                smallestWidth = size.width;
304                smallestHeight = size.height;
305            }
306        }
307        if (closestWidth == -1) {
308            // Requested size is smaller than any listed size; match with smallest possible
309            closestWidth = smallestWidth;
310            closestHeight = smallestHeight;
311        }
312
313        if (mLogVerbose) {
314            Log.v(TAG,
315                  "Requested resolution: (" + width + ", " + height
316                  + "). Closest match: (" + closestWidth + ", "
317                  + closestHeight + ").");
318        }
319        int[] closestSize = {closestWidth, closestHeight};
320        return closestSize;
321    }
322
323    private int[] findClosestFpsRange(int fps, Camera.Parameters params) {
324        List<int[]> supportedFpsRanges = params.getSupportedPreviewFpsRange();
325        int[] closestRange = supportedFpsRanges.get(0);
326        for (int[] range : supportedFpsRanges) {
327            if (range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] < fps*1000 &&
328                range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] > fps*1000 &&
329                range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] >
330                closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] &&
331                range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] <
332                closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]) {
333                closestRange = range;
334            }
335        }
336        if (mLogVerbose) Log.v(TAG, "Requested fps: " + fps
337                               + ".Closest frame rate range: ["
338                               + closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] / 1000.
339                               + ","
340                               + closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] / 1000.
341                               + "]");
342
343        return closestRange;
344    }
345
346    private SurfaceTexture.OnFrameAvailableListener onCameraFrameAvailableListener =
347            new SurfaceTexture.OnFrameAvailableListener() {
348        @Override
349        public void onFrameAvailable(SurfaceTexture surfaceTexture) {
350            if (mLogVerbose) Log.v(TAG, "New frame from camera");
351            synchronized(CameraSource.this) {
352                mNewFrameAvailable = true;
353                CameraSource.this.notify();
354            }
355        }
356    };
357
358}
359