1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17
18package android.filterpacks.videosrc;
19
20import android.filterfw.core.Filter;
21import android.filterfw.core.FilterContext;
22import android.filterfw.core.Frame;
23import android.filterfw.core.FrameFormat;
24import android.filterfw.core.GenerateFieldPort;
25import android.filterfw.core.GenerateFinalPort;
26import android.filterfw.core.GLFrame;
27import android.filterfw.core.MutableFrameFormat;
28import android.filterfw.core.ShaderProgram;
29import android.filterfw.format.ImageFormat;
30import android.graphics.SurfaceTexture;
31import android.hardware.Camera;
32import android.opengl.Matrix;
33
34import java.io.IOException;
35import java.util.List;
36
37import android.util.Log;
38
39/**
40 * @hide
41 */
42public class CameraSource extends Filter {
43
44    /** User-visible parameters */
45
46    /** Camera ID to use for input. Defaults to 0. */
47    @GenerateFieldPort(name = "id", hasDefault = true)
48    private int mCameraId = 0;
49
50    /** Frame width to request from camera. Actual size may not match requested. */
51    @GenerateFieldPort(name = "width", hasDefault = true)
52    private int mWidth = 320;
53
54    /** Frame height to request from camera. Actual size may not match requested. */
55    @GenerateFieldPort(name = "height", hasDefault = true)
56    private int mHeight = 240;
57
58    /** Stream framerate to request from camera. Actual frame rate may not match requested. */
59    @GenerateFieldPort(name = "framerate", hasDefault = true)
60    private int mFps = 30;
61
62    /** Whether the filter should always wait for a new frame from the camera
63     * before providing output.  If set to false, the filter will keep
64     * outputting the last frame it received from the camera if multiple process
65     * calls are received before the next update from the Camera. Defaults to true.
66     */
67    @GenerateFinalPort(name = "waitForNewFrame", hasDefault = true)
68    private boolean mWaitForNewFrame = true;
69
70    private Camera mCamera;
71    private GLFrame mCameraFrame;
72    private SurfaceTexture mSurfaceTexture;
73    private ShaderProgram mFrameExtractor;
74    private MutableFrameFormat mOutputFormat;
75
76    private float[] mCameraTransform;
77    private float[] mMappedCoords;
78    // These default source coordinates perform the necessary flip
79    // for converting from OpenGL origin to MFF/Bitmap origin.
80    private static final float[] mSourceCoords = { 0, 1, 0, 1,
81                                                   1, 1, 0, 1,
82                                                   0, 0, 0, 1,
83                                                   1, 0, 0, 1 };
84
85    private static final int NEWFRAME_TIMEOUT = 100; //ms
86    private static final int NEWFRAME_TIMEOUT_REPEAT = 10;
87
88    private boolean mNewFrameAvailable;
89
90    private Camera.Parameters mCameraParameters;
91
92    private static final String mFrameShader =
93            "#extension GL_OES_EGL_image_external : require\n" +
94            "precision mediump float;\n" +
95            "uniform samplerExternalOES tex_sampler_0;\n" +
96            "varying vec2 v_texcoord;\n" +
97            "void main() {\n" +
98            "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
99            "}\n";
100
101    private final boolean mLogVerbose;
102    private static final String TAG = "CameraSource";
103
104    public CameraSource(String name) {
105        super(name);
106        mCameraTransform = new float[16];
107        mMappedCoords = new float[16];
108
109        mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
110    }
111
112    @Override
113    public void setupPorts() {
114        // Add input port
115        addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
116                                                  FrameFormat.TARGET_GPU));
117    }
118
119    private void createFormats() {
120        mOutputFormat = ImageFormat.create(mWidth, mHeight,
121                                           ImageFormat.COLORSPACE_RGBA,
122                                           FrameFormat.TARGET_GPU);
123    }
124
125    @Override
126    public void prepare(FilterContext context) {
127        if (mLogVerbose) Log.v(TAG, "Preparing");
128        // Compile shader TODO: Move to onGLEnvSomething?
129        mFrameExtractor = new ShaderProgram(context, mFrameShader);
130    }
131
132    @Override
133    public void open(FilterContext context) {
134        if (mLogVerbose) Log.v(TAG, "Opening");
135        // Open camera
136        mCamera = Camera.open(mCameraId);
137
138        // Set parameters
139        getCameraParameters();
140        mCamera.setParameters(mCameraParameters);
141
142        // Create frame formats
143        createFormats();
144
145        // Bind it to our camera frame
146        mCameraFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat,
147                                                                        GLFrame.EXTERNAL_TEXTURE,
148                                                                        0);
149        mSurfaceTexture = new SurfaceTexture(mCameraFrame.getTextureId());
150        try {
151            mCamera.setPreviewTexture(mSurfaceTexture);
152        } catch (IOException e) {
153            throw new RuntimeException("Could not bind camera surface texture: " +
154                                       e.getMessage() + "!");
155        }
156
157        // Connect SurfaceTexture to callback
158        mSurfaceTexture.setOnFrameAvailableListener(onCameraFrameAvailableListener);
159        // Start the preview
160        mNewFrameAvailable = false;
161        mCamera.startPreview();
162    }
163
164    @Override
165    public void process(FilterContext context) {
166        if (mLogVerbose) Log.v(TAG, "Processing new frame");
167
168        if (mWaitForNewFrame) {
169            int waitCount = 0;
170            while (!mNewFrameAvailable) {
171                if (waitCount == NEWFRAME_TIMEOUT_REPEAT) {
172                    throw new RuntimeException("Timeout waiting for new frame");
173                }
174                try {
175                    this.wait(NEWFRAME_TIMEOUT);
176                } catch (InterruptedException e) {
177                    if (mLogVerbose) Log.v(TAG, "Interrupted while waiting for new frame");
178                }
179            }
180            mNewFrameAvailable = false;
181            if (mLogVerbose) Log.v(TAG, "Got new frame");
182        }
183
184        mSurfaceTexture.updateTexImage();
185
186        if (mLogVerbose) Log.v(TAG, "Using frame extractor in thread: " + Thread.currentThread());
187        mSurfaceTexture.getTransformMatrix(mCameraTransform);
188        Matrix.multiplyMM(mMappedCoords, 0,
189                          mCameraTransform, 0,
190                          mSourceCoords, 0);
191        mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1],
192                                        mMappedCoords[4], mMappedCoords[5],
193                                        mMappedCoords[8], mMappedCoords[9],
194                                        mMappedCoords[12], mMappedCoords[13]);
195
196        Frame output = context.getFrameManager().newFrame(mOutputFormat);
197        mFrameExtractor.process(mCameraFrame, output);
198
199        long timestamp = mSurfaceTexture.getTimestamp();
200        if (mLogVerbose) Log.v(TAG, "Timestamp: " + (timestamp / 1000000000.0) + " s");
201        output.setTimestamp(timestamp);
202
203        pushOutput("video", output);
204
205        // Release pushed frame
206        output.release();
207
208        if (mLogVerbose) Log.v(TAG, "Done processing new frame");
209    }
210
211    @Override
212    public void close(FilterContext context) {
213        if (mLogVerbose) Log.v(TAG, "Closing");
214
215        mCamera.release();
216        mCamera = null;
217        mSurfaceTexture.release();
218        mSurfaceTexture = null;
219    }
220
221    @Override
222    public void tearDown(FilterContext context) {
223        if (mCameraFrame != null) {
224            mCameraFrame.release();
225        }
226    }
227
228    @Override
229    public void fieldPortValueUpdated(String name, FilterContext context) {
230        if (name.equals("framerate")) {
231            getCameraParameters();
232            int closestRange[] = findClosestFpsRange(mFps, mCameraParameters);
233            mCameraParameters.setPreviewFpsRange(closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX],
234                                                 closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]);
235            mCamera.setParameters(mCameraParameters);
236        }
237    }
238
239    synchronized public Camera.Parameters getCameraParameters() {
240        boolean closeCamera = false;
241        if (mCameraParameters == null) {
242            if (mCamera == null) {
243                mCamera = Camera.open(mCameraId);
244                closeCamera = true;
245            }
246            mCameraParameters = mCamera.getParameters();
247
248            if (closeCamera) {
249                mCamera.release();
250                mCamera = null;
251            }
252        }
253
254        int closestSize[] = findClosestSize(mWidth, mHeight, mCameraParameters);
255        mWidth = closestSize[0];
256        mHeight = closestSize[1];
257        mCameraParameters.setPreviewSize(mWidth, mHeight);
258
259        int closestRange[] = findClosestFpsRange(mFps, mCameraParameters);
260
261        mCameraParameters.setPreviewFpsRange(closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX],
262                                             closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]);
263
264        return mCameraParameters;
265    }
266
267    /** Update camera parameters. Image resolution cannot be changed. */
268    synchronized public void setCameraParameters(Camera.Parameters params) {
269        params.setPreviewSize(mWidth, mHeight);
270        mCameraParameters = params;
271        if (isOpen()) {
272            mCamera.setParameters(mCameraParameters);
273        }
274    }
275
276    private int[] findClosestSize(int width, int height, Camera.Parameters parameters) {
277        List<Camera.Size> previewSizes = parameters.getSupportedPreviewSizes();
278        int closestWidth = -1;
279        int closestHeight = -1;
280        int smallestWidth = previewSizes.get(0).width;
281        int smallestHeight =  previewSizes.get(0).height;
282        for (Camera.Size size : previewSizes) {
283            // Best match defined as not being larger in either dimension than
284            // the requested size, but as close as possible. The below isn't a
285            // stable selection (reording the size list can give different
286            // results), but since this is a fallback nicety, that's acceptable.
287            if ( size.width <= width &&
288                 size.height <= height &&
289                 size.width >= closestWidth &&
290                 size.height >= closestHeight) {
291                closestWidth = size.width;
292                closestHeight = size.height;
293            }
294            if ( size.width < smallestWidth &&
295                 size.height < smallestHeight) {
296                smallestWidth = size.width;
297                smallestHeight = size.height;
298            }
299        }
300        if (closestWidth == -1) {
301            // Requested size is smaller than any listed size; match with smallest possible
302            closestWidth = smallestWidth;
303            closestHeight = smallestHeight;
304        }
305
306        if (mLogVerbose) {
307            Log.v(TAG,
308                  "Requested resolution: (" + width + ", " + height
309                  + "). Closest match: (" + closestWidth + ", "
310                  + closestHeight + ").");
311        }
312        int[] closestSize = {closestWidth, closestHeight};
313        return closestSize;
314    }
315
316    private int[] findClosestFpsRange(int fps, Camera.Parameters params) {
317        List<int[]> supportedFpsRanges = params.getSupportedPreviewFpsRange();
318        int[] closestRange = supportedFpsRanges.get(0);
319        for (int[] range : supportedFpsRanges) {
320            if (range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] < fps*1000 &&
321                range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] > fps*1000 &&
322                range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] >
323                closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] &&
324                range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] <
325                closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]) {
326                closestRange = range;
327            }
328        }
329        if (mLogVerbose) Log.v(TAG, "Requested fps: " + fps
330                               + ".Closest frame rate range: ["
331                               + closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] / 1000.
332                               + ","
333                               + closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] / 1000.
334                               + "]");
335
336        return closestRange;
337    }
338
339    private SurfaceTexture.OnFrameAvailableListener onCameraFrameAvailableListener =
340            new SurfaceTexture.OnFrameAvailableListener() {
341        @Override
342        public void onFrameAvailable(SurfaceTexture surfaceTexture) {
343            if (mLogVerbose) Log.v(TAG, "New frame from camera");
344            synchronized(CameraSource.this) {
345                mNewFrameAvailable = true;
346                CameraSource.this.notify();
347            }
348        }
349    };
350
351}
352