1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package android.filterpacks.videosrc;
18
19import android.content.Context;
20import android.content.res.AssetFileDescriptor;
21import android.filterfw.core.Filter;
22import android.filterfw.core.FilterContext;
23import android.filterfw.core.Frame;
24import android.filterfw.core.FrameFormat;
25import android.filterfw.core.FrameManager;
26import android.filterfw.core.GenerateFieldPort;
27import android.filterfw.core.GenerateFinalPort;
28import android.filterfw.core.GLFrame;
29import android.filterfw.core.KeyValueMap;
30import android.filterfw.core.MutableFrameFormat;
31import android.filterfw.core.ShaderProgram;
32import android.filterfw.format.ImageFormat;
33import android.graphics.SurfaceTexture;
34import android.media.MediaPlayer;
35import android.os.ConditionVariable;
36import android.opengl.Matrix;
37
38import java.io.IOException;
39import java.io.FileDescriptor;
40import java.lang.IllegalArgumentException;
41import java.util.List;
42import java.util.Set;
43
44import android.util.Log;
45
46/** <p>A filter that converts textures from a SurfaceTexture object into frames for
47 * processing in the filter framework.</p>
48 *
49 * <p>To use, connect up the sourceListener callback, and then when executing
50 * the graph, use the SurfaceTexture object passed to the callback to feed
51 * frames into the filter graph. For example, pass the SurfaceTexture into
52 * {#link
53 * android.hardware.Camera.setPreviewTexture(android.graphics.SurfaceTexture)}.
54 * This filter is intended for applications that need for flexibility than the
55 * CameraSource and MediaSource provide. Note that the application needs to
56 * provide width and height information for the SurfaceTextureSource, which it
57 * should obtain from wherever the SurfaceTexture data is coming from to avoid
58 * unnecessary resampling.</p>
59 *
60 * @hide
61 */
62public class SurfaceTextureSource extends Filter {
63
64    /** User-visible parameters */
65
66    /** The callback interface for the sourceListener parameter */
67    public interface SurfaceTextureSourceListener {
68        public void onSurfaceTextureSourceReady(SurfaceTexture source);
69    }
70    /** A callback to send the internal SurfaceTexture object to, once it is
71     * created. This callback will be called when the the filter graph is
72     * preparing to execute, but before any processing has actually taken
73     * place. The SurfaceTexture object passed to this callback is the only way
74     * to feed this filter. When the filter graph is shutting down, this
75     * callback will be called again with null as the source.
76     *
77     * This callback may be called from an arbitrary thread, so it should not
78     * assume it is running in the UI thread in particular.
79     */
80    @GenerateFinalPort(name = "sourceListener")
81    private SurfaceTextureSourceListener mSourceListener;
82
83    /** The width of the output image frame. If the texture width for the
84     * SurfaceTexture source is known, use it here to minimize resampling. */
85    @GenerateFieldPort(name = "width")
86    private int mWidth;
87
88    /** The height of the output image frame. If the texture height for the
89     * SurfaceTexture source is known, use it here to minimize resampling. */
90    @GenerateFieldPort(name = "height")
91    private int mHeight;
92
93    /** Whether the filter will always wait for a new frame from its
94     * SurfaceTexture, or whether it will output an old frame again if a new
95     * frame isn't available. The filter will always wait for the first frame,
96     * to avoid outputting a blank frame. Defaults to true.
97     */
98    @GenerateFieldPort(name = "waitForNewFrame", hasDefault = true)
99    private boolean mWaitForNewFrame = true;
100
101    /** Maximum timeout before signaling error when waiting for a new frame. Set
102     * this to zero to disable the timeout and wait indefinitely. In milliseconds.
103     */
104    @GenerateFieldPort(name = "waitTimeout", hasDefault = true)
105    private int mWaitTimeout = 1000;
106
107    /** Whether a timeout is an exception-causing failure, or just causes the
108     * filter to close.
109     */
110    @GenerateFieldPort(name = "closeOnTimeout", hasDefault = true)
111    private boolean mCloseOnTimeout = false;
112
113    // Variables for input->output conversion
114    private GLFrame mMediaFrame;
115    private ShaderProgram mFrameExtractor;
116    private SurfaceTexture mSurfaceTexture;
117    private MutableFrameFormat mOutputFormat;
118    private ConditionVariable mNewFrameAvailable;
119    private boolean mFirstFrame;
120
121    private float[] mFrameTransform;
122    private float[] mMappedCoords;
123    // These default source coordinates perform the necessary flip
124    // for converting from MFF/Bitmap origin to OpenGL origin.
125    private static final float[] mSourceCoords = { 0, 1, 0, 1,
126                                                   1, 1, 0, 1,
127                                                   0, 0, 0, 1,
128                                                   1, 0, 0, 1 };
129    // Shader for output
130    private final String mRenderShader =
131            "#extension GL_OES_EGL_image_external : require\n" +
132            "precision mediump float;\n" +
133            "uniform samplerExternalOES tex_sampler_0;\n" +
134            "varying vec2 v_texcoord;\n" +
135            "void main() {\n" +
136            "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
137            "}\n";
138
139    // Variables for logging
140
141    private static final String TAG = "SurfaceTextureSource";
142    private static final boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
143
144    public SurfaceTextureSource(String name) {
145        super(name);
146        mNewFrameAvailable = new ConditionVariable();
147        mFrameTransform = new float[16];
148        mMappedCoords = new float[16];
149    }
150
151    @Override
152    public void setupPorts() {
153        // Add input port
154        addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
155                                                  FrameFormat.TARGET_GPU));
156    }
157
158    private void createFormats() {
159        mOutputFormat = ImageFormat.create(mWidth, mHeight,
160                                           ImageFormat.COLORSPACE_RGBA,
161                                           FrameFormat.TARGET_GPU);
162    }
163
164    @Override
165    protected void prepare(FilterContext context) {
166        if (mLogVerbose) Log.v(TAG, "Preparing SurfaceTextureSource");
167
168        createFormats();
169
170        // Prepare input
171        mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat,
172                                                                       GLFrame.EXTERNAL_TEXTURE,
173                                                                       0);
174
175        // Prepare output
176        mFrameExtractor = new ShaderProgram(context, mRenderShader);
177    }
178
179    @Override
180    public void open(FilterContext context) {
181        if (mLogVerbose) Log.v(TAG, "Opening SurfaceTextureSource");
182        // Create SurfaceTexture anew each time - it can use substantial memory.
183        mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId());
184        // Connect SurfaceTexture to callback
185        mSurfaceTexture.setOnFrameAvailableListener(onFrameAvailableListener);
186        // Connect SurfaceTexture to source
187        mSourceListener.onSurfaceTextureSourceReady(mSurfaceTexture);
188        mFirstFrame = true;
189    }
190
191    @Override
192    public void process(FilterContext context) {
193        if (mLogVerbose) Log.v(TAG, "Processing new frame");
194
195        // First, get new frame if available
196        if (mWaitForNewFrame || mFirstFrame) {
197            boolean gotNewFrame;
198            if (mWaitTimeout != 0) {
199                gotNewFrame = mNewFrameAvailable.block(mWaitTimeout);
200                if (!gotNewFrame) {
201                    if (!mCloseOnTimeout) {
202                        throw new RuntimeException("Timeout waiting for new frame");
203                    } else {
204                        if (mLogVerbose) Log.v(TAG, "Timeout waiting for a new frame. Closing.");
205                        closeOutputPort("video");
206                        return;
207                    }
208                }
209            } else {
210                mNewFrameAvailable.block();
211            }
212            mNewFrameAvailable.close();
213            mFirstFrame = false;
214        }
215
216        mSurfaceTexture.updateTexImage();
217
218        mSurfaceTexture.getTransformMatrix(mFrameTransform);
219        Matrix.multiplyMM(mMappedCoords, 0,
220                          mFrameTransform, 0,
221                          mSourceCoords, 0);
222        mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1],
223                                        mMappedCoords[4], mMappedCoords[5],
224                                        mMappedCoords[8], mMappedCoords[9],
225                                        mMappedCoords[12], mMappedCoords[13]);
226        // Next, render to output
227        Frame output = context.getFrameManager().newFrame(mOutputFormat);
228        mFrameExtractor.process(mMediaFrame, output);
229
230        output.setTimestamp(mSurfaceTexture.getTimestamp());
231
232        pushOutput("video", output);
233        output.release();
234    }
235
236    @Override
237    public void close(FilterContext context) {
238        if (mLogVerbose) Log.v(TAG, "SurfaceTextureSource closed");
239        mSourceListener.onSurfaceTextureSourceReady(null);
240        mSurfaceTexture.release();
241        mSurfaceTexture = null;
242    }
243
244    @Override
245    public void tearDown(FilterContext context) {
246        if (mMediaFrame != null) {
247            mMediaFrame.release();
248        }
249    }
250
251    @Override
252    public void fieldPortValueUpdated(String name, FilterContext context) {
253        if (name.equals("width") || name.equals("height") ) {
254            mOutputFormat.setDimensions(mWidth, mHeight);
255        }
256    }
257
258    private SurfaceTexture.OnFrameAvailableListener onFrameAvailableListener =
259            new SurfaceTexture.OnFrameAvailableListener() {
260        public void onFrameAvailable(SurfaceTexture surfaceTexture) {
261            if (mLogVerbose) Log.v(TAG, "New frame from SurfaceTexture");
262            mNewFrameAvailable.open();
263        }
264    };
265}
266