1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package android.filterpacks.videosrc;
18
19import android.filterfw.core.Filter;
20import android.filterfw.core.FilterContext;
21import android.filterfw.core.Frame;
22import android.filterfw.core.FrameFormat;
23import android.filterfw.core.GenerateFieldPort;
24import android.filterfw.core.GenerateFinalPort;
25import android.filterfw.core.GLFrame;
26import android.filterfw.core.MutableFrameFormat;
27import android.filterfw.core.ShaderProgram;
28import android.filterfw.format.ImageFormat;
29import android.graphics.SurfaceTexture;
30import android.os.ConditionVariable;
31import android.opengl.Matrix;
32
33import android.util.Log;
34
35/** <p>A filter that converts textures from a SurfaceTexture object into frames for
36 * processing in the filter framework.</p>
37 *
38 * <p>To use, connect up the sourceListener callback, and then when executing
39 * the graph, use the SurfaceTexture object passed to the callback to feed
40 * frames into the filter graph. For example, pass the SurfaceTexture into
41 * {#link
42 * android.hardware.Camera.setPreviewTexture(android.graphics.SurfaceTexture)}.
43 * This filter is intended for applications that need for flexibility than the
44 * CameraSource and MediaSource provide. Note that the application needs to
45 * provide width and height information for the SurfaceTextureSource, which it
46 * should obtain from wherever the SurfaceTexture data is coming from to avoid
47 * unnecessary resampling.</p>
48 *
49 * @hide
50 */
51public class SurfaceTextureSource extends Filter {
52
53    /** User-visible parameters */
54
55    /** The callback interface for the sourceListener parameter */
56    public interface SurfaceTextureSourceListener {
57        public void onSurfaceTextureSourceReady(SurfaceTexture source);
58    }
59    /** A callback to send the internal SurfaceTexture object to, once it is
60     * created. This callback will be called when the the filter graph is
61     * preparing to execute, but before any processing has actually taken
62     * place. The SurfaceTexture object passed to this callback is the only way
63     * to feed this filter. When the filter graph is shutting down, this
64     * callback will be called again with null as the source.
65     *
66     * This callback may be called from an arbitrary thread, so it should not
67     * assume it is running in the UI thread in particular.
68     */
69    @GenerateFinalPort(name = "sourceListener")
70    private SurfaceTextureSourceListener mSourceListener;
71
72    /** The width of the output image frame. If the texture width for the
73     * SurfaceTexture source is known, use it here to minimize resampling. */
74    @GenerateFieldPort(name = "width")
75    private int mWidth;
76
77    /** The height of the output image frame. If the texture height for the
78     * SurfaceTexture source is known, use it here to minimize resampling. */
79    @GenerateFieldPort(name = "height")
80    private int mHeight;
81
82    /** Whether the filter will always wait for a new frame from its
83     * SurfaceTexture, or whether it will output an old frame again if a new
84     * frame isn't available. The filter will always wait for the first frame,
85     * to avoid outputting a blank frame. Defaults to true.
86     */
87    @GenerateFieldPort(name = "waitForNewFrame", hasDefault = true)
88    private boolean mWaitForNewFrame = true;
89
90    /** Maximum timeout before signaling error when waiting for a new frame. Set
91     * this to zero to disable the timeout and wait indefinitely. In milliseconds.
92     */
93    @GenerateFieldPort(name = "waitTimeout", hasDefault = true)
94    private int mWaitTimeout = 1000;
95
96    /** Whether a timeout is an exception-causing failure, or just causes the
97     * filter to close.
98     */
99    @GenerateFieldPort(name = "closeOnTimeout", hasDefault = true)
100    private boolean mCloseOnTimeout = false;
101
102    // Variables for input->output conversion
103    private GLFrame mMediaFrame;
104    private ShaderProgram mFrameExtractor;
105    private SurfaceTexture mSurfaceTexture;
106    private MutableFrameFormat mOutputFormat;
107    private ConditionVariable mNewFrameAvailable;
108    private boolean mFirstFrame;
109
110    private float[] mFrameTransform;
111    private float[] mMappedCoords;
112    // These default source coordinates perform the necessary flip
113    // for converting from MFF/Bitmap origin to OpenGL origin.
114    private static final float[] mSourceCoords = { 0, 1, 0, 1,
115                                                   1, 1, 0, 1,
116                                                   0, 0, 0, 1,
117                                                   1, 0, 0, 1 };
118    // Shader for output
119    private final String mRenderShader =
120            "#extension GL_OES_EGL_image_external : require\n" +
121            "precision mediump float;\n" +
122            "uniform samplerExternalOES tex_sampler_0;\n" +
123            "varying vec2 v_texcoord;\n" +
124            "void main() {\n" +
125            "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
126            "}\n";
127
128    // Variables for logging
129
130    private static final String TAG = "SurfaceTextureSource";
131    private static final boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
132
133    public SurfaceTextureSource(String name) {
134        super(name);
135        mNewFrameAvailable = new ConditionVariable();
136        mFrameTransform = new float[16];
137        mMappedCoords = new float[16];
138    }
139
140    @Override
141    public void setupPorts() {
142        // Add input port
143        addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
144                                                  FrameFormat.TARGET_GPU));
145    }
146
147    private void createFormats() {
148        mOutputFormat = ImageFormat.create(mWidth, mHeight,
149                                           ImageFormat.COLORSPACE_RGBA,
150                                           FrameFormat.TARGET_GPU);
151    }
152
153    @Override
154    protected void prepare(FilterContext context) {
155        if (mLogVerbose) Log.v(TAG, "Preparing SurfaceTextureSource");
156
157        createFormats();
158
159        // Prepare input
160        mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat,
161                                                                       GLFrame.EXTERNAL_TEXTURE,
162                                                                       0);
163
164        // Prepare output
165        mFrameExtractor = new ShaderProgram(context, mRenderShader);
166    }
167
168    @Override
169    public void open(FilterContext context) {
170        if (mLogVerbose) Log.v(TAG, "Opening SurfaceTextureSource");
171        // Create SurfaceTexture anew each time - it can use substantial memory.
172        mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId());
173        // Connect SurfaceTexture to callback
174        mSurfaceTexture.setOnFrameAvailableListener(onFrameAvailableListener);
175        // Connect SurfaceTexture to source
176        mSourceListener.onSurfaceTextureSourceReady(mSurfaceTexture);
177        mFirstFrame = true;
178    }
179
180    @Override
181    public void process(FilterContext context) {
182        if (mLogVerbose) Log.v(TAG, "Processing new frame");
183
184        // First, get new frame if available
185        if (mWaitForNewFrame || mFirstFrame) {
186            boolean gotNewFrame;
187            if (mWaitTimeout != 0) {
188                gotNewFrame = mNewFrameAvailable.block(mWaitTimeout);
189                if (!gotNewFrame) {
190                    if (!mCloseOnTimeout) {
191                        throw new RuntimeException("Timeout waiting for new frame");
192                    } else {
193                        if (mLogVerbose) Log.v(TAG, "Timeout waiting for a new frame. Closing.");
194                        closeOutputPort("video");
195                        return;
196                    }
197                }
198            } else {
199                mNewFrameAvailable.block();
200            }
201            mNewFrameAvailable.close();
202            mFirstFrame = false;
203        }
204
205        mSurfaceTexture.updateTexImage();
206
207        mSurfaceTexture.getTransformMatrix(mFrameTransform);
208        Matrix.multiplyMM(mMappedCoords, 0,
209                          mFrameTransform, 0,
210                          mSourceCoords, 0);
211        mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1],
212                                        mMappedCoords[4], mMappedCoords[5],
213                                        mMappedCoords[8], mMappedCoords[9],
214                                        mMappedCoords[12], mMappedCoords[13]);
215        // Next, render to output
216        Frame output = context.getFrameManager().newFrame(mOutputFormat);
217        mFrameExtractor.process(mMediaFrame, output);
218
219        output.setTimestamp(mSurfaceTexture.getTimestamp());
220
221        pushOutput("video", output);
222        output.release();
223    }
224
225    @Override
226    public void close(FilterContext context) {
227        if (mLogVerbose) Log.v(TAG, "SurfaceTextureSource closed");
228        mSourceListener.onSurfaceTextureSourceReady(null);
229        mSurfaceTexture.release();
230        mSurfaceTexture = null;
231    }
232
233    @Override
234    public void tearDown(FilterContext context) {
235        if (mMediaFrame != null) {
236            mMediaFrame.release();
237        }
238    }
239
240    @Override
241    public void fieldPortValueUpdated(String name, FilterContext context) {
242        if (name.equals("width") || name.equals("height") ) {
243            mOutputFormat.setDimensions(mWidth, mHeight);
244        }
245    }
246
247    private SurfaceTexture.OnFrameAvailableListener onFrameAvailableListener =
248            new SurfaceTexture.OnFrameAvailableListener() {
249        public void onFrameAvailable(SurfaceTexture surfaceTexture) {
250            if (mLogVerbose) Log.v(TAG, "New frame from SurfaceTexture");
251            mNewFrameAvailable.open();
252        }
253    };
254}
255