165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn/* 265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * Copyright (C) 2011 The Android Open Source Project 365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * 465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * Licensed under the Apache License, Version 2.0 (the "License"); 565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * you may not use this file except in compliance with the License. 665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * You may obtain a copy of the License at 765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * 865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * http://www.apache.org/licenses/LICENSE-2.0 965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * 1065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * Unless required by applicable law or agreed to in writing, software 1165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * distributed under the License is distributed on an "AS IS" BASIS, 1265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * See the License for the specific language governing permissions and 1465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * limitations under the License. 1565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn */ 1665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 1765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennpackage android.filterpacks.videosrc; 1865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 1965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.content.Context; 2065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.content.res.AssetFileDescriptor; 2165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.Filter; 2265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.FilterContext; 2365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.Frame; 2465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.FrameFormat; 2565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.FrameManager; 2665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.GenerateFieldPort; 2765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.GenerateFinalPort; 2865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.GLFrame; 2965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.KeyValueMap; 3065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.MutableFrameFormat; 3165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.ShaderProgram; 3265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.format.ImageFormat; 3365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.graphics.SurfaceTexture; 3465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.media.MediaPlayer; 3565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.os.ConditionVariable; 3665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.opengl.Matrix; 3765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 3865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport java.io.IOException; 3965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport java.io.FileDescriptor; 4065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport java.lang.IllegalArgumentException; 4165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport java.util.List; 4265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport java.util.Set; 4365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 4465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.util.Log; 4565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 4665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn/** <p>A filter that converts textures from a SurfaceTexture object into frames for 4765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * processing in the filter framework.</p> 4865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * 4965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * <p>To use, connect up the sourceListener callback, and then when executing 5065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * the graph, use the SurfaceTexture object passed to the callback to feed 5165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * frames into the filter graph. For example, pass the SurfaceTexture into 5265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * {#link 5365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * android.hardware.Camera.setPreviewTexture(android.graphics.SurfaceTexture)}. 5465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * This filter is intended for applications that need for flexibility than the 5565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * CameraSource and MediaSource provide. Note that the application needs to 5665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * provide width and height information for the SurfaceTextureSource, which it 5765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * should obtain from wherever the SurfaceTexture data is coming from to avoid 5865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * unnecessary resampling.</p> 5965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * 6065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * @hide 6165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn */ 6265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennpublic class SurfaceTextureSource extends Filter { 6365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 6465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** User-visible parameters */ 6565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 6665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** The callback interface for the sourceListener parameter */ 6765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public interface SurfaceTextureSourceListener { 6865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void onSurfaceTextureSourceReady(SurfaceTexture source); 6965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 7065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** A callback to send the internal SurfaceTexture object to, once it is 7165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * created. This callback will be called when the the filter graph is 7265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * preparing to execute, but before any processing has actually taken 7365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * place. The SurfaceTexture object passed to this callback is the only way 7465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * to feed this filter. When the filter graph is shutting down, this 7565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * callback will be called again with null as the source. 7665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * 7765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * This callback may be called from an arbitrary thread, so it should not 7865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * assume it is running in the UI thread in particular. 7965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn */ 8065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFinalPort(name = "sourceListener") 8165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private SurfaceTextureSourceListener mSourceListener; 8265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 8365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** The width of the output image frame. If the texture width for the 8465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * SurfaceTexture source is known, use it here to minimize resampling. */ 8565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "width") 8665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mWidth; 8765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 8865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** The height of the output image frame. If the texture height for the 8965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * SurfaceTexture source is known, use it here to minimize resampling. */ 9065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "height") 9165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mHeight; 9265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 9365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Whether the filter will always wait for a new frame from its 9465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * SurfaceTexture, or whether it will output an old frame again if a new 9565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * frame isn't available. The filter will always wait for the first frame, 9665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * to avoid outputting a blank frame. Defaults to true. 9765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn */ 9865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "waitForNewFrame", hasDefault = true) 9965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private boolean mWaitForNewFrame = true; 10065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 10165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Maximum timeout before signaling error when waiting for a new frame. Set 10265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * this to zero to disable the timeout and wait indefinitely. In milliseconds. 10365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn */ 10465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "waitTimeout", hasDefault = true) 10565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mWaitTimeout = 1000; 10665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 10765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Whether a timeout is an exception-causing failure, or just causes the 10865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * filter to close. 10965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn */ 11065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "closeOnTimeout", hasDefault = true) 11165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private boolean mCloseOnTimeout = false; 11265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 11365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Variables for input->output conversion 11465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private GLFrame mMediaFrame; 11565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private ShaderProgram mFrameExtractor; 11665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private SurfaceTexture mSurfaceTexture; 11765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private MutableFrameFormat mOutputFormat; 11865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private ConditionVariable mNewFrameAvailable; 11965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private boolean mFirstFrame; 12065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 12165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float[] mFrameTransform; 12265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float[] mMappedCoords; 12365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // These default source coordinates perform the necessary flip 12465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // for converting from MFF/Bitmap origin to OpenGL origin. 12565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float[] mSourceCoords = { 0, 1, 0, 1, 12665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 1, 1, 0, 1, 12765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 0, 0, 0, 1, 12865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 1, 0, 0, 1 }; 12965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Shader for output 13065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private final String mRenderShader = 13165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "#extension GL_OES_EGL_image_external : require\n" + 13265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "precision mediump float;\n" + 13365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform samplerExternalOES tex_sampler_0;\n" + 13465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "varying vec2 v_texcoord;\n" + 13565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "void main() {\n" + 13665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + 13765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "}\n"; 13865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 13965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Variables for logging 14065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 14165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String TAG = "SurfaceTextureSource"; 14265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 14365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 14465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public SurfaceTextureSource(String name) { 14565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn super(name); 14665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mNewFrameAvailable = new ConditionVariable(); 14765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mFrameTransform = new float[16]; 14865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMappedCoords = new float[16]; 14965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 15065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 15165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 15265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void setupPorts() { 15365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Add input port 15465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA, 15565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn FrameFormat.TARGET_GPU)); 15665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 15765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 15865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private void createFormats() { 15965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mOutputFormat = ImageFormat.create(mWidth, mHeight, 16065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn ImageFormat.COLORSPACE_RGBA, 16165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn FrameFormat.TARGET_GPU); 16265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 16365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 16465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 16565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn protected void prepare(FilterContext context) { 16665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "Preparing SurfaceTextureSource"); 16765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 16865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn createFormats(); 16965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 17065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Prepare input 17165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat, 17265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn GLFrame.EXTERNAL_TEXTURE, 17365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 0); 17465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 17565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Prepare output 17665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mFrameExtractor = new ShaderProgram(context, mRenderShader); 17765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 17865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 17965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 18065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void open(FilterContext context) { 18165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "Opening SurfaceTextureSource"); 18265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Create SurfaceTexture anew each time - it can use substantial memory. 18365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId()); 18465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Connect SurfaceTexture to callback 18565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mSurfaceTexture.setOnFrameAvailableListener(onFrameAvailableListener); 18665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Connect SurfaceTexture to source 18765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mSourceListener.onSurfaceTextureSourceReady(mSurfaceTexture); 18865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mFirstFrame = true; 18965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 19065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 19165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 19265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void process(FilterContext context) { 19365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "Processing new frame"); 19465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 19565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // First, get new frame if available 19665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mWaitForNewFrame || mFirstFrame) { 19765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn boolean gotNewFrame; 19865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mWaitTimeout != 0) { 19965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn gotNewFrame = mNewFrameAvailable.block(mWaitTimeout); 20065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (!gotNewFrame) { 20165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (!mCloseOnTimeout) { 20265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn throw new RuntimeException("Timeout waiting for new frame"); 20365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 20465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "Timeout waiting for a new frame. Closing."); 20565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn closeOutputPort("video"); 20665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn return; 20765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 20865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 20965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 21065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mNewFrameAvailable.block(); 21165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 21265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mNewFrameAvailable.close(); 21365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mFirstFrame = false; 21465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 21565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 21665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mSurfaceTexture.updateTexImage(); 21765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 21865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mSurfaceTexture.getTransformMatrix(mFrameTransform); 21965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Matrix.multiplyMM(mMappedCoords, 0, 22065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mFrameTransform, 0, 22165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mSourceCoords, 0); 22265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1], 22365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMappedCoords[4], mMappedCoords[5], 22465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMappedCoords[8], mMappedCoords[9], 22565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMappedCoords[12], mMappedCoords[13]); 22665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Next, render to output 22765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame output = context.getFrameManager().newFrame(mOutputFormat); 22865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mFrameExtractor.process(mMediaFrame, output); 22965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 23065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn output.setTimestamp(mSurfaceTexture.getTimestamp()); 23165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 23265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn pushOutput("video", output); 23365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn output.release(); 23465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 23565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 23665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 23765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void close(FilterContext context) { 23865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "SurfaceTextureSource closed"); 23965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mSourceListener.onSurfaceTextureSourceReady(null); 24065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mSurfaceTexture.release(); 24165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mSurfaceTexture = null; 24265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 24365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 24465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 24565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void tearDown(FilterContext context) { 24665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mMediaFrame != null) { 24765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMediaFrame.release(); 24865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 24965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 25065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 25165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 25265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void fieldPortValueUpdated(String name, FilterContext context) { 25365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (name.equals("width") || name.equals("height") ) { 25465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mOutputFormat.setDimensions(mWidth, mHeight); 25565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 25665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 25765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 25865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private SurfaceTexture.OnFrameAvailableListener onFrameAvailableListener = 25965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn new SurfaceTexture.OnFrameAvailableListener() { 26065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void onFrameAvailable(SurfaceTexture surfaceTexture) { 26165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "New frame from SurfaceTexture"); 26265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mNewFrameAvailable.open(); 26365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 26465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn }; 26565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn} 266