A filter that converts textures from a SurfaceTexture object into frames for * processing in the filter framework.
* *To use, connect up the sourceListener callback, and then when executing * the graph, use the SurfaceTexture object passed to the callback to feed * frames into the filter graph. For example, pass the SurfaceTexture into * {#link * android.hardware.Camera.setPreviewTexture(android.graphics.SurfaceTexture)}. * This filter is intended for applications that need for flexibility than the * CameraSource and MediaSource provide. Note that the application needs to * provide width and height information for the SurfaceTextureSource, which it * should obtain from wherever the SurfaceTexture data is coming from to avoid * unnecessary resampling.
* * @hide */ public class SurfaceTextureSource extends Filter { /** User-visible parameters */ /** The callback interface for the sourceListener parameter */ public interface SurfaceTextureSourceListener { public void onSurfaceTextureSourceReady(SurfaceTexture source); } /** A callback to send the internal SurfaceTexture object to, once it is * created. This callback will be called when the the filter graph is * preparing to execute, but before any processing has actually taken * place. The SurfaceTexture object passed to this callback is the only way * to feed this filter. When the filter graph is shutting down, this * callback will be called again with null as the source. * * This callback may be called from an arbitrary thread, so it should not * assume it is running in the UI thread in particular. */ @GenerateFinalPort(name = "sourceListener") private SurfaceTextureSourceListener mSourceListener; /** The width of the output image frame. If the texture width for the * SurfaceTexture source is known, use it here to minimize resampling. */ @GenerateFieldPort(name = "width") private int mWidth; /** The height of the output image frame. If the texture height for the * SurfaceTexture source is known, use it here to minimize resampling. */ @GenerateFieldPort(name = "height") private int mHeight; /** Whether the filter will always wait for a new frame from its * SurfaceTexture, or whether it will output an old frame again if a new * frame isn't available. The filter will always wait for the first frame, * to avoid outputting a blank frame. Defaults to true. */ @GenerateFieldPort(name = "waitForNewFrame", hasDefault = true) private boolean mWaitForNewFrame = true; /** Maximum timeout before signaling error when waiting for a new frame. Set * this to zero to disable the timeout and wait indefinitely. In milliseconds. */ @GenerateFieldPort(name = "waitTimeout", hasDefault = true) private int mWaitTimeout = 1000; /** Whether a timeout is an exception-causing failure, or just causes the * filter to close. */ @GenerateFieldPort(name = "closeOnTimeout", hasDefault = true) private boolean mCloseOnTimeout = false; // Variables for input->output conversion private GLFrame mMediaFrame; private ShaderProgram mFrameExtractor; private SurfaceTexture mSurfaceTexture; private MutableFrameFormat mOutputFormat; private ConditionVariable mNewFrameAvailable; private boolean mFirstFrame; private float[] mFrameTransform; private float[] mMappedCoords; // These default source coordinates perform the necessary flip // for converting from MFF/Bitmap origin to OpenGL origin. private static final float[] mSourceCoords = { 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1 }; // Shader for output private final String mRenderShader = "#extension GL_OES_EGL_image_external : require\n" + "precision mediump float;\n" + "uniform samplerExternalOES tex_sampler_0;\n" + "varying vec2 v_texcoord;\n" + "void main() {\n" + " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + "}\n"; // Variables for logging private static final String TAG = "SurfaceTextureSource"; private static final boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); public SurfaceTextureSource(String name) { super(name); mNewFrameAvailable = new ConditionVariable(); mFrameTransform = new float[16]; mMappedCoords = new float[16]; } @Override public void setupPorts() { // Add input port addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA, FrameFormat.TARGET_GPU)); } private void createFormats() { mOutputFormat = ImageFormat.create(mWidth, mHeight, ImageFormat.COLORSPACE_RGBA, FrameFormat.TARGET_GPU); } @Override protected void prepare(FilterContext context) { if (mLogVerbose) Log.v(TAG, "Preparing SurfaceTextureSource"); createFormats(); // Prepare input mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat, GLFrame.EXTERNAL_TEXTURE, 0); // Prepare output mFrameExtractor = new ShaderProgram(context, mRenderShader); } @Override public void open(FilterContext context) { if (mLogVerbose) Log.v(TAG, "Opening SurfaceTextureSource"); // Create SurfaceTexture anew each time - it can use substantial memory. mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId()); // Connect SurfaceTexture to callback mSurfaceTexture.setOnFrameAvailableListener(onFrameAvailableListener); // Connect SurfaceTexture to source mSourceListener.onSurfaceTextureSourceReady(mSurfaceTexture); mFirstFrame = true; } @Override public void process(FilterContext context) { if (mLogVerbose) Log.v(TAG, "Processing new frame"); // First, get new frame if available if (mWaitForNewFrame || mFirstFrame) { boolean gotNewFrame; if (mWaitTimeout != 0) { gotNewFrame = mNewFrameAvailable.block(mWaitTimeout); if (!gotNewFrame) { if (!mCloseOnTimeout) { throw new RuntimeException("Timeout waiting for new frame"); } else { if (mLogVerbose) Log.v(TAG, "Timeout waiting for a new frame. Closing."); closeOutputPort("video"); return; } } } else { mNewFrameAvailable.block(); } mNewFrameAvailable.close(); mFirstFrame = false; } mSurfaceTexture.updateTexImage(); mSurfaceTexture.getTransformMatrix(mFrameTransform); Matrix.multiplyMM(mMappedCoords, 0, mFrameTransform, 0, mSourceCoords, 0); mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1], mMappedCoords[4], mMappedCoords[5], mMappedCoords[8], mMappedCoords[9], mMappedCoords[12], mMappedCoords[13]); // Next, render to output Frame output = context.getFrameManager().newFrame(mOutputFormat); mFrameExtractor.process(mMediaFrame, output); output.setTimestamp(mSurfaceTexture.getTimestamp()); pushOutput("video", output); output.release(); } @Override public void close(FilterContext context) { if (mLogVerbose) Log.v(TAG, "SurfaceTextureSource closed"); mSourceListener.onSurfaceTextureSourceReady(null); mSurfaceTexture.release(); mSurfaceTexture = null; } @Override public void tearDown(FilterContext context) { if (mMediaFrame != null) { mMediaFrame.release(); } } @Override public void fieldPortValueUpdated(String name, FilterContext context) { if (name.equals("width") || name.equals("height") ) { mOutputFormat.setDimensions(mWidth, mHeight); } } private SurfaceTexture.OnFrameAvailableListener onFrameAvailableListener = new SurfaceTexture.OnFrameAvailableListener() { public void onFrameAvailable(SurfaceTexture surfaceTexture) { if (mLogVerbose) Log.v(TAG, "New frame from SurfaceTexture"); mNewFrameAvailable.open(); } }; }