xref: /frameworks/base/media/mca/filterpacks/java/android/filterpacks/videosrc/SurfaceTextureSource.java

/*
 * Copyright (C) 2011 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.filterpacks.videosrc;

import android.content.Context;
import android.content.res.AssetFileDescriptor;
import android.filterfw.core.Filter;
import android.filterfw.core.FilterContext;
import android.filterfw.core.Frame;
import android.filterfw.core.FrameFormat;
import android.filterfw.core.FrameManager;
import android.filterfw.core.GenerateFieldPort;
import android.filterfw.core.GenerateFinalPort;
import android.filterfw.core.GLFrame;
import android.filterfw.core.KeyValueMap;
import android.filterfw.core.MutableFrameFormat;
import android.filterfw.core.ShaderProgram;
import android.filterfw.format.ImageFormat;
import android.graphics.SurfaceTexture;
import android.media.MediaPlayer;
import android.os.ConditionVariable;
import android.opengl.Matrix;

import java.io.IOException;
import java.io.FileDescriptor;
import java.lang.IllegalArgumentException;
import java.util.List;
import java.util.Set;

import android.util.Log;

/** <p>A filter that converts textures from a SurfaceTexture object into frames for
 * processing in the filter framework.</p>
 *
 * <p>To use, connect up the sourceListener callback, and then when executing
 * the graph, use the SurfaceTexture object passed to the callback to feed
 * frames into the filter graph. For example, pass the SurfaceTexture into
 * {#link
 * android.hardware.Camera.setPreviewTexture(android.graphics.SurfaceTexture)}.
 * This filter is intended for applications that need for flexibility than the
 * CameraSource and MediaSource provide. Note that the application needs to
 * provide width and height information for the SurfaceTextureSource, which it
 * should obtain from wherever the SurfaceTexture data is coming from to avoid
 * unnecessary resampling.</p>
 *
 * @hide
 */
public class SurfaceTextureSource extends Filter {

    /** User-visible parameters */

    /** The callback interface for the sourceListener parameter */
    public interface SurfaceTextureSourceListener {
        public void onSurfaceTextureSourceReady(SurfaceTexture source);
    }
    /** A callback to send the internal SurfaceTexture object to, once it is
     * created. This callback will be called when the the filter graph is
     * preparing to execute, but before any processing has actually taken
     * place. The SurfaceTexture object passed to this callback is the only way
     * to feed this filter. When the filter graph is shutting down, this
     * callback will be called again with null as the source.
     *
     * This callback may be called from an arbitrary thread, so it should not
     * assume it is running in the UI thread in particular.
     */
    @GenerateFinalPort(name = "sourceListener")
    private SurfaceTextureSourceListener mSourceListener;

    /** The width of the output image frame. If the texture width for the
     * SurfaceTexture source is known, use it here to minimize resampling. */
    @GenerateFieldPort(name = "width")
    private int mWidth;

    /** The height of the output image frame. If the texture height for the
     * SurfaceTexture source is known, use it here to minimize resampling. */
    @GenerateFieldPort(name = "height")
    private int mHeight;

    /** Whether the filter will always wait for a new frame from its
     * SurfaceTexture, or whether it will output an old frame again if a new
     * frame isn't available. The filter will always wait for the first frame,
     * to avoid outputting a blank frame. Defaults to true.
     */
    @GenerateFieldPort(name = "waitForNewFrame", hasDefault = true)
    private boolean mWaitForNewFrame = true;

    /** Maximum timeout before signaling error when waiting for a new frame. Set
     * this to zero to disable the timeout and wait indefinitely. In milliseconds.
     */
    @GenerateFieldPort(name = "waitTimeout", hasDefault = true)
    private int mWaitTimeout = 1000;

    /** Whether a timeout is an exception-causing failure, or just causes the
     * filter to close.
     */
    @GenerateFieldPort(name = "closeOnTimeout", hasDefault = true)
    private boolean mCloseOnTimeout = false;

    // Variables for input->output conversion
    private GLFrame mMediaFrame;
    private ShaderProgram mFrameExtractor;
    private SurfaceTexture mSurfaceTexture;
    private MutableFrameFormat mOutputFormat;
    private ConditionVariable mNewFrameAvailable;
    private boolean mFirstFrame;

    private float[] mFrameTransform;
    private float[] mMappedCoords;
    // These default source coordinates perform the necessary flip
    // for converting from MFF/Bitmap origin to OpenGL origin.
    private static final float[] mSourceCoords = { 0, 1, 0, 1,
                                                   1, 1, 0, 1,
                                                   0, 0, 0, 1,
                                                   1, 0, 0, 1 };
    // Shader for output
    private final String mRenderShader =
            "#extension GL_OES_EGL_image_external : require\n" +
            "precision mediump float;\n" +
            "uniform samplerExternalOES tex_sampler_0;\n" +
            "varying vec2 v_texcoord;\n" +
            "void main() {\n" +
            "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
            "}\n";

    // Variables for logging

    private static final String TAG = "SurfaceTextureSource";
    private static final boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);

    public SurfaceTextureSource(String name) {
        super(name);
        mNewFrameAvailable = new ConditionVariable();
        mFrameTransform = new float[16];
        mMappedCoords = new float[16];
    }

    @Override
    public void setupPorts() {
        // Add input port
        addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
                                                  FrameFormat.TARGET_GPU));
    }

    private void createFormats() {
        mOutputFormat = ImageFormat.create(mWidth, mHeight,
                                           ImageFormat.COLORSPACE_RGBA,
                                           FrameFormat.TARGET_GPU);
    }

    @Override
    protected void prepare(FilterContext context) {
        if (mLogVerbose) Log.v(TAG, "Preparing SurfaceTextureSource");

        createFormats();

        // Prepare input
        mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat,
                                                                       GLFrame.EXTERNAL_TEXTURE,
                                                                       0);

        // Prepare output
        mFrameExtractor = new ShaderProgram(context, mRenderShader);
    }

    @Override
    public void open(FilterContext context) {
        if (mLogVerbose) Log.v(TAG, "Opening SurfaceTextureSource");
        // Create SurfaceTexture anew each time - it can use substantial memory.
        mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId());
        // Connect SurfaceTexture to callback
        mSurfaceTexture.setOnFrameAvailableListener(onFrameAvailableListener);
        // Connect SurfaceTexture to source
        mSourceListener.onSurfaceTextureSourceReady(mSurfaceTexture);
        mFirstFrame = true;
    }

    @Override
    public void process(FilterContext context) {
        if (mLogVerbose) Log.v(TAG, "Processing new frame");

        // First, get new frame if available
        if (mWaitForNewFrame || mFirstFrame) {
            boolean gotNewFrame;
            if (mWaitTimeout != 0) {
                gotNewFrame = mNewFrameAvailable.block(mWaitTimeout);
                if (!gotNewFrame) {
                    if (!mCloseOnTimeout) {
                        throw new RuntimeException("Timeout waiting for new frame");
                    } else {
                        if (mLogVerbose) Log.v(TAG, "Timeout waiting for a new frame. Closing.");
                        closeOutputPort("video");
                        return;
                    }
                }
            } else {
                mNewFrameAvailable.block();
            }
            mNewFrameAvailable.close();
            mFirstFrame = false;
        }

        mSurfaceTexture.updateTexImage();

        mSurfaceTexture.getTransformMatrix(mFrameTransform);
        Matrix.multiplyMM(mMappedCoords, 0,
                          mFrameTransform, 0,
                          mSourceCoords, 0);
        mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1],
                                        mMappedCoords[4], mMappedCoords[5],
                                        mMappedCoords[8], mMappedCoords[9],
                                        mMappedCoords[12], mMappedCoords[13]);
        // Next, render to output
        Frame output = context.getFrameManager().newFrame(mOutputFormat);
        mFrameExtractor.process(mMediaFrame, output);

        output.setTimestamp(mSurfaceTexture.getTimestamp());

        pushOutput("video", output);
        output.release();
    }

    @Override
    public void close(FilterContext context) {
        if (mLogVerbose) Log.v(TAG, "SurfaceTextureSource closed");
        mSourceListener.onSurfaceTextureSourceReady(null);
        mSurfaceTexture.release();
        mSurfaceTexture = null;
    }

    @Override
    public void tearDown(FilterContext context) {
        if (mMediaFrame != null) {
            mMediaFrame.release();
        }
    }

    @Override
    public void fieldPortValueUpdated(String name, FilterContext context) {
        if (name.equals("width") || name.equals("height") ) {
            mOutputFormat.setDimensions(mWidth, mHeight);
        }
    }

    private SurfaceTexture.OnFrameAvailableListener onFrameAvailableListener =
            new SurfaceTexture.OnFrameAvailableListener() {
        public void onFrameAvailable(SurfaceTexture surfaceTexture) {
            if (mLogVerbose) Log.v(TAG, "New frame from SurfaceTexture");
            mNewFrameAvailable.open();
        }
    };
}