/* * Copyright (C) 2011 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package android.filterpacks.videosrc; import android.filterfw.core.Filter; import android.filterfw.core.FilterContext; import android.filterfw.core.FilterSurfaceView; import android.filterfw.core.Frame; import android.filterfw.core.FrameFormat; import android.filterfw.core.GenerateFieldPort; import android.filterfw.core.GenerateFinalPort; import android.filterfw.core.GLEnvironment; import android.filterfw.core.GLFrame; import android.filterfw.core.KeyValueMap; import android.filterfw.core.MutableFrameFormat; import android.filterfw.core.NativeProgram; import android.filterfw.core.NativeFrame; import android.filterfw.core.Program; import android.filterfw.core.ShaderProgram; import android.filterfw.format.ImageFormat; import android.filterfw.geometry.Quad; import android.filterfw.geometry.Point; import android.view.Surface; import android.view.SurfaceHolder; import android.view.SurfaceView; import android.graphics.Rect; import android.graphics.SurfaceTexture; import android.util.Log; /** * @hide */ public class SurfaceTextureTarget extends Filter { private final int RENDERMODE_STRETCH = 0; private final int RENDERMODE_FIT = 1; private final int RENDERMODE_FILL_CROP = 2; private final int RENDERMODE_CUSTOMIZE = 3; /** Required. Sets the destination surfaceTexture. */ @GenerateFinalPort(name = "surfaceTexture") private SurfaceTexture mSurfaceTexture; /** Required. Sets the width of the output surfaceTexture images */ @GenerateFinalPort(name = "width") private int mScreenWidth; /** Required. Sets the height of the output surfaceTexture images */ @GenerateFinalPort(name = "height") private int mScreenHeight; /** Optional. Control how the incoming frames are rendered onto the * output. Default is FIT. * RENDERMODE_STRETCH: Just fill the output surfaceView. * RENDERMODE_FIT: Keep aspect ratio and fit without cropping. May * have black bars. * RENDERMODE_FILL_CROP: Keep aspect ratio and fit without black * bars. May crop. */ @GenerateFieldPort(name = "renderMode", hasDefault = true) private String mRenderModeString; @GenerateFieldPort(name = "sourceQuad", hasDefault = true) private Quad mSourceQuad = new Quad(new Point(0.0f, 1.0f), new Point(1.0f, 1.0f), new Point(0.0f, 0.0f), new Point(1.0f, 0.0f)); @GenerateFieldPort(name = "targetQuad", hasDefault = true) private Quad mTargetQuad = new Quad(new Point(0.0f, 0.0f), new Point(1.0f, 0.0f), new Point(0.0f, 1.0f), new Point(1.0f, 1.0f)); private int mSurfaceId; private ShaderProgram mProgram; private GLFrame mScreen; private int mRenderMode = RENDERMODE_FIT; private float mAspectRatio = 1.f; private boolean mLogVerbose; private static final String TAG = "SurfaceTextureTarget"; public SurfaceTextureTarget(String name) { super(name); mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); } @Override public synchronized void setupPorts() { // Make sure we have a SurfaceView if (mSurfaceTexture == null) { throw new RuntimeException("Null SurfaceTexture passed to SurfaceTextureTarget"); } // Add input port - will accept anything that's 4-channel. addMaskedInputPort("frame", ImageFormat.create(ImageFormat.COLORSPACE_RGBA)); } public void updateRenderMode() { if (mLogVerbose) Log.v(TAG, "updateRenderMode. Thread: " + Thread.currentThread()); if (mRenderModeString != null) { if (mRenderModeString.equals("stretch")) { mRenderMode = RENDERMODE_STRETCH; } else if (mRenderModeString.equals("fit")) { mRenderMode = RENDERMODE_FIT; } else if (mRenderModeString.equals("fill_crop")) { mRenderMode = RENDERMODE_FILL_CROP; } else if (mRenderModeString.equals("customize")) { mRenderMode = RENDERMODE_CUSTOMIZE; } else { throw new RuntimeException("Unknown render mode '" + mRenderModeString + "'!"); } } updateTargetRect(); } @Override public void prepare(FilterContext context) { if (mLogVerbose) Log.v(TAG, "Prepare. Thread: " + Thread.currentThread()); // Create identity shader to render, and make sure to render upside-down, as textures // are stored internally bottom-to-top. mProgram = ShaderProgram.createIdentity(context); mProgram.setSourceRect(0, 1, 1, -1); mProgram.setClearColor(0.0f, 0.0f, 0.0f); updateRenderMode(); // Create a frame representing the screen MutableFrameFormat screenFormat = new MutableFrameFormat(FrameFormat.TYPE_BYTE, FrameFormat.TARGET_GPU); screenFormat.setBytesPerSample(4); screenFormat.setDimensions(mScreenWidth, mScreenHeight); mScreen = (GLFrame)context.getFrameManager().newBoundFrame(screenFormat, GLFrame.EXISTING_FBO_BINDING, 0); } @Override public synchronized void open(FilterContext context) { // Set up SurfaceTexture internals if (mSurfaceTexture == null) { Log.e(TAG, "SurfaceTexture is null!!"); throw new RuntimeException("Could not register SurfaceTexture: " + mSurfaceTexture); } mSurfaceId = context.getGLEnvironment().registerSurfaceTexture( mSurfaceTexture, mScreenWidth, mScreenHeight); if (mSurfaceId <= 0) { throw new RuntimeException("Could not register SurfaceTexture: " + mSurfaceTexture); } } // Once the surface is unregistered, we still need the surfacetexture reference. // That is because when the the filter graph stops and starts again, the app // may not set the mSurfaceTexture again on the filter. In some cases, the app // may not even know that the graph has re-started. So it is difficult to enforce // that condition on an app using this filter. The only case where we need // to let go of the mSurfaceTexure reference is when the app wants to shut // down the graph on purpose, such as in the disconnect call. @Override public synchronized void close(FilterContext context) { if (mSurfaceId > 0) { context.getGLEnvironment().unregisterSurfaceId(mSurfaceId); mSurfaceId = -1; } } // This should be called from the client side when the surfacetexture is no longer // valid. e.g. from onPause() in the application using the filter graph. // In this case, we need to let go of our surfacetexture reference. public synchronized void disconnect(FilterContext context) { if (mLogVerbose) Log.v(TAG, "disconnect"); if (mSurfaceTexture == null) { Log.d(TAG, "SurfaceTexture is already null. Nothing to disconnect."); return; } mSurfaceTexture = null; // Make sure we unregister the surface as well if a surface was registered. // There can be a situation where the surface was not registered but the // surfacetexture was valid. For example, the disconnect can be called before // the filter was opened. Hence, the surfaceId may not be a valid one here, // and need to check for its validity. if (mSurfaceId > 0) { context.getGLEnvironment().unregisterSurfaceId(mSurfaceId); mSurfaceId = -1; } } @Override public synchronized void process(FilterContext context) { // Surface is not registered. Nothing to render into. if (mSurfaceId <= 0) { return; } GLEnvironment glEnv = context.getGLEnvironment(); // Get input frame Frame input = pullInput("frame"); boolean createdFrame = false; float currentAspectRatio = (float)input.getFormat().getWidth() / input.getFormat().getHeight(); if (currentAspectRatio != mAspectRatio) { if (mLogVerbose) { Log.v(TAG, "Process. New aspect ratio: " + currentAspectRatio + ", previously: " + mAspectRatio + ". Thread: " + Thread.currentThread()); } mAspectRatio = currentAspectRatio; updateTargetRect(); } // See if we need to copy to GPU Frame gpuFrame = null; int target = input.getFormat().getTarget(); if (target != FrameFormat.TARGET_GPU) { gpuFrame = context.getFrameManager().duplicateFrameToTarget(input, FrameFormat.TARGET_GPU); createdFrame = true; } else { gpuFrame = input; } // Activate our surface glEnv.activateSurfaceWithId(mSurfaceId); // Process mProgram.process(gpuFrame, mScreen); glEnv.setSurfaceTimestamp(input.getTimestamp()); // And swap buffers glEnv.swapBuffers(); if (createdFrame) { gpuFrame.release(); } } @Override public void fieldPortValueUpdated(String name, FilterContext context) { if (mLogVerbose) Log.v(TAG, "FPVU. Thread: " + Thread.currentThread()); updateRenderMode(); } @Override public void tearDown(FilterContext context) { if (mScreen != null) { mScreen.release(); } } private void updateTargetRect() { if (mLogVerbose) Log.v(TAG, "updateTargetRect. Thread: " + Thread.currentThread()); if (mScreenWidth > 0 && mScreenHeight > 0 && mProgram != null) { float screenAspectRatio = (float)mScreenWidth / mScreenHeight; float relativeAspectRatio = screenAspectRatio / mAspectRatio; if (mLogVerbose) { Log.v(TAG, "UTR. screen w = " + (float)mScreenWidth + " x screen h = " + (float)mScreenHeight + " Screen AR: " + screenAspectRatio + ", frame AR: " + mAspectRatio + ", relative AR: " + relativeAspectRatio); } if (relativeAspectRatio == 1.0f && mRenderMode != RENDERMODE_CUSTOMIZE) { mProgram.setTargetRect(0, 0, 1, 1); mProgram.setClearsOutput(false); } else { switch (mRenderMode) { case RENDERMODE_STRETCH: mTargetQuad.p0.set(0f, 0.0f); mTargetQuad.p1.set(1f, 0.0f); mTargetQuad.p2.set(0f, 1.0f); mTargetQuad.p3.set(1f, 1.0f); mProgram.setClearsOutput(false); break; case RENDERMODE_FIT: if (relativeAspectRatio > 1.0f) { // Screen is wider than the camera, scale down X mTargetQuad.p0.set(0.5f - 0.5f / relativeAspectRatio, 0.0f); mTargetQuad.p1.set(0.5f + 0.5f / relativeAspectRatio, 0.0f); mTargetQuad.p2.set(0.5f - 0.5f / relativeAspectRatio, 1.0f); mTargetQuad.p3.set(0.5f + 0.5f / relativeAspectRatio, 1.0f); } else { // Screen is taller than the camera, scale down Y mTargetQuad.p0.set(0.0f, 0.5f - 0.5f * relativeAspectRatio); mTargetQuad.p1.set(1.0f, 0.5f - 0.5f * relativeAspectRatio); mTargetQuad.p2.set(0.0f, 0.5f + 0.5f * relativeAspectRatio); mTargetQuad.p3.set(1.0f, 0.5f + 0.5f * relativeAspectRatio); } mProgram.setClearsOutput(true); break; case RENDERMODE_FILL_CROP: if (relativeAspectRatio > 1) { // Screen is wider than the camera, crop in Y mTargetQuad.p0.set(0.0f, 0.5f - 0.5f * relativeAspectRatio); mTargetQuad.p1.set(1.0f, 0.5f - 0.5f * relativeAspectRatio); mTargetQuad.p2.set(0.0f, 0.5f + 0.5f * relativeAspectRatio); mTargetQuad.p3.set(1.0f, 0.5f + 0.5f * relativeAspectRatio); } else { // Screen is taller than the camera, crop in X mTargetQuad.p0.set(0.5f - 0.5f / relativeAspectRatio, 0.0f); mTargetQuad.p1.set(0.5f + 0.5f / relativeAspectRatio, 0.0f); mTargetQuad.p2.set(0.5f - 0.5f / relativeAspectRatio, 1.0f); mTargetQuad.p3.set(0.5f + 0.5f / relativeAspectRatio, 1.0f); } mProgram.setClearsOutput(true); break; case RENDERMODE_CUSTOMIZE: ((ShaderProgram) mProgram).setSourceRegion(mSourceQuad); break; } if (mLogVerbose) Log.v(TAG, "UTR. quad: " + mTargetQuad); ((ShaderProgram) mProgram).setTargetRegion(mTargetQuad); } } } }