165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn/* 265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * Copyright (C) 2011 The Android Open Source Project 365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * 465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * Licensed under the Apache License, Version 2.0 (the "License"); 565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * you may not use this file except in compliance with the License. 665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * You may obtain a copy of the License at 765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * 865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * http://www.apache.org/licenses/LICENSE-2.0 965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * 1065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * Unless required by applicable law or agreed to in writing, software 1165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * distributed under the License is distributed on an "AS IS" BASIS, 1265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * See the License for the specific language governing permissions and 1465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * limitations under the License. 1565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn */ 1665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 1765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 1865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennpackage android.filterpacks.videosrc; 1965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 2065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.Filter; 2165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.FilterContext; 2265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.FilterSurfaceView; 2365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.Frame; 2465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.FrameFormat; 2565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.GenerateFieldPort; 2665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.GenerateFinalPort; 2765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.GLEnvironment; 2865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.GLFrame; 2965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.KeyValueMap; 3065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.MutableFrameFormat; 3165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.NativeProgram; 3265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.NativeFrame; 3365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.Program; 3465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.ShaderProgram; 3565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.format.ImageFormat; 3665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 3765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.geometry.Quad; 3865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.geometry.Point; 3965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 4065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.view.Surface; 4165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.view.SurfaceHolder; 4265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.view.SurfaceView; 4365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 4465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.graphics.Rect; 4565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.graphics.SurfaceTexture; 4665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 4765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.util.Log; 4865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 4965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn/** 5065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * @hide 5165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn */ 5265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennpublic class SurfaceTextureTarget extends Filter { 5365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 5465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private final int RENDERMODE_STRETCH = 0; 5565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private final int RENDERMODE_FIT = 1; 5665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private final int RENDERMODE_FILL_CROP = 2; 5765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private final int RENDERMODE_CUSTOMIZE = 3; 5865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 5965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Required. Sets the destination surfaceTexture. 6065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn */ 6165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFinalPort(name = "surfaceTexture") 6265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private SurfaceTexture mSurfaceTexture; 6365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 6465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Required. Sets the width of the output surfaceTexture images */ 6565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFinalPort(name = "width") 6665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mScreenWidth; 6765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 6865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Required. Sets the height of the output surfaceTexture images */ 6965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFinalPort(name = "height") 7065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mScreenHeight; 7165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 7265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 7365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Optional. Control how the incoming frames are rendered onto the 7465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * output. Default is FIT. 7565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * RENDERMODE_STRETCH: Just fill the output surfaceView. 7665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * RENDERMODE_FIT: Keep aspect ratio and fit without cropping. May 7765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * have black bars. 7865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * RENDERMODE_FILL_CROP: Keep aspect ratio and fit without black 7965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * bars. May crop. 8065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn */ 8165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "renderMode", hasDefault = true) 8265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private String mRenderModeString; 8365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 8465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "sourceQuad", hasDefault = true) 8565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private Quad mSourceQuad = new Quad(new Point(0.0f, 1.0f), 8665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn new Point(1.0f, 1.0f), 8765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn new Point(0.0f, 0.0f), 8865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn new Point(1.0f, 0.0f)); 8965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 9065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "targetQuad", hasDefault = true) 9165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private Quad mTargetQuad = new Quad(new Point(0.0f, 0.0f), 9265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn new Point(1.0f, 0.0f), 9365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn new Point(0.0f, 1.0f), 9465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn new Point(1.0f, 1.0f)); 9565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 9665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mSurfaceId; 9765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 9865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private ShaderProgram mProgram; 9965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private GLFrame mScreen; 10065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mRenderMode = RENDERMODE_FIT; 10165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mAspectRatio = 1.f; 10265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 10365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private boolean mLogVerbose; 10465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String TAG = "SurfaceTextureTarget"; 10565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 10665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public SurfaceTextureTarget(String name) { 10765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn super(name); 10865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 10965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 11065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 11165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 11265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 113b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi public synchronized void setupPorts() { 11465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Make sure we have a SurfaceView 11565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mSurfaceTexture == null) { 11665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn throw new RuntimeException("Null SurfaceTexture passed to SurfaceTextureTarget"); 11765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 11865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 11965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Add input port - will accept anything that's 4-channel. 12065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn addMaskedInputPort("frame", ImageFormat.create(ImageFormat.COLORSPACE_RGBA)); 12165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 12265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 12365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void updateRenderMode() { 124b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi if (mLogVerbose) Log.v(TAG, "updateRenderMode. Thread: " + Thread.currentThread()); 12565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mRenderModeString != null) { 12665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mRenderModeString.equals("stretch")) { 12765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mRenderMode = RENDERMODE_STRETCH; 12865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (mRenderModeString.equals("fit")) { 12965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mRenderMode = RENDERMODE_FIT; 13065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (mRenderModeString.equals("fill_crop")) { 13165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mRenderMode = RENDERMODE_FILL_CROP; 13265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (mRenderModeString.equals("customize")) { 13365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mRenderMode = RENDERMODE_CUSTOMIZE; 13465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 13565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn throw new RuntimeException("Unknown render mode '" + mRenderModeString + "'!"); 13665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 13765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 13865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn updateTargetRect(); 13965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 14065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 14165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 14265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void prepare(FilterContext context) { 143b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi if (mLogVerbose) Log.v(TAG, "Prepare. Thread: " + Thread.currentThread()); 14465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Create identity shader to render, and make sure to render upside-down, as textures 14565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // are stored internally bottom-to-top. 14665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mProgram = ShaderProgram.createIdentity(context); 14765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mProgram.setSourceRect(0, 1, 1, -1); 14865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mProgram.setClearColor(0.0f, 0.0f, 0.0f); 14965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 15065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn updateRenderMode(); 15165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 15265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Create a frame representing the screen 15365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn MutableFrameFormat screenFormat = new MutableFrameFormat(FrameFormat.TYPE_BYTE, 15465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn FrameFormat.TARGET_GPU); 15565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn screenFormat.setBytesPerSample(4); 15665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn screenFormat.setDimensions(mScreenWidth, mScreenHeight); 15765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mScreen = (GLFrame)context.getFrameManager().newBoundFrame(screenFormat, 15865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn GLFrame.EXISTING_FBO_BINDING, 15965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 0); 16065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 16165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 16265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 163b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi public synchronized void open(FilterContext context) { 16465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Set up SurfaceTexture internals 16522f2a8728ee2000a01aa6fc6108d8478d7c0ced9Pannag Sanketi if (mSurfaceTexture == null) { 16622f2a8728ee2000a01aa6fc6108d8478d7c0ced9Pannag Sanketi Log.e(TAG, "SurfaceTexture is null!!"); 16722f2a8728ee2000a01aa6fc6108d8478d7c0ced9Pannag Sanketi throw new RuntimeException("Could not register SurfaceTexture: " + mSurfaceTexture); 16822f2a8728ee2000a01aa6fc6108d8478d7c0ced9Pannag Sanketi } 1692f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi mSurfaceId = context.getGLEnvironment().registerSurfaceTexture( 1702f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi mSurfaceTexture, mScreenWidth, mScreenHeight); 1712f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi if (mSurfaceId <= 0) { 1722f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi throw new RuntimeException("Could not register SurfaceTexture: " + mSurfaceTexture); 1732f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi } 17465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 17565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 1762f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi 17722f2a8728ee2000a01aa6fc6108d8478d7c0ced9Pannag Sanketi // Once the surface is unregistered, we still need the surfacetexture reference. 17822f2a8728ee2000a01aa6fc6108d8478d7c0ced9Pannag Sanketi // That is because when the the filter graph stops and starts again, the app 17922f2a8728ee2000a01aa6fc6108d8478d7c0ced9Pannag Sanketi // may not set the mSurfaceTexture again on the filter. In some cases, the app 18022f2a8728ee2000a01aa6fc6108d8478d7c0ced9Pannag Sanketi // may not even know that the graph has re-started. So it is difficult to enforce 18122f2a8728ee2000a01aa6fc6108d8478d7c0ced9Pannag Sanketi // that condition on an app using this filter. The only case where we need 18222f2a8728ee2000a01aa6fc6108d8478d7c0ced9Pannag Sanketi // to let go of the mSurfaceTexure reference is when the app wants to shut 18322f2a8728ee2000a01aa6fc6108d8478d7c0ced9Pannag Sanketi // down the graph on purpose, such as in the disconnect call. 1842f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi @Override 185b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi public synchronized void close(FilterContext context) { 1862f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi if (mSurfaceId > 0) { 1872f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi context.getGLEnvironment().unregisterSurfaceId(mSurfaceId); 188b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi mSurfaceId = -1; 1892f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi } 1902f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi } 1912f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi 192b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi // This should be called from the client side when the surfacetexture is no longer 193b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi // valid. e.g. from onPause() in the application using the filter graph. 19422f2a8728ee2000a01aa6fc6108d8478d7c0ced9Pannag Sanketi // In this case, we need to let go of our surfacetexture reference. 195b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi public synchronized void disconnect(FilterContext context) { 196b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi if (mLogVerbose) Log.v(TAG, "disconnect"); 197b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi if (mSurfaceTexture == null) { 198b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi Log.d(TAG, "SurfaceTexture is already null. Nothing to disconnect."); 199b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi return; 200b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi } 201b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi mSurfaceTexture = null; 202b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi // Make sure we unregister the surface as well if a surface was registered. 203b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi // There can be a situation where the surface was not registered but the 204b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi // surfacetexture was valid. For example, the disconnect can be called before 205b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi // the filter was opened. Hence, the surfaceId may not be a valid one here, 206b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi // and need to check for its validity. 207b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi if (mSurfaceId > 0) { 208b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi context.getGLEnvironment().unregisterSurfaceId(mSurfaceId); 209b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi mSurfaceId = -1; 210b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi } 211b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi } 2122f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi 21365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 214b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi public synchronized void process(FilterContext context) { 215b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi // Surface is not registered. Nothing to render into. 216b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi if (mSurfaceId <= 0) { 217b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi return; 218b939760679caa9fdd06c862cf8218cc8f4a90ef1Pannag Sanketi } 21965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn GLEnvironment glEnv = context.getGLEnvironment(); 22065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 22165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Get input frame 22265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame input = pullInput("frame"); 22365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn boolean createdFrame = false; 22465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 2252f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi float currentAspectRatio = 2262f708ce9cc7fc2e4d498bcc20a095bdf8e9c803dPannag Sanketi (float)input.getFormat().getWidth() / input.getFormat().getHeight(); 22765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (currentAspectRatio != mAspectRatio) { 228b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi if (mLogVerbose) { 229b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi Log.v(TAG, "Process. New aspect ratio: " + currentAspectRatio + 230b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi ", previously: " + mAspectRatio + ". Thread: " + Thread.currentThread()); 231b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi } 23265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mAspectRatio = currentAspectRatio; 23365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn updateTargetRect(); 23465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 23565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 23665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // See if we need to copy to GPU 23765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame gpuFrame = null; 23865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn int target = input.getFormat().getTarget(); 23965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (target != FrameFormat.TARGET_GPU) { 24065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn gpuFrame = context.getFrameManager().duplicateFrameToTarget(input, 24165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn FrameFormat.TARGET_GPU); 24265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn createdFrame = true; 24365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 24465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn gpuFrame = input; 24565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 24665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 24765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Activate our surface 24865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn glEnv.activateSurfaceWithId(mSurfaceId); 24965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 25065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Process 25165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mProgram.process(gpuFrame, mScreen); 25265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 25365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn glEnv.setSurfaceTimestamp(input.getTimestamp()); 25465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 25565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // And swap buffers 25665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn glEnv.swapBuffers(); 25765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 25865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (createdFrame) { 25965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn gpuFrame.release(); 26065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 26165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 26265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 26365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 26465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void fieldPortValueUpdated(String name, FilterContext context) { 265b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi if (mLogVerbose) Log.v(TAG, "FPVU. Thread: " + Thread.currentThread()); 26665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn updateRenderMode(); 26765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 26865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 26965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 27065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void tearDown(FilterContext context) { 27165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mScreen != null) { 27265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mScreen.release(); 27365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 27465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 27565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 27665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private void updateTargetRect() { 277b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi if (mLogVerbose) Log.v(TAG, "updateTargetRect. Thread: " + Thread.currentThread()); 27865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mScreenWidth > 0 && mScreenHeight > 0 && mProgram != null) { 27965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn float screenAspectRatio = (float)mScreenWidth / mScreenHeight; 28065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn float relativeAspectRatio = screenAspectRatio / mAspectRatio; 281b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi if (mLogVerbose) { 282b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi Log.v(TAG, "UTR. screen w = " + (float)mScreenWidth + " x screen h = " + 283b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi (float)mScreenHeight + " Screen AR: " + screenAspectRatio + 284b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi ", frame AR: " + mAspectRatio + ", relative AR: " + relativeAspectRatio); 285b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi } 28665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 28765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (relativeAspectRatio == 1.0f && mRenderMode != RENDERMODE_CUSTOMIZE) { 288b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi mProgram.setTargetRect(0, 0, 1, 1); 28965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mProgram.setClearsOutput(false); 29065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 29165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn switch (mRenderMode) { 29265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn case RENDERMODE_STRETCH: 29365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p0.set(0f, 0.0f); 29465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p1.set(1f, 0.0f); 29565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p2.set(0f, 1.0f); 29665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p3.set(1f, 1.0f); 29765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mProgram.setClearsOutput(false); 29865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn break; 29965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn case RENDERMODE_FIT: 30065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (relativeAspectRatio > 1.0f) { 30165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Screen is wider than the camera, scale down X 30265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p0.set(0.5f - 0.5f / relativeAspectRatio, 0.0f); 30365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p1.set(0.5f + 0.5f / relativeAspectRatio, 0.0f); 30465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p2.set(0.5f - 0.5f / relativeAspectRatio, 1.0f); 30565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p3.set(0.5f + 0.5f / relativeAspectRatio, 1.0f); 30665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 30765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 30865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Screen is taller than the camera, scale down Y 30965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p0.set(0.0f, 0.5f - 0.5f * relativeAspectRatio); 31065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p1.set(1.0f, 0.5f - 0.5f * relativeAspectRatio); 31165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p2.set(0.0f, 0.5f + 0.5f * relativeAspectRatio); 31265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p3.set(1.0f, 0.5f + 0.5f * relativeAspectRatio); 31365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 31465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mProgram.setClearsOutput(true); 31565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn break; 31665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn case RENDERMODE_FILL_CROP: 31765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (relativeAspectRatio > 1) { 31865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Screen is wider than the camera, crop in Y 31965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p0.set(0.0f, 0.5f - 0.5f * relativeAspectRatio); 32065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p1.set(1.0f, 0.5f - 0.5f * relativeAspectRatio); 32165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p2.set(0.0f, 0.5f + 0.5f * relativeAspectRatio); 32265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p3.set(1.0f, 0.5f + 0.5f * relativeAspectRatio); 32365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 32465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Screen is taller than the camera, crop in X 32565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p0.set(0.5f - 0.5f / relativeAspectRatio, 0.0f); 32665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p1.set(0.5f + 0.5f / relativeAspectRatio, 0.0f); 32765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p2.set(0.5f - 0.5f / relativeAspectRatio, 1.0f); 32865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mTargetQuad.p3.set(0.5f + 0.5f / relativeAspectRatio, 1.0f); 32965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 33065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mProgram.setClearsOutput(true); 33165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn break; 33265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn case RENDERMODE_CUSTOMIZE: 33365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn ((ShaderProgram) mProgram).setSourceRegion(mSourceQuad); 33465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn break; 33565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 336b5af71f2b108607149032ce9817c5897b67b4032Pannag Sanketi if (mLogVerbose) Log.v(TAG, "UTR. quad: " + mTargetQuad); 33765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn ((ShaderProgram) mProgram).setTargetRegion(mTargetQuad); 33865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 33965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 34065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 34165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn} 342