1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 18package android.filterpacks.videosrc; 19 20import android.filterfw.core.Filter; 21import android.filterfw.core.FilterContext; 22import android.filterfw.core.Frame; 23import android.filterfw.core.FrameFormat; 24import android.filterfw.core.GenerateFieldPort; 25import android.filterfw.core.GenerateFinalPort; 26import android.filterfw.core.GLEnvironment; 27import android.filterfw.core.GLFrame; 28import android.filterfw.core.MutableFrameFormat; 29import android.filterfw.core.ShaderProgram; 30import android.filterfw.format.ImageFormat; 31 32import android.filterfw.geometry.Quad; 33import android.filterfw.geometry.Point; 34 35import android.graphics.SurfaceTexture; 36 37import android.util.Log; 38 39/** 40 * @hide 41 */ 42public class SurfaceTextureTarget extends Filter { 43 44 private final int RENDERMODE_STRETCH = 0; 45 private final int RENDERMODE_FIT = 1; 46 private final int RENDERMODE_FILL_CROP = 2; 47 private final int RENDERMODE_CUSTOMIZE = 3; 48 49 /** Required. Sets the destination surfaceTexture. 50 */ 51 @GenerateFinalPort(name = "surfaceTexture") 52 private SurfaceTexture mSurfaceTexture; 53 54 /** Required. Sets the width of the output surfaceTexture images */ 55 @GenerateFinalPort(name = "width") 56 private int mScreenWidth; 57 58 /** Required. Sets the height of the output surfaceTexture images */ 59 @GenerateFinalPort(name = "height") 60 private int mScreenHeight; 61 62 63 /** Optional. Control how the incoming frames are rendered onto the 64 * output. Default is FIT. 65 * RENDERMODE_STRETCH: Just fill the output surfaceView. 66 * RENDERMODE_FIT: Keep aspect ratio and fit without cropping. May 67 * have black bars. 68 * RENDERMODE_FILL_CROP: Keep aspect ratio and fit without black 69 * bars. May crop. 70 */ 71 @GenerateFieldPort(name = "renderMode", hasDefault = true) 72 private String mRenderModeString; 73 74 @GenerateFieldPort(name = "sourceQuad", hasDefault = true) 75 private Quad mSourceQuad = new Quad(new Point(0.0f, 1.0f), 76 new Point(1.0f, 1.0f), 77 new Point(0.0f, 0.0f), 78 new Point(1.0f, 0.0f)); 79 80 @GenerateFieldPort(name = "targetQuad", hasDefault = true) 81 private Quad mTargetQuad = new Quad(new Point(0.0f, 0.0f), 82 new Point(1.0f, 0.0f), 83 new Point(0.0f, 1.0f), 84 new Point(1.0f, 1.0f)); 85 86 private int mSurfaceId; 87 88 private ShaderProgram mProgram; 89 private GLFrame mScreen; 90 private int mRenderMode = RENDERMODE_FIT; 91 private float mAspectRatio = 1.f; 92 93 private boolean mLogVerbose; 94 private static final String TAG = "SurfaceTextureTarget"; 95 96 public SurfaceTextureTarget(String name) { 97 super(name); 98 99 mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 100 } 101 102 @Override 103 public synchronized void setupPorts() { 104 // Make sure we have a SurfaceView 105 if (mSurfaceTexture == null) { 106 throw new RuntimeException("Null SurfaceTexture passed to SurfaceTextureTarget"); 107 } 108 109 // Add input port - will accept anything that's 4-channel. 110 addMaskedInputPort("frame", ImageFormat.create(ImageFormat.COLORSPACE_RGBA)); 111 } 112 113 public void updateRenderMode() { 114 if (mLogVerbose) Log.v(TAG, "updateRenderMode. Thread: " + Thread.currentThread()); 115 if (mRenderModeString != null) { 116 if (mRenderModeString.equals("stretch")) { 117 mRenderMode = RENDERMODE_STRETCH; 118 } else if (mRenderModeString.equals("fit")) { 119 mRenderMode = RENDERMODE_FIT; 120 } else if (mRenderModeString.equals("fill_crop")) { 121 mRenderMode = RENDERMODE_FILL_CROP; 122 } else if (mRenderModeString.equals("customize")) { 123 mRenderMode = RENDERMODE_CUSTOMIZE; 124 } else { 125 throw new RuntimeException("Unknown render mode '" + mRenderModeString + "'!"); 126 } 127 } 128 updateTargetRect(); 129 } 130 131 @Override 132 public void prepare(FilterContext context) { 133 if (mLogVerbose) Log.v(TAG, "Prepare. Thread: " + Thread.currentThread()); 134 // Create identity shader to render, and make sure to render upside-down, as textures 135 // are stored internally bottom-to-top. 136 mProgram = ShaderProgram.createIdentity(context); 137 mProgram.setSourceRect(0, 1, 1, -1); 138 mProgram.setClearColor(0.0f, 0.0f, 0.0f); 139 140 updateRenderMode(); 141 142 // Create a frame representing the screen 143 MutableFrameFormat screenFormat = new MutableFrameFormat(FrameFormat.TYPE_BYTE, 144 FrameFormat.TARGET_GPU); 145 screenFormat.setBytesPerSample(4); 146 screenFormat.setDimensions(mScreenWidth, mScreenHeight); 147 mScreen = (GLFrame)context.getFrameManager().newBoundFrame(screenFormat, 148 GLFrame.EXISTING_FBO_BINDING, 149 0); 150 } 151 152 @Override 153 public synchronized void open(FilterContext context) { 154 // Set up SurfaceTexture internals 155 if (mSurfaceTexture == null) { 156 Log.e(TAG, "SurfaceTexture is null!!"); 157 throw new RuntimeException("Could not register SurfaceTexture: " + mSurfaceTexture); 158 } 159 mSurfaceId = context.getGLEnvironment().registerSurfaceTexture( 160 mSurfaceTexture, mScreenWidth, mScreenHeight); 161 if (mSurfaceId <= 0) { 162 throw new RuntimeException("Could not register SurfaceTexture: " + mSurfaceTexture); 163 } 164 } 165 166 167 // Once the surface is unregistered, we still need the surfacetexture reference. 168 // That is because when the the filter graph stops and starts again, the app 169 // may not set the mSurfaceTexture again on the filter. In some cases, the app 170 // may not even know that the graph has re-started. So it is difficult to enforce 171 // that condition on an app using this filter. The only case where we need 172 // to let go of the mSurfaceTexure reference is when the app wants to shut 173 // down the graph on purpose, such as in the disconnect call. 174 @Override 175 public synchronized void close(FilterContext context) { 176 if (mSurfaceId > 0) { 177 context.getGLEnvironment().unregisterSurfaceId(mSurfaceId); 178 mSurfaceId = -1; 179 } 180 } 181 182 // This should be called from the client side when the surfacetexture is no longer 183 // valid. e.g. from onPause() in the application using the filter graph. 184 // In this case, we need to let go of our surfacetexture reference. 185 public synchronized void disconnect(FilterContext context) { 186 if (mLogVerbose) Log.v(TAG, "disconnect"); 187 if (mSurfaceTexture == null) { 188 Log.d(TAG, "SurfaceTexture is already null. Nothing to disconnect."); 189 return; 190 } 191 mSurfaceTexture = null; 192 // Make sure we unregister the surface as well if a surface was registered. 193 // There can be a situation where the surface was not registered but the 194 // surfacetexture was valid. For example, the disconnect can be called before 195 // the filter was opened. Hence, the surfaceId may not be a valid one here, 196 // and need to check for its validity. 197 if (mSurfaceId > 0) { 198 context.getGLEnvironment().unregisterSurfaceId(mSurfaceId); 199 mSurfaceId = -1; 200 } 201 } 202 203 @Override 204 public synchronized void process(FilterContext context) { 205 // Surface is not registered. Nothing to render into. 206 if (mSurfaceId <= 0) { 207 return; 208 } 209 GLEnvironment glEnv = context.getGLEnvironment(); 210 211 // Get input frame 212 Frame input = pullInput("frame"); 213 boolean createdFrame = false; 214 215 float currentAspectRatio = 216 (float)input.getFormat().getWidth() / input.getFormat().getHeight(); 217 if (currentAspectRatio != mAspectRatio) { 218 if (mLogVerbose) { 219 Log.v(TAG, "Process. New aspect ratio: " + currentAspectRatio + 220 ", previously: " + mAspectRatio + ". Thread: " + Thread.currentThread()); 221 } 222 mAspectRatio = currentAspectRatio; 223 updateTargetRect(); 224 } 225 226 // See if we need to copy to GPU 227 Frame gpuFrame = null; 228 int target = input.getFormat().getTarget(); 229 if (target != FrameFormat.TARGET_GPU) { 230 gpuFrame = context.getFrameManager().duplicateFrameToTarget(input, 231 FrameFormat.TARGET_GPU); 232 createdFrame = true; 233 } else { 234 gpuFrame = input; 235 } 236 237 // Activate our surface 238 glEnv.activateSurfaceWithId(mSurfaceId); 239 240 // Process 241 mProgram.process(gpuFrame, mScreen); 242 243 glEnv.setSurfaceTimestamp(input.getTimestamp()); 244 245 // And swap buffers 246 glEnv.swapBuffers(); 247 248 if (createdFrame) { 249 gpuFrame.release(); 250 } 251 } 252 253 @Override 254 public void fieldPortValueUpdated(String name, FilterContext context) { 255 if (mLogVerbose) Log.v(TAG, "FPVU. Thread: " + Thread.currentThread()); 256 updateRenderMode(); 257 } 258 259 @Override 260 public void tearDown(FilterContext context) { 261 if (mScreen != null) { 262 mScreen.release(); 263 } 264 } 265 266 private void updateTargetRect() { 267 if (mLogVerbose) Log.v(TAG, "updateTargetRect. Thread: " + Thread.currentThread()); 268 if (mScreenWidth > 0 && mScreenHeight > 0 && mProgram != null) { 269 float screenAspectRatio = (float)mScreenWidth / mScreenHeight; 270 float relativeAspectRatio = screenAspectRatio / mAspectRatio; 271 if (mLogVerbose) { 272 Log.v(TAG, "UTR. screen w = " + (float)mScreenWidth + " x screen h = " + 273 (float)mScreenHeight + " Screen AR: " + screenAspectRatio + 274 ", frame AR: " + mAspectRatio + ", relative AR: " + relativeAspectRatio); 275 } 276 277 if (relativeAspectRatio == 1.0f && mRenderMode != RENDERMODE_CUSTOMIZE) { 278 mProgram.setTargetRect(0, 0, 1, 1); 279 mProgram.setClearsOutput(false); 280 } else { 281 switch (mRenderMode) { 282 case RENDERMODE_STRETCH: 283 mTargetQuad.p0.set(0f, 0.0f); 284 mTargetQuad.p1.set(1f, 0.0f); 285 mTargetQuad.p2.set(0f, 1.0f); 286 mTargetQuad.p3.set(1f, 1.0f); 287 mProgram.setClearsOutput(false); 288 break; 289 case RENDERMODE_FIT: 290 if (relativeAspectRatio > 1.0f) { 291 // Screen is wider than the camera, scale down X 292 mTargetQuad.p0.set(0.5f - 0.5f / relativeAspectRatio, 0.0f); 293 mTargetQuad.p1.set(0.5f + 0.5f / relativeAspectRatio, 0.0f); 294 mTargetQuad.p2.set(0.5f - 0.5f / relativeAspectRatio, 1.0f); 295 mTargetQuad.p3.set(0.5f + 0.5f / relativeAspectRatio, 1.0f); 296 297 } else { 298 // Screen is taller than the camera, scale down Y 299 mTargetQuad.p0.set(0.0f, 0.5f - 0.5f * relativeAspectRatio); 300 mTargetQuad.p1.set(1.0f, 0.5f - 0.5f * relativeAspectRatio); 301 mTargetQuad.p2.set(0.0f, 0.5f + 0.5f * relativeAspectRatio); 302 mTargetQuad.p3.set(1.0f, 0.5f + 0.5f * relativeAspectRatio); 303 } 304 mProgram.setClearsOutput(true); 305 break; 306 case RENDERMODE_FILL_CROP: 307 if (relativeAspectRatio > 1) { 308 // Screen is wider than the camera, crop in Y 309 mTargetQuad.p0.set(0.0f, 0.5f - 0.5f * relativeAspectRatio); 310 mTargetQuad.p1.set(1.0f, 0.5f - 0.5f * relativeAspectRatio); 311 mTargetQuad.p2.set(0.0f, 0.5f + 0.5f * relativeAspectRatio); 312 mTargetQuad.p3.set(1.0f, 0.5f + 0.5f * relativeAspectRatio); 313 } else { 314 // Screen is taller than the camera, crop in X 315 mTargetQuad.p0.set(0.5f - 0.5f / relativeAspectRatio, 0.0f); 316 mTargetQuad.p1.set(0.5f + 0.5f / relativeAspectRatio, 0.0f); 317 mTargetQuad.p2.set(0.5f - 0.5f / relativeAspectRatio, 1.0f); 318 mTargetQuad.p3.set(0.5f + 0.5f / relativeAspectRatio, 1.0f); 319 } 320 mProgram.setClearsOutput(true); 321 break; 322 case RENDERMODE_CUSTOMIZE: 323 ((ShaderProgram) mProgram).setSourceRegion(mSourceQuad); 324 break; 325 } 326 if (mLogVerbose) Log.v(TAG, "UTR. quad: " + mTargetQuad); 327 ((ShaderProgram) mProgram).setTargetRegion(mTargetQuad); 328 } 329 } 330 } 331} 332