1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 18package android.filterpacks.videosrc; 19 20import android.filterfw.core.Filter; 21import android.filterfw.core.FilterContext; 22import android.filterfw.core.FilterSurfaceView; 23import android.filterfw.core.Frame; 24import android.filterfw.core.FrameFormat; 25import android.filterfw.core.GenerateFieldPort; 26import android.filterfw.core.GenerateFinalPort; 27import android.filterfw.core.GLEnvironment; 28import android.filterfw.core.GLFrame; 29import android.filterfw.core.KeyValueMap; 30import android.filterfw.core.MutableFrameFormat; 31import android.filterfw.core.NativeProgram; 32import android.filterfw.core.NativeFrame; 33import android.filterfw.core.Program; 34import android.filterfw.core.ShaderProgram; 35import android.filterfw.format.ImageFormat; 36 37import android.filterfw.geometry.Quad; 38import android.filterfw.geometry.Point; 39 40import android.view.Surface; 41import android.view.SurfaceHolder; 42import android.view.SurfaceView; 43 44import android.graphics.Rect; 45import android.graphics.SurfaceTexture; 46 47import android.util.Log; 48 49/** 50 * @hide 51 */ 52public class SurfaceTextureTarget extends Filter { 53 54 private final int RENDERMODE_STRETCH = 0; 55 private final int RENDERMODE_FIT = 1; 56 private final int RENDERMODE_FILL_CROP = 2; 57 private final int RENDERMODE_CUSTOMIZE = 3; 58 59 /** Required. Sets the destination surfaceTexture. 60 */ 61 @GenerateFinalPort(name = "surfaceTexture") 62 private SurfaceTexture mSurfaceTexture; 63 64 /** Required. Sets the width of the output surfaceTexture images */ 65 @GenerateFinalPort(name = "width") 66 private int mScreenWidth; 67 68 /** Required. Sets the height of the output surfaceTexture images */ 69 @GenerateFinalPort(name = "height") 70 private int mScreenHeight; 71 72 73 /** Optional. Control how the incoming frames are rendered onto the 74 * output. Default is FIT. 75 * RENDERMODE_STRETCH: Just fill the output surfaceView. 76 * RENDERMODE_FIT: Keep aspect ratio and fit without cropping. May 77 * have black bars. 78 * RENDERMODE_FILL_CROP: Keep aspect ratio and fit without black 79 * bars. May crop. 80 */ 81 @GenerateFieldPort(name = "renderMode", hasDefault = true) 82 private String mRenderModeString; 83 84 @GenerateFieldPort(name = "sourceQuad", hasDefault = true) 85 private Quad mSourceQuad = new Quad(new Point(0.0f, 1.0f), 86 new Point(1.0f, 1.0f), 87 new Point(0.0f, 0.0f), 88 new Point(1.0f, 0.0f)); 89 90 @GenerateFieldPort(name = "targetQuad", hasDefault = true) 91 private Quad mTargetQuad = new Quad(new Point(0.0f, 0.0f), 92 new Point(1.0f, 0.0f), 93 new Point(0.0f, 1.0f), 94 new Point(1.0f, 1.0f)); 95 96 private int mSurfaceId; 97 98 private ShaderProgram mProgram; 99 private GLFrame mScreen; 100 private int mRenderMode = RENDERMODE_FIT; 101 private float mAspectRatio = 1.f; 102 103 private boolean mLogVerbose; 104 private static final String TAG = "SurfaceTextureTarget"; 105 106 public SurfaceTextureTarget(String name) { 107 super(name); 108 109 mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 110 } 111 112 @Override 113 public synchronized void setupPorts() { 114 // Make sure we have a SurfaceView 115 if (mSurfaceTexture == null) { 116 throw new RuntimeException("Null SurfaceTexture passed to SurfaceTextureTarget"); 117 } 118 119 // Add input port - will accept anything that's 4-channel. 120 addMaskedInputPort("frame", ImageFormat.create(ImageFormat.COLORSPACE_RGBA)); 121 } 122 123 public void updateRenderMode() { 124 if (mLogVerbose) Log.v(TAG, "updateRenderMode. Thread: " + Thread.currentThread()); 125 if (mRenderModeString != null) { 126 if (mRenderModeString.equals("stretch")) { 127 mRenderMode = RENDERMODE_STRETCH; 128 } else if (mRenderModeString.equals("fit")) { 129 mRenderMode = RENDERMODE_FIT; 130 } else if (mRenderModeString.equals("fill_crop")) { 131 mRenderMode = RENDERMODE_FILL_CROP; 132 } else if (mRenderModeString.equals("customize")) { 133 mRenderMode = RENDERMODE_CUSTOMIZE; 134 } else { 135 throw new RuntimeException("Unknown render mode '" + mRenderModeString + "'!"); 136 } 137 } 138 updateTargetRect(); 139 } 140 141 @Override 142 public void prepare(FilterContext context) { 143 if (mLogVerbose) Log.v(TAG, "Prepare. Thread: " + Thread.currentThread()); 144 // Create identity shader to render, and make sure to render upside-down, as textures 145 // are stored internally bottom-to-top. 146 mProgram = ShaderProgram.createIdentity(context); 147 mProgram.setSourceRect(0, 1, 1, -1); 148 mProgram.setClearColor(0.0f, 0.0f, 0.0f); 149 150 updateRenderMode(); 151 152 // Create a frame representing the screen 153 MutableFrameFormat screenFormat = new MutableFrameFormat(FrameFormat.TYPE_BYTE, 154 FrameFormat.TARGET_GPU); 155 screenFormat.setBytesPerSample(4); 156 screenFormat.setDimensions(mScreenWidth, mScreenHeight); 157 mScreen = (GLFrame)context.getFrameManager().newBoundFrame(screenFormat, 158 GLFrame.EXISTING_FBO_BINDING, 159 0); 160 } 161 162 @Override 163 public synchronized void open(FilterContext context) { 164 // Set up SurfaceTexture internals 165 if (mSurfaceTexture == null) { 166 Log.e(TAG, "SurfaceTexture is null!!"); 167 throw new RuntimeException("Could not register SurfaceTexture: " + mSurfaceTexture); 168 } 169 mSurfaceId = context.getGLEnvironment().registerSurfaceTexture( 170 mSurfaceTexture, mScreenWidth, mScreenHeight); 171 if (mSurfaceId <= 0) { 172 throw new RuntimeException("Could not register SurfaceTexture: " + mSurfaceTexture); 173 } 174 } 175 176 177 // Once the surface is unregistered, we still need the surfacetexture reference. 178 // That is because when the the filter graph stops and starts again, the app 179 // may not set the mSurfaceTexture again on the filter. In some cases, the app 180 // may not even know that the graph has re-started. So it is difficult to enforce 181 // that condition on an app using this filter. The only case where we need 182 // to let go of the mSurfaceTexure reference is when the app wants to shut 183 // down the graph on purpose, such as in the disconnect call. 184 @Override 185 public synchronized void close(FilterContext context) { 186 if (mSurfaceId > 0) { 187 context.getGLEnvironment().unregisterSurfaceId(mSurfaceId); 188 mSurfaceId = -1; 189 } 190 } 191 192 // This should be called from the client side when the surfacetexture is no longer 193 // valid. e.g. from onPause() in the application using the filter graph. 194 // In this case, we need to let go of our surfacetexture reference. 195 public synchronized void disconnect(FilterContext context) { 196 if (mLogVerbose) Log.v(TAG, "disconnect"); 197 if (mSurfaceTexture == null) { 198 Log.d(TAG, "SurfaceTexture is already null. Nothing to disconnect."); 199 return; 200 } 201 mSurfaceTexture = null; 202 // Make sure we unregister the surface as well if a surface was registered. 203 // There can be a situation where the surface was not registered but the 204 // surfacetexture was valid. For example, the disconnect can be called before 205 // the filter was opened. Hence, the surfaceId may not be a valid one here, 206 // and need to check for its validity. 207 if (mSurfaceId > 0) { 208 context.getGLEnvironment().unregisterSurfaceId(mSurfaceId); 209 mSurfaceId = -1; 210 } 211 } 212 213 @Override 214 public synchronized void process(FilterContext context) { 215 // Surface is not registered. Nothing to render into. 216 if (mSurfaceId <= 0) { 217 return; 218 } 219 GLEnvironment glEnv = context.getGLEnvironment(); 220 221 // Get input frame 222 Frame input = pullInput("frame"); 223 boolean createdFrame = false; 224 225 float currentAspectRatio = 226 (float)input.getFormat().getWidth() / input.getFormat().getHeight(); 227 if (currentAspectRatio != mAspectRatio) { 228 if (mLogVerbose) { 229 Log.v(TAG, "Process. New aspect ratio: " + currentAspectRatio + 230 ", previously: " + mAspectRatio + ". Thread: " + Thread.currentThread()); 231 } 232 mAspectRatio = currentAspectRatio; 233 updateTargetRect(); 234 } 235 236 // See if we need to copy to GPU 237 Frame gpuFrame = null; 238 int target = input.getFormat().getTarget(); 239 if (target != FrameFormat.TARGET_GPU) { 240 gpuFrame = context.getFrameManager().duplicateFrameToTarget(input, 241 FrameFormat.TARGET_GPU); 242 createdFrame = true; 243 } else { 244 gpuFrame = input; 245 } 246 247 // Activate our surface 248 glEnv.activateSurfaceWithId(mSurfaceId); 249 250 // Process 251 mProgram.process(gpuFrame, mScreen); 252 253 glEnv.setSurfaceTimestamp(input.getTimestamp()); 254 255 // And swap buffers 256 glEnv.swapBuffers(); 257 258 if (createdFrame) { 259 gpuFrame.release(); 260 } 261 } 262 263 @Override 264 public void fieldPortValueUpdated(String name, FilterContext context) { 265 if (mLogVerbose) Log.v(TAG, "FPVU. Thread: " + Thread.currentThread()); 266 updateRenderMode(); 267 } 268 269 @Override 270 public void tearDown(FilterContext context) { 271 if (mScreen != null) { 272 mScreen.release(); 273 } 274 } 275 276 private void updateTargetRect() { 277 if (mLogVerbose) Log.v(TAG, "updateTargetRect. Thread: " + Thread.currentThread()); 278 if (mScreenWidth > 0 && mScreenHeight > 0 && mProgram != null) { 279 float screenAspectRatio = (float)mScreenWidth / mScreenHeight; 280 float relativeAspectRatio = screenAspectRatio / mAspectRatio; 281 if (mLogVerbose) { 282 Log.v(TAG, "UTR. screen w = " + (float)mScreenWidth + " x screen h = " + 283 (float)mScreenHeight + " Screen AR: " + screenAspectRatio + 284 ", frame AR: " + mAspectRatio + ", relative AR: " + relativeAspectRatio); 285 } 286 287 if (relativeAspectRatio == 1.0f && mRenderMode != RENDERMODE_CUSTOMIZE) { 288 mProgram.setTargetRect(0, 0, 1, 1); 289 mProgram.setClearsOutput(false); 290 } else { 291 switch (mRenderMode) { 292 case RENDERMODE_STRETCH: 293 mTargetQuad.p0.set(0f, 0.0f); 294 mTargetQuad.p1.set(1f, 0.0f); 295 mTargetQuad.p2.set(0f, 1.0f); 296 mTargetQuad.p3.set(1f, 1.0f); 297 mProgram.setClearsOutput(false); 298 break; 299 case RENDERMODE_FIT: 300 if (relativeAspectRatio > 1.0f) { 301 // Screen is wider than the camera, scale down X 302 mTargetQuad.p0.set(0.5f - 0.5f / relativeAspectRatio, 0.0f); 303 mTargetQuad.p1.set(0.5f + 0.5f / relativeAspectRatio, 0.0f); 304 mTargetQuad.p2.set(0.5f - 0.5f / relativeAspectRatio, 1.0f); 305 mTargetQuad.p3.set(0.5f + 0.5f / relativeAspectRatio, 1.0f); 306 307 } else { 308 // Screen is taller than the camera, scale down Y 309 mTargetQuad.p0.set(0.0f, 0.5f - 0.5f * relativeAspectRatio); 310 mTargetQuad.p1.set(1.0f, 0.5f - 0.5f * relativeAspectRatio); 311 mTargetQuad.p2.set(0.0f, 0.5f + 0.5f * relativeAspectRatio); 312 mTargetQuad.p3.set(1.0f, 0.5f + 0.5f * relativeAspectRatio); 313 } 314 mProgram.setClearsOutput(true); 315 break; 316 case RENDERMODE_FILL_CROP: 317 if (relativeAspectRatio > 1) { 318 // Screen is wider than the camera, crop in Y 319 mTargetQuad.p0.set(0.0f, 0.5f - 0.5f * relativeAspectRatio); 320 mTargetQuad.p1.set(1.0f, 0.5f - 0.5f * relativeAspectRatio); 321 mTargetQuad.p2.set(0.0f, 0.5f + 0.5f * relativeAspectRatio); 322 mTargetQuad.p3.set(1.0f, 0.5f + 0.5f * relativeAspectRatio); 323 } else { 324 // Screen is taller than the camera, crop in X 325 mTargetQuad.p0.set(0.5f - 0.5f / relativeAspectRatio, 0.0f); 326 mTargetQuad.p1.set(0.5f + 0.5f / relativeAspectRatio, 0.0f); 327 mTargetQuad.p2.set(0.5f - 0.5f / relativeAspectRatio, 1.0f); 328 mTargetQuad.p3.set(0.5f + 0.5f / relativeAspectRatio, 1.0f); 329 } 330 mProgram.setClearsOutput(true); 331 break; 332 case RENDERMODE_CUSTOMIZE: 333 ((ShaderProgram) mProgram).setSourceRegion(mSourceQuad); 334 break; 335 } 336 if (mLogVerbose) Log.v(TAG, "UTR. quad: " + mTargetQuad); 337 ((ShaderProgram) mProgram).setTargetRegion(mTargetQuad); 338 } 339 } 340 } 341} 342