1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17
18package android.filterpacks.ui;
19
20import android.filterfw.core.Filter;
21import android.filterfw.core.FilterContext;
22import android.filterfw.core.FilterSurfaceView;
23import android.filterfw.core.Frame;
24import android.filterfw.core.FrameFormat;
25import android.filterfw.core.GenerateFieldPort;
26import android.filterfw.core.GenerateFinalPort;
27import android.filterfw.core.GLEnvironment;
28import android.filterfw.core.GLFrame;
29import android.filterfw.core.KeyValueMap;
30import android.filterfw.core.MutableFrameFormat;
31import android.filterfw.core.NativeProgram;
32import android.filterfw.core.NativeFrame;
33import android.filterfw.core.Program;
34import android.filterfw.core.ShaderProgram;
35import android.filterfw.format.ImageFormat;
36
37import android.view.Surface;
38import android.view.SurfaceHolder;
39import android.view.SurfaceView;
40
41import android.graphics.Rect;
42
43import android.util.Log;
44
45/**
46 * @hide
47 */
48public class SurfaceRenderFilter extends Filter implements SurfaceHolder.Callback {
49
50    private final int RENDERMODE_STRETCH   = 0;
51    private final int RENDERMODE_FIT       = 1;
52    private final int RENDERMODE_FILL_CROP = 2;
53
54    /** Required. Sets the destination filter surface view for this
55     * node.
56     */
57    @GenerateFinalPort(name = "surfaceView")
58    private FilterSurfaceView mSurfaceView;
59
60    /** Optional. Control how the incoming frames are rendered onto the
61     * output. Default is FIT.
62     * RENDERMODE_STRETCH: Just fill the output surfaceView.
63     * RENDERMODE_FIT: Keep aspect ratio and fit without cropping. May
64     * have black bars.
65     * RENDERMODE_FILL_CROP: Keep aspect ratio and fit without black
66     * bars. May crop.
67     */
68    @GenerateFieldPort(name = "renderMode", hasDefault = true)
69    private String mRenderModeString;
70
71    private boolean mIsBound = false;
72
73    private ShaderProgram mProgram;
74    private GLFrame mScreen;
75    private int mRenderMode = RENDERMODE_FIT;
76    private float mAspectRatio = 1.f;
77
78    private int mScreenWidth;
79    private int mScreenHeight;
80
81    private boolean mLogVerbose;
82    private static final String TAG = "SurfaceRenderFilter";
83
84    public SurfaceRenderFilter(String name) {
85        super(name);
86
87        mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
88    }
89
90    @Override
91    public void setupPorts() {
92        // Make sure we have a SurfaceView
93        if (mSurfaceView == null) {
94            throw new RuntimeException("NULL SurfaceView passed to SurfaceRenderFilter");
95        }
96
97        // Add input port
98        addMaskedInputPort("frame", ImageFormat.create(ImageFormat.COLORSPACE_RGBA));
99    }
100
101    public void updateRenderMode() {
102        if (mRenderModeString != null) {
103            if (mRenderModeString.equals("stretch")) {
104                mRenderMode = RENDERMODE_STRETCH;
105            } else if (mRenderModeString.equals("fit")) {
106                mRenderMode = RENDERMODE_FIT;
107            } else if (mRenderModeString.equals("fill_crop")) {
108                mRenderMode = RENDERMODE_FILL_CROP;
109            } else {
110                throw new RuntimeException("Unknown render mode '" + mRenderModeString + "'!");
111            }
112        }
113        updateTargetRect();
114    }
115
116    @Override
117    public void prepare(FilterContext context) {
118        // Create identity shader to render, and make sure to render upside-down, as textures
119        // are stored internally bottom-to-top.
120        mProgram = ShaderProgram.createIdentity(context);
121        mProgram.setSourceRect(0, 1, 1, -1);
122        mProgram.setClearsOutput(true);
123        mProgram.setClearColor(0.0f, 0.0f, 0.0f);
124
125        updateRenderMode();
126
127        // Create a frame representing the screen
128        MutableFrameFormat screenFormat = ImageFormat.create(mSurfaceView.getWidth(),
129                                                             mSurfaceView.getHeight(),
130                                                             ImageFormat.COLORSPACE_RGBA,
131                                                             FrameFormat.TARGET_GPU);
132        mScreen = (GLFrame)context.getFrameManager().newBoundFrame(screenFormat,
133                                                                   GLFrame.EXISTING_FBO_BINDING,
134                                                                   0);
135    }
136
137    @Override
138    public void open(FilterContext context) {
139        // Bind surface view to us. This will emit a surfaceCreated and surfaceChanged call that
140        // will update our screen width and height.
141        mSurfaceView.unbind();
142        mSurfaceView.bindToListener(this, context.getGLEnvironment());
143    }
144
145    @Override
146    public void process(FilterContext context) {
147        // Make sure we are bound to a surface before rendering
148        if (!mIsBound) {
149            Log.w("SurfaceRenderFilter",
150                  this + ": Ignoring frame as there is no surface to render to!");
151            return;
152        }
153
154        if (mLogVerbose) Log.v(TAG, "Starting frame processing");
155
156        GLEnvironment glEnv = mSurfaceView.getGLEnv();
157        if (glEnv != context.getGLEnvironment()) {
158            throw new RuntimeException("Surface created under different GLEnvironment!");
159        }
160
161
162        // Get input frame
163        Frame input = pullInput("frame");
164        boolean createdFrame = false;
165
166        float currentAspectRatio = (float)input.getFormat().getWidth() / input.getFormat().getHeight();
167        if (currentAspectRatio != mAspectRatio) {
168            if (mLogVerbose) Log.v(TAG, "New aspect ratio: " + currentAspectRatio +", previously: " + mAspectRatio);
169            mAspectRatio = currentAspectRatio;
170            updateTargetRect();
171        }
172
173        // See if we need to copy to GPU
174        Frame gpuFrame = null;
175        if (mLogVerbose) Log.v("SurfaceRenderFilter", "Got input format: " + input.getFormat());
176        int target = input.getFormat().getTarget();
177        if (target != FrameFormat.TARGET_GPU) {
178            gpuFrame = context.getFrameManager().duplicateFrameToTarget(input,
179                                                                        FrameFormat.TARGET_GPU);
180            createdFrame = true;
181        } else {
182            gpuFrame = input;
183        }
184
185        // Activate our surface
186        glEnv.activateSurfaceWithId(mSurfaceView.getSurfaceId());
187
188        // Process
189        mProgram.process(gpuFrame, mScreen);
190
191        // And swap buffers
192        glEnv.swapBuffers();
193
194        if (createdFrame) {
195            gpuFrame.release();
196        }
197    }
198
199    @Override
200    public void fieldPortValueUpdated(String name, FilterContext context) {
201        updateTargetRect();
202    }
203
204    @Override
205    public void close(FilterContext context) {
206        mSurfaceView.unbind();
207    }
208
209    @Override
210    public void tearDown(FilterContext context) {
211        if (mScreen != null) {
212            mScreen.release();
213        }
214    }
215
216    @Override
217    public synchronized void surfaceCreated(SurfaceHolder holder) {
218        mIsBound = true;
219    }
220
221    @Override
222    public synchronized void surfaceChanged(SurfaceHolder holder,
223                                            int format,
224                                            int width,
225                                            int height) {
226        // If the screen is null, we do not care about surface changes (yet). Once we have a
227        // screen object, we need to keep track of these changes.
228        if (mScreen != null) {
229            mScreenWidth = width;
230            mScreenHeight = height;
231            mScreen.setViewport(0, 0, mScreenWidth, mScreenHeight);
232            updateTargetRect();
233        }
234    }
235
236    @Override
237    public synchronized void surfaceDestroyed(SurfaceHolder holder) {
238        mIsBound = false;
239    }
240
241    private void updateTargetRect() {
242        if (mScreenWidth > 0 && mScreenHeight > 0 && mProgram != null) {
243            float screenAspectRatio = (float)mScreenWidth / mScreenHeight;
244            float relativeAspectRatio = screenAspectRatio / mAspectRatio;
245
246            switch (mRenderMode) {
247                case RENDERMODE_STRETCH:
248                    mProgram.setTargetRect(0, 0, 1, 1);
249                    break;
250                case RENDERMODE_FIT:
251                    if (relativeAspectRatio > 1.0f) {
252                        // Screen is wider than the camera, scale down X
253                        mProgram.setTargetRect(0.5f - 0.5f / relativeAspectRatio, 0.0f,
254                                               1.0f / relativeAspectRatio, 1.0f);
255                    } else {
256                        // Screen is taller than the camera, scale down Y
257                        mProgram.setTargetRect(0.0f, 0.5f - 0.5f * relativeAspectRatio,
258                                               1.0f, relativeAspectRatio);
259                    }
260                    break;
261                case RENDERMODE_FILL_CROP:
262                    if (relativeAspectRatio > 1) {
263                        // Screen is wider than the camera, crop in Y
264                        mProgram.setTargetRect(0.0f, 0.5f - 0.5f * relativeAspectRatio,
265                                               1.0f, relativeAspectRatio);
266                    } else {
267                        // Screen is taller than the camera, crop in X
268                        mProgram.setTargetRect(0.5f - 0.5f / relativeAspectRatio, 0.0f,
269                                               1.0f / relativeAspectRatio, 1.0f);
270                    }
271                    break;
272            }
273        }
274    }
275}
276