1/*
2 * Copyright 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package androidx.media.filterpacks.image;
18
19import androidx.media.filterfw.Filter;
20import androidx.media.filterfw.Frame;
21import androidx.media.filterfw.FrameImage2D;
22import androidx.media.filterfw.FrameType;
23import androidx.media.filterfw.ImageShader;
24import androidx.media.filterfw.MffContext;
25import androidx.media.filterfw.OutputPort;
26import androidx.media.filterfw.Signature;
27
28import java.nio.ByteBuffer;
29
30public class SobelFilter extends Filter {
31
32    private static final String mGradientXSource =
33              "precision mediump float;\n"
34            + "uniform sampler2D tex_sampler_0;\n"
35            + "uniform vec2 pix;\n"
36            + "varying vec2 v_texcoord;\n"
37            + "void main() {\n"
38            + "  vec4 a1 = -1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, -pix.y));\n"
39            + "  vec4 a2 = -2.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, 0.0));\n"
40            + "  vec4 a3 = -1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, +pix.y));\n"
41            + "  vec4 b1 = +1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, -pix.y));\n"
42            + "  vec4 b2 = +2.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, 0.0));\n"
43            + "  vec4 b3 = +1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, +pix.y));\n"
44            + "  gl_FragColor = 0.5 + (a1 + a2 + a3 + b1 + b2 + b3) / 8.0;\n"
45            + "}\n";
46
47    private static final String mGradientYSource =
48              "precision mediump float;\n"
49            + "uniform sampler2D tex_sampler_0;\n"
50            + "uniform vec2 pix;\n"
51            + "varying vec2 v_texcoord;\n"
52            + "void main() {\n"
53            + "  vec4 a1 = -1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, -pix.y));\n"
54            + "  vec4 a2 = -2.0 * texture2D(tex_sampler_0, v_texcoord + vec2(0.0,    -pix.y));\n"
55            + "  vec4 a3 = -1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, -pix.y));\n"
56            + "  vec4 b1 = +1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, +pix.y));\n"
57            + "  vec4 b2 = +2.0 * texture2D(tex_sampler_0, v_texcoord + vec2(0.0,    +pix.y));\n"
58            + "  vec4 b3 = +1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, +pix.y));\n"
59            + "  gl_FragColor = 0.5 + (a1 + a2 + a3 + b1 + b2 + b3) / 8.0;\n"
60            + "}\n";
61
62    private static final String mMagnitudeSource =
63            "precision mediump float;\n"
64          + "uniform sampler2D tex_sampler_0;\n"
65          + "uniform sampler2D tex_sampler_1;\n"
66          + "varying vec2 v_texcoord;\n"
67          + "void main() {\n"
68          + "  vec4 gx = 2.0 * texture2D(tex_sampler_0, v_texcoord) - 1.0;\n"
69          + "  vec4 gy = 2.0 * texture2D(tex_sampler_1, v_texcoord) - 1.0;\n"
70          + "  gl_FragColor = vec4(sqrt(gx.rgb * gx.rgb + gy.rgb * gy.rgb), 1.0);\n"
71          + "}\n";
72
73    private static final String mDirectionSource =
74            "precision mediump float;\n"
75          + "uniform sampler2D tex_sampler_0;\n"
76          + "uniform sampler2D tex_sampler_1;\n"
77          + "varying vec2 v_texcoord;\n"
78          + "void main() {\n"
79          + "  vec4 gy = 2.0 * texture2D(tex_sampler_1, v_texcoord) - 1.0;\n"
80          + "  vec4 gx = 2.0 * texture2D(tex_sampler_0, v_texcoord) - 1.0;\n"
81          + "  gl_FragColor = vec4((atan(gy.rgb, gx.rgb) + 3.14) / (2.0 * 3.14), 1.0);\n"
82          + "}\n";
83
84    private ImageShader mGradientXShader;
85    private ImageShader mGradientYShader;
86    private ImageShader mMagnitudeShader;
87    private ImageShader mDirectionShader;
88
89    private FrameType mImageType;
90
91    public SobelFilter(MffContext context, String name) {
92        super(context, name);
93    }
94
95    @Override
96    public Signature getSignature() {
97        // TODO: we will address the issue of READ_GPU / WRITE_GPU when using CPU filters later.
98        FrameType imageIn = FrameType.image2D(FrameType.ELEMENT_RGBA8888, FrameType.READ_GPU);
99        FrameType imageOut = FrameType.image2D(FrameType.ELEMENT_RGBA8888, FrameType.WRITE_GPU);
100        return new Signature().addInputPort("image", Signature.PORT_REQUIRED, imageIn)
101                .addOutputPort("direction", Signature.PORT_OPTIONAL, imageOut)
102                .addOutputPort("magnitude", Signature.PORT_OPTIONAL, imageOut).disallowOtherPorts();
103    }
104
105    @Override
106    protected void onPrepare() {
107        if (isOpenGLSupported()) {
108            mGradientXShader = new ImageShader(mGradientXSource);
109            mGradientYShader = new ImageShader(mGradientYSource);
110            mMagnitudeShader = new ImageShader(mMagnitudeSource);
111            mDirectionShader = new ImageShader(mDirectionSource);
112            mImageType = FrameType.image2D(
113                    FrameType.ELEMENT_RGBA8888, FrameType.READ_GPU | FrameType.WRITE_GPU);
114        }
115    }
116
117    @Override
118    protected void onProcess() {
119        OutputPort magnitudePort = getConnectedOutputPort("magnitude");
120        OutputPort directionPort = getConnectedOutputPort("direction");
121        FrameImage2D inputImage = getConnectedInputPort("image").pullFrame().asFrameImage2D();
122        int[] inputDims = inputImage.getDimensions();
123
124        FrameImage2D magImage = (magnitudePort != null) ?
125                magnitudePort.fetchAvailableFrame(inputDims).asFrameImage2D() : null;
126        FrameImage2D dirImage = (directionPort != null) ?
127                directionPort.fetchAvailableFrame(inputDims).asFrameImage2D() : null;
128        if (isOpenGLSupported()) {
129            FrameImage2D gxFrame = Frame.create(mImageType, inputDims).asFrameImage2D();
130            FrameImage2D gyFrame = Frame.create(mImageType, inputDims).asFrameImage2D();
131            mGradientXShader.setUniformValue("pix", new float[] {1f/inputDims[0], 1f/inputDims[1]});
132            mGradientYShader.setUniformValue("pix", new float[] {1f/inputDims[0], 1f/inputDims[1]});
133            mGradientXShader.process(inputImage, gxFrame);
134            mGradientYShader.process(inputImage, gyFrame);
135            FrameImage2D[] gradientFrames = new FrameImage2D[] { gxFrame, gyFrame };
136            if (magnitudePort != null) {
137                mMagnitudeShader.processMulti(gradientFrames, magImage);
138            }
139            if (directionPort != null) {
140                mDirectionShader.processMulti(gradientFrames, dirImage);
141            }
142            gxFrame.release();
143            gyFrame.release();
144        } else {
145            ByteBuffer inputBuffer  = inputImage.lockBytes(Frame.MODE_READ);
146            ByteBuffer magBuffer  = (magImage != null) ?
147                    magImage.lockBytes(Frame.MODE_WRITE) : null;
148            ByteBuffer dirBuffer  = (dirImage != null) ?
149                    dirImage.lockBytes(Frame.MODE_WRITE) : null;
150            sobelOperator(inputImage.getWidth(), inputImage.getHeight(),
151                    inputBuffer, magBuffer, dirBuffer);
152            inputImage.unlock();
153            if (magImage != null) {
154                magImage.unlock();
155            }
156            if (dirImage != null) {
157                dirImage.unlock();
158            }
159        }
160        if (magImage != null) {
161            magnitudePort.pushFrame(magImage);
162        }
163        if (dirImage != null) {
164            directionPort.pushFrame(dirImage);
165        }
166    }
167
168    private static native boolean sobelOperator(int width, int height,
169            ByteBuffer imageBuffer, ByteBuffer magBuffer, ByteBuffer dirBudder);
170
171    static {
172        System.loadLibrary("smartcamera_jni");
173    }
174}
175