1227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks/* 2227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * Copyright 2013 The Android Open Source Project 3227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * 4227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * Licensed under the Apache License, Version 2.0 (the "License"); 5227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * you may not use this file except in compliance with the License. 6227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * You may obtain a copy of the License at 7227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * 8227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * http://www.apache.org/licenses/LICENSE-2.0 9227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * 10227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * Unless required by applicable law or agreed to in writing, software 11227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * distributed under the License is distributed on an "AS IS" BASIS, 12227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * See the License for the specific language governing permissions and 14227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks * limitations under the License. 15227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks */ 16227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 17227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendrickspackage androidx.media.filterpacks.image; 18227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 19227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricksimport androidx.media.filterfw.Filter; 20227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricksimport androidx.media.filterfw.Frame; 21227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricksimport androidx.media.filterfw.FrameImage2D; 22227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricksimport androidx.media.filterfw.FrameType; 23227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricksimport androidx.media.filterfw.ImageShader; 24227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricksimport androidx.media.filterfw.MffContext; 25227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricksimport androidx.media.filterfw.OutputPort; 26227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricksimport androidx.media.filterfw.Signature; 27227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 28227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricksimport java.nio.ByteBuffer; 29227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 30227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendrickspublic class SobelFilter extends Filter { 31227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 32227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks private static final String mGradientXSource = 33227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks "precision mediump float;\n" 34227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "uniform sampler2D tex_sampler_0;\n" 35227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "uniform vec2 pix;\n" 36227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "varying vec2 v_texcoord;\n" 37227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "void main() {\n" 38227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 a1 = -1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, -pix.y));\n" 39227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 a2 = -2.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, 0.0));\n" 40227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 a3 = -1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, +pix.y));\n" 41227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 b1 = +1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, -pix.y));\n" 42227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 b2 = +2.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, 0.0));\n" 43227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 b3 = +1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, +pix.y));\n" 44227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " gl_FragColor = 0.5 + (a1 + a2 + a3 + b1 + b2 + b3) / 8.0;\n" 45227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "}\n"; 46227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 47227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks private static final String mGradientYSource = 48227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks "precision mediump float;\n" 49227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "uniform sampler2D tex_sampler_0;\n" 50227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "uniform vec2 pix;\n" 51227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "varying vec2 v_texcoord;\n" 52227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "void main() {\n" 53227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 a1 = -1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, -pix.y));\n" 54227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 a2 = -2.0 * texture2D(tex_sampler_0, v_texcoord + vec2(0.0, -pix.y));\n" 55227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 a3 = -1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, -pix.y));\n" 56227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 b1 = +1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, +pix.y));\n" 57227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 b2 = +2.0 * texture2D(tex_sampler_0, v_texcoord + vec2(0.0, +pix.y));\n" 58227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 b3 = +1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, +pix.y));\n" 59227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " gl_FragColor = 0.5 + (a1 + a2 + a3 + b1 + b2 + b3) / 8.0;\n" 60227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "}\n"; 61227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 62227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks private static final String mMagnitudeSource = 63227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks "precision mediump float;\n" 64227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "uniform sampler2D tex_sampler_0;\n" 65227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "uniform sampler2D tex_sampler_1;\n" 66227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "varying vec2 v_texcoord;\n" 67227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "void main() {\n" 68227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 gx = 2.0 * texture2D(tex_sampler_0, v_texcoord) - 1.0;\n" 69227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 gy = 2.0 * texture2D(tex_sampler_1, v_texcoord) - 1.0;\n" 70227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " gl_FragColor = vec4(sqrt(gx.rgb * gx.rgb + gy.rgb * gy.rgb), 1.0);\n" 71227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "}\n"; 72227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 73227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks private static final String mDirectionSource = 74227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks "precision mediump float;\n" 75227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "uniform sampler2D tex_sampler_0;\n" 76227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "uniform sampler2D tex_sampler_1;\n" 77227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "varying vec2 v_texcoord;\n" 78227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "void main() {\n" 79227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 gy = 2.0 * texture2D(tex_sampler_1, v_texcoord) - 1.0;\n" 80227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " vec4 gx = 2.0 * texture2D(tex_sampler_0, v_texcoord) - 1.0;\n" 81227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + " gl_FragColor = vec4((atan(gy.rgb, gx.rgb) + 3.14) / (2.0 * 3.14), 1.0);\n" 82227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks + "}\n"; 83227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 84227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks private ImageShader mGradientXShader; 85227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks private ImageShader mGradientYShader; 86227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks private ImageShader mMagnitudeShader; 87227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks private ImageShader mDirectionShader; 88227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 89227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks private FrameType mImageType; 90227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 91227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks public SobelFilter(MffContext context, String name) { 92227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks super(context, name); 93227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 94227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 95227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks @Override 96227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks public Signature getSignature() { 97227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks // TODO: we will address the issue of READ_GPU / WRITE_GPU when using CPU filters later. 98227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks FrameType imageIn = FrameType.image2D(FrameType.ELEMENT_RGBA8888, FrameType.READ_GPU); 99227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks FrameType imageOut = FrameType.image2D(FrameType.ELEMENT_RGBA8888, FrameType.WRITE_GPU); 100227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks return new Signature().addInputPort("image", Signature.PORT_REQUIRED, imageIn) 101227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks .addOutputPort("direction", Signature.PORT_OPTIONAL, imageOut) 102227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks .addOutputPort("magnitude", Signature.PORT_OPTIONAL, imageOut).disallowOtherPorts(); 103227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 104227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 105227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks @Override 106227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks protected void onPrepare() { 107227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks if (isOpenGLSupported()) { 108227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks mGradientXShader = new ImageShader(mGradientXSource); 109227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks mGradientYShader = new ImageShader(mGradientYSource); 110227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks mMagnitudeShader = new ImageShader(mMagnitudeSource); 111227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks mDirectionShader = new ImageShader(mDirectionSource); 112227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks mImageType = FrameType.image2D( 113227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks FrameType.ELEMENT_RGBA8888, FrameType.READ_GPU | FrameType.WRITE_GPU); 114227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 115227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 116227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 117227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks @Override 118227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks protected void onProcess() { 119227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks OutputPort magnitudePort = getConnectedOutputPort("magnitude"); 120227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks OutputPort directionPort = getConnectedOutputPort("direction"); 121227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks FrameImage2D inputImage = getConnectedInputPort("image").pullFrame().asFrameImage2D(); 122227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks int[] inputDims = inputImage.getDimensions(); 123227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 124227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks FrameImage2D magImage = (magnitudePort != null) ? 125227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks magnitudePort.fetchAvailableFrame(inputDims).asFrameImage2D() : null; 126227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks FrameImage2D dirImage = (directionPort != null) ? 127227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks directionPort.fetchAvailableFrame(inputDims).asFrameImage2D() : null; 128227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks if (isOpenGLSupported()) { 129227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks FrameImage2D gxFrame = Frame.create(mImageType, inputDims).asFrameImage2D(); 130227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks FrameImage2D gyFrame = Frame.create(mImageType, inputDims).asFrameImage2D(); 131227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks mGradientXShader.setUniformValue("pix", new float[] {1f/inputDims[0], 1f/inputDims[1]}); 132227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks mGradientYShader.setUniformValue("pix", new float[] {1f/inputDims[0], 1f/inputDims[1]}); 133227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks mGradientXShader.process(inputImage, gxFrame); 134227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks mGradientYShader.process(inputImage, gyFrame); 135227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks FrameImage2D[] gradientFrames = new FrameImage2D[] { gxFrame, gyFrame }; 136227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks if (magnitudePort != null) { 137227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks mMagnitudeShader.processMulti(gradientFrames, magImage); 138227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 139227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks if (directionPort != null) { 140227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks mDirectionShader.processMulti(gradientFrames, dirImage); 141227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 142227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks gxFrame.release(); 143227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks gyFrame.release(); 144227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } else { 145227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks ByteBuffer inputBuffer = inputImage.lockBytes(Frame.MODE_READ); 146227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks ByteBuffer magBuffer = (magImage != null) ? 147227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks magImage.lockBytes(Frame.MODE_WRITE) : null; 148227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks ByteBuffer dirBuffer = (dirImage != null) ? 149227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks dirImage.lockBytes(Frame.MODE_WRITE) : null; 150227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks sobelOperator(inputImage.getWidth(), inputImage.getHeight(), 151227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks inputBuffer, magBuffer, dirBuffer); 152227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks inputImage.unlock(); 153227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks if (magImage != null) { 154227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks magImage.unlock(); 155227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 156227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks if (dirImage != null) { 157227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks dirImage.unlock(); 158227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 159227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 160227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks if (magImage != null) { 161227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks magnitudePort.pushFrame(magImage); 162227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 163227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks if (dirImage != null) { 164227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks directionPort.pushFrame(dirImage); 165227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 166227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 167227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 168227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks private static native boolean sobelOperator(int width, int height, 169227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks ByteBuffer imageBuffer, ByteBuffer magBuffer, ByteBuffer dirBudder); 170227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks 171227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks static { 172227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks System.loadLibrary("smartcamera_jni"); 173227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks } 174227b47625d7482b5b47ad0e4c70ce0a246236adeBenjamin Hendricks} 175