1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.filterpacks.imageproc; 18 19import android.filterfw.core.Filter; 20import android.filterfw.core.FilterContext; 21import android.filterfw.core.Frame; 22import android.filterfw.core.FrameFormat; 23import android.filterfw.core.GenerateFieldPort; 24import android.filterfw.core.NativeProgram; 25import android.filterfw.core.Program; 26import android.filterfw.core.ShaderProgram; 27import android.filterfw.format.ImageFormat; 28 29import android.util.Log; 30 31public class AutoFixFilter extends Filter { 32 33 @GenerateFieldPort(name = "tile_size", hasDefault = true) 34 private int mTileSize = 640; 35 36 @GenerateFieldPort(name = "scale") 37 private float mScale; 38 39 private static final int normal_cdf[] = { 40 9, 33, 50, 64, 75, 84, 92, 99, 106, 112, 117, 122, 126, 130, 134, 138, 142, 41 145, 148, 150, 154, 157, 159, 162, 164, 166, 169, 170, 173, 175, 177, 179, 42 180, 182, 184, 186, 188, 189, 190, 192, 194, 195, 197, 198, 199, 200, 202, 43 203, 205, 206, 207, 208, 209, 210, 212, 213, 214, 215, 216, 217, 218, 219, 44 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 229, 230, 231, 232, 233, 45 234, 235, 236, 236, 237, 238, 239, 239, 240, 240, 242, 242, 243, 244, 245, 46 245, 246, 247, 247, 248, 249, 249, 250, 250, 251, 252, 253, 253, 254, 255, 47 255, 256, 256, 257, 258, 258, 259, 259, 259, 260, 261, 262, 262, 263, 263, 48 264, 264, 265, 265, 266, 267, 267, 268, 268, 269, 269, 269, 270, 270, 271, 49 272, 272, 273, 273, 274, 274, 275, 275, 276, 276, 277, 277, 277, 278, 278, 50 279, 279, 279, 280, 280, 281, 282, 282, 282, 283, 283, 284, 284, 285, 285, 51 285, 286, 286, 287, 287, 288, 288, 288, 289, 289, 289, 290, 290, 290, 291, 52 292, 292, 292, 293, 293, 294, 294, 294, 295, 295, 296, 296, 296, 297, 297, 53 297, 298, 298, 298, 299, 299, 299, 299, 300, 300, 301, 301, 302, 302, 302, 54 303, 303, 304, 304, 304, 305, 305, 305, 306, 306, 306, 307, 307, 307, 308, 55 308, 308, 309, 309, 309, 309, 310, 310, 310, 310, 311, 312, 312, 312, 313, 56 313, 313, 314, 314, 314, 315, 315, 315, 315, 316, 316, 316, 317, 317, 317, 57 318, 318, 318, 319, 319, 319, 319, 319, 320, 320, 320, 321, 321, 322, 322, 58 322, 323, 323, 323, 323, 324, 324, 324, 325, 325, 325, 325, 326, 326, 326, 59 327, 327, 327, 327, 328, 328, 328, 329, 329, 329, 329, 329, 330, 330, 330, 60 330, 331, 331, 332, 332, 332, 333, 333, 333, 333, 334, 334, 334, 334, 335, 61 335, 335, 336, 336, 336, 336, 337, 337, 337, 337, 338, 338, 338, 339, 339, 62 339, 339, 339, 339, 340, 340, 340, 340, 341, 341, 342, 342, 342, 342, 343, 63 343, 343, 344, 344, 344, 344, 345, 345, 345, 345, 346, 346, 346, 346, 347, 64 347, 347, 347, 348, 348, 348, 348, 349, 349, 349, 349, 349, 349, 350, 350, 65 350, 350, 351, 351, 352, 352, 352, 352, 353, 353, 353, 353, 354, 354, 354, 66 354, 355, 355, 355, 355, 356, 356, 356, 356, 357, 357, 357, 357, 358, 358, 67 358, 358, 359, 359, 359, 359, 359, 359, 359, 360, 360, 360, 360, 361, 361, 68 362, 362, 362, 362, 363, 363, 363, 363, 364, 364, 364, 364, 365, 365, 365, 69 365, 366, 366, 366, 366, 366, 367, 367, 367, 367, 368, 368, 368, 368, 369, 70 369, 369, 369, 369, 369, 370, 370, 370, 370, 370, 371, 371, 372, 372, 372, 71 372, 373, 373, 373, 373, 374, 374, 374, 374, 374, 375, 375, 375, 375, 376, 72 376, 376, 376, 377, 377, 377, 377, 378, 378, 378, 378, 378, 379, 379, 379, 73 379, 379, 379, 380, 380, 380, 380, 381, 381, 381, 382, 382, 382, 382, 383, 74 383, 383, 383, 384, 384, 384, 384, 385, 385, 385, 385, 385, 386, 386, 386, 75 386, 387, 387, 387, 387, 388, 388, 388, 388, 388, 389, 389, 389, 389, 389, 76 389, 390, 390, 390, 390, 391, 391, 392, 392, 392, 392, 392, 393, 393, 393, 77 393, 394, 394, 394, 394, 395, 395, 395, 395, 396, 396, 396, 396, 396, 397, 78 397, 397, 397, 398, 398, 398, 398, 399, 399, 399, 399, 399, 399, 400, 400, 79 400, 400, 400, 401, 401, 402, 402, 402, 402, 403, 403, 403, 403, 404, 404, 80 404, 404, 405, 405, 405, 405, 406, 406, 406, 406, 406, 407, 407, 407, 407, 81 408, 408, 408, 408, 409, 409, 409, 409, 409, 409, 410, 410, 410, 410, 411, 82 411, 412, 412, 412, 412, 413, 413, 413, 413, 414, 414, 414, 414, 415, 415, 83 415, 415, 416, 416, 416, 416, 417, 417, 417, 417, 418, 418, 418, 418, 419, 84 419, 419, 419, 419, 419, 420, 420, 420, 420, 421, 421, 422, 422, 422, 422, 85 423, 423, 423, 423, 424, 424, 424, 425, 425, 425, 425, 426, 426, 426, 426, 86 427, 427, 427, 427, 428, 428, 428, 429, 429, 429, 429, 429, 429, 430, 430, 87 430, 430, 431, 431, 432, 432, 432, 433, 433, 433, 433, 434, 434, 434, 435, 88 435, 435, 435, 436, 436, 436, 436, 437, 437, 437, 438, 438, 438, 438, 439, 89 439, 439, 439, 439, 440, 440, 440, 441, 441, 442, 442, 442, 443, 443, 443, 90 443, 444, 444, 444, 445, 445, 445, 446, 446, 446, 446, 447, 447, 447, 448, 91 448, 448, 449, 449, 449, 449, 449, 450, 450, 450, 451, 451, 452, 452, 452, 92 453, 453, 453, 454, 454, 454, 455, 455, 455, 456, 456, 456, 457, 457, 457, 93 458, 458, 458, 459, 459, 459, 459, 460, 460, 460, 461, 461, 462, 462, 462, 94 463, 463, 463, 464, 464, 465, 465, 465, 466, 466, 466, 467, 467, 467, 468, 95 468, 469, 469, 469, 469, 470, 470, 470, 471, 472, 472, 472, 473, 473, 474, 96 474, 474, 475, 475, 476, 476, 476, 477, 477, 478, 478, 478, 479, 479, 479, 97 480, 480, 480, 481, 482, 482, 483, 483, 484, 484, 484, 485, 485, 486, 486, 98 487, 487, 488, 488, 488, 489, 489, 489, 490, 490, 491, 492, 492, 493, 493, 99 494, 494, 495, 495, 496, 496, 497, 497, 498, 498, 499, 499, 499, 500, 501, 100 502, 502, 503, 503, 504, 504, 505, 505, 506, 507, 507, 508, 508, 509, 509, 101 510, 510, 511, 512, 513, 513, 514, 515, 515, 516, 517, 517, 518, 519, 519, 102 519, 520, 521, 522, 523, 524, 524, 525, 526, 526, 527, 528, 529, 529, 530, 103 531, 532, 533, 534, 535, 535, 536, 537, 538, 539, 539, 540, 542, 543, 544, 104 545, 546, 547, 548, 549, 549, 550, 552, 553, 554, 555, 556, 558, 559, 559, 105 561, 562, 564, 565, 566, 568, 569, 570, 572, 574, 575, 577, 578, 579, 582, 106 583, 585, 587, 589, 590, 593, 595, 597, 599, 602, 604, 607, 609, 612, 615, 107 618, 620, 624, 628, 631, 635, 639, 644, 649, 654, 659, 666, 673, 680, 690, 108 700, 714 }; 109 110 private final String mAutoFixShader = 111 "precision mediump float;\n" + 112 "uniform sampler2D tex_sampler_0;\n" + 113 "uniform sampler2D tex_sampler_1;\n" + 114 "uniform sampler2D tex_sampler_2;\n" + 115 "uniform float scale;\n" + 116 "uniform float shift_scale;\n" + 117 "uniform float hist_offset;\n" + 118 "uniform float hist_scale;\n" + 119 "uniform float density_offset;\n" + 120 "uniform float density_scale;\n" + 121 "varying vec2 v_texcoord;\n" + 122 "void main() {\n" + 123 " const vec3 weights = vec3(0.33333, 0.33333, 0.33333);\n" + 124 " vec4 color = texture2D(tex_sampler_0, v_texcoord);\n" + 125 " float energy = dot(color.rgb, weights);\n" + 126 " float mask_value = energy - 0.5;\n" + 127 " float alpha;\n" + 128 " if (mask_value > 0.0) {\n" + 129 " alpha = (pow(2.0 * mask_value, 1.5) - 1.0) * scale + 1.0;\n" + 130 " } else { \n" + 131 " alpha = (pow(2.0 * mask_value, 2.0) - 1.0) * scale + 1.0;\n" + 132 " }\n" + 133 " float index = energy * hist_scale + hist_offset;\n" + 134 " vec4 temp = texture2D(tex_sampler_1, vec2(index, 0.5));\n" + 135 " float value = temp.g + temp.r * shift_scale;\n" + 136 " index = value * density_scale + density_offset;\n" + 137 " temp = texture2D(tex_sampler_2, vec2(index, 0.5));\n" + 138 " value = temp.g + temp.r * shift_scale;\n" + 139 " float dst_energy = energy * alpha + value * (1.0 - alpha);\n" + 140 " float max_energy = energy / max(color.r, max(color.g, color.b));\n" + 141 " if (dst_energy > max_energy) {\n" + 142 " dst_energy = max_energy;\n" + 143 " }\n" + 144 " if (energy == 0.0) {\n" + 145 " gl_FragColor = color;\n" + 146 " } else {\n" + 147 " gl_FragColor = vec4(color.rgb * dst_energy / energy, color.a);\n" + 148 " }\n" + 149 "}\n"; 150 151 private Program mShaderProgram; 152 private Program mNativeProgram; 153 154 private int mWidth = 0; 155 private int mHeight = 0; 156 private int mTarget = FrameFormat.TARGET_UNSPECIFIED; 157 158 private Frame mHistFrame; 159 private Frame mDensityFrame; 160 161 public AutoFixFilter(String name) { 162 super(name); 163 } 164 165 @Override 166 public void setupPorts() { 167 addMaskedInputPort("image", ImageFormat.create(ImageFormat.COLORSPACE_RGBA)); 168 addOutputBasedOnInput("image", "image"); 169 } 170 171 @Override 172 public FrameFormat getOutputFormat(String portName, FrameFormat inputFormat) { 173 return inputFormat; 174 } 175 176 public void initProgram(FilterContext context, int target) { 177 switch (target) { 178 case FrameFormat.TARGET_GPU: 179 ShaderProgram shaderProgram = new ShaderProgram(context, mAutoFixShader); 180 shaderProgram.setMaximumTileSize(mTileSize); 181 mShaderProgram = shaderProgram; 182 break; 183 184 default: 185 throw new RuntimeException("Filter Sharpen does not support frames of " + 186 "target " + target + "!"); 187 } 188 mTarget = target; 189 } 190 191 private void initParameters() { 192 mShaderProgram.setHostValue("shift_scale", 1.0f / 256f); 193 mShaderProgram.setHostValue("hist_offset", 0.5f / 766f); 194 mShaderProgram.setHostValue("hist_scale", 765f / 766f); 195 mShaderProgram.setHostValue("density_offset", 0.5f / 1024f); 196 mShaderProgram.setHostValue("density_scale", 1023f / 1024f); 197 mShaderProgram.setHostValue("scale", mScale); 198 } 199 200 @Override 201 protected void prepare(FilterContext context) { 202 int densityDim = 1024; 203 int histDim = 255 * 3 + 1; 204 long precision = (256l * 256l - 1l); 205 206 int[] densityTable = new int[densityDim]; 207 for (int i = 0; i < densityDim; ++i) { 208 long temp = normal_cdf[i] * precision / histDim; 209 densityTable[i] = (int) temp; 210 } 211 212 FrameFormat densityFormat = ImageFormat.create(densityDim, 1, 213 ImageFormat.COLORSPACE_RGBA, 214 FrameFormat.TARGET_GPU); 215 mDensityFrame = context.getFrameManager().newFrame(densityFormat); 216 mDensityFrame.setInts(densityTable); 217 } 218 219 @Override 220 public void tearDown(FilterContext context) { 221 if (mDensityFrame != null) { 222 mDensityFrame.release(); 223 mDensityFrame = null; 224 } 225 226 if (mHistFrame != null) { 227 mHistFrame.release(); 228 mHistFrame = null; 229 } 230 } 231 232 @Override 233 public void fieldPortValueUpdated(String name, FilterContext context) { 234 if (mShaderProgram != null) { 235 mShaderProgram.setHostValue("scale", mScale); 236 } 237 } 238 239 @Override 240 public void process(FilterContext context) { 241 // Get input frame 242 Frame input = pullInput("image"); 243 FrameFormat inputFormat = input.getFormat(); 244 245 // Create program if not created already 246 if (mShaderProgram == null || inputFormat.getTarget() != mTarget) { 247 initProgram(context, inputFormat.getTarget()); 248 initParameters(); 249 } 250 251 // Check if the frame size has changed 252 if (inputFormat.getWidth() != mWidth || inputFormat.getHeight() != mHeight) { 253 mWidth = inputFormat.getWidth(); 254 mHeight = inputFormat.getHeight(); 255 createHistogramFrame(context, mWidth, mHeight, input.getInts()); 256 } 257 258 // Create output frame 259 Frame output = context.getFrameManager().newFrame(inputFormat); 260 261 // Process 262 Frame[] inputs = {input, mHistFrame, mDensityFrame}; 263 mShaderProgram.process(inputs, output); 264 265 // Push output 266 pushOutput("image", output); 267 268 // Release pushed frame 269 output.release(); 270 } 271 272 private void createHistogramFrame(FilterContext context, int width, int height, int[] data) { 273 int histDims = 255 * 3 + 1; 274 int[] histArray = new int[histDims]; 275 276 float border_thickness_ratio = 0.05f; 277 int y_border_thickness = (int) (height * border_thickness_ratio); 278 int x_border_thickness = (int) (width * border_thickness_ratio); 279 int pixels = (width - 2 * x_border_thickness) * (height - 2 * y_border_thickness); 280 281 float count = 0f; 282 for (int y = y_border_thickness; y < height - y_border_thickness; ++y) { 283 for (int x = x_border_thickness; x < width - x_border_thickness; ++x) { 284 int index = y * width + x; 285 int energy = (data[index] & 0xFF) + ((data[index] >> 8) & 0xFF) + 286 ((data[index] >> 16) & 0xFF); 287 histArray[energy] ++; 288 } 289 } 290 291 for (int i = 1; i < histDims; i++) { 292 histArray[i] += histArray[i-1]; 293 } 294 295 for (int i = 0; i < histDims; i++) { 296 long temp = (256 * 256 - 1l) * histArray[i] / pixels; 297 histArray[i] = (int) temp; 298 } 299 300 FrameFormat shaderHistFormat = ImageFormat.create(histDims, 1, 301 ImageFormat.COLORSPACE_RGBA, 302 FrameFormat.TARGET_GPU); 303 if (mHistFrame != null) 304 mHistFrame.release(); 305 306 mHistFrame = context.getFrameManager().newFrame(shaderHistFormat); 307 mHistFrame.setInts(histArray); 308 } 309} 310