1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package android.filterpacks.imageproc; 18 19import android.filterfw.core.Filter; 20import android.filterfw.core.FilterContext; 21import android.filterfw.core.Frame; 22import android.filterfw.core.FrameFormat; 23import android.filterfw.core.GenerateFieldPort; 24import android.filterfw.core.Program; 25import android.filterfw.core.ShaderProgram; 26import android.filterfw.format.ImageFormat; 27 28public class AutoFixFilter extends Filter { 29 30 @GenerateFieldPort(name = "tile_size", hasDefault = true) 31 private int mTileSize = 640; 32 33 @GenerateFieldPort(name = "scale") 34 private float mScale; 35 36 private static final int normal_cdf[] = { 37 9, 33, 50, 64, 75, 84, 92, 99, 106, 112, 117, 122, 126, 130, 134, 138, 142, 38 145, 148, 150, 154, 157, 159, 162, 164, 166, 169, 170, 173, 175, 177, 179, 39 180, 182, 184, 186, 188, 189, 190, 192, 194, 195, 197, 198, 199, 200, 202, 40 203, 205, 206, 207, 208, 209, 210, 212, 213, 214, 215, 216, 217, 218, 219, 41 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 229, 230, 231, 232, 233, 42 234, 235, 236, 236, 237, 238, 239, 239, 240, 240, 242, 242, 243, 244, 245, 43 245, 246, 247, 247, 248, 249, 249, 250, 250, 251, 252, 253, 253, 254, 255, 44 255, 256, 256, 257, 258, 258, 259, 259, 259, 260, 261, 262, 262, 263, 263, 45 264, 264, 265, 265, 266, 267, 267, 268, 268, 269, 269, 269, 270, 270, 271, 46 272, 272, 273, 273, 274, 274, 275, 275, 276, 276, 277, 277, 277, 278, 278, 47 279, 279, 279, 280, 280, 281, 282, 282, 282, 283, 283, 284, 284, 285, 285, 48 285, 286, 286, 287, 287, 288, 288, 288, 289, 289, 289, 290, 290, 290, 291, 49 292, 292, 292, 293, 293, 294, 294, 294, 295, 295, 296, 296, 296, 297, 297, 50 297, 298, 298, 298, 299, 299, 299, 299, 300, 300, 301, 301, 302, 302, 302, 51 303, 303, 304, 304, 304, 305, 305, 305, 306, 306, 306, 307, 307, 307, 308, 52 308, 308, 309, 309, 309, 309, 310, 310, 310, 310, 311, 312, 312, 312, 313, 53 313, 313, 314, 314, 314, 315, 315, 315, 315, 316, 316, 316, 317, 317, 317, 54 318, 318, 318, 319, 319, 319, 319, 319, 320, 320, 320, 321, 321, 322, 322, 55 322, 323, 323, 323, 323, 324, 324, 324, 325, 325, 325, 325, 326, 326, 326, 56 327, 327, 327, 327, 328, 328, 328, 329, 329, 329, 329, 329, 330, 330, 330, 57 330, 331, 331, 332, 332, 332, 333, 333, 333, 333, 334, 334, 334, 334, 335, 58 335, 335, 336, 336, 336, 336, 337, 337, 337, 337, 338, 338, 338, 339, 339, 59 339, 339, 339, 339, 340, 340, 340, 340, 341, 341, 342, 342, 342, 342, 343, 60 343, 343, 344, 344, 344, 344, 345, 345, 345, 345, 346, 346, 346, 346, 347, 61 347, 347, 347, 348, 348, 348, 348, 349, 349, 349, 349, 349, 349, 350, 350, 62 350, 350, 351, 351, 352, 352, 352, 352, 353, 353, 353, 353, 354, 354, 354, 63 354, 355, 355, 355, 355, 356, 356, 356, 356, 357, 357, 357, 357, 358, 358, 64 358, 358, 359, 359, 359, 359, 359, 359, 359, 360, 360, 360, 360, 361, 361, 65 362, 362, 362, 362, 363, 363, 363, 363, 364, 364, 364, 364, 365, 365, 365, 66 365, 366, 366, 366, 366, 366, 367, 367, 367, 367, 368, 368, 368, 368, 369, 67 369, 369, 369, 369, 369, 370, 370, 370, 370, 370, 371, 371, 372, 372, 372, 68 372, 373, 373, 373, 373, 374, 374, 374, 374, 374, 375, 375, 375, 375, 376, 69 376, 376, 376, 377, 377, 377, 377, 378, 378, 378, 378, 378, 379, 379, 379, 70 379, 379, 379, 380, 380, 380, 380, 381, 381, 381, 382, 382, 382, 382, 383, 71 383, 383, 383, 384, 384, 384, 384, 385, 385, 385, 385, 385, 386, 386, 386, 72 386, 387, 387, 387, 387, 388, 388, 388, 388, 388, 389, 389, 389, 389, 389, 73 389, 390, 390, 390, 390, 391, 391, 392, 392, 392, 392, 392, 393, 393, 393, 74 393, 394, 394, 394, 394, 395, 395, 395, 395, 396, 396, 396, 396, 396, 397, 75 397, 397, 397, 398, 398, 398, 398, 399, 399, 399, 399, 399, 399, 400, 400, 76 400, 400, 400, 401, 401, 402, 402, 402, 402, 403, 403, 403, 403, 404, 404, 77 404, 404, 405, 405, 405, 405, 406, 406, 406, 406, 406, 407, 407, 407, 407, 78 408, 408, 408, 408, 409, 409, 409, 409, 409, 409, 410, 410, 410, 410, 411, 79 411, 412, 412, 412, 412, 413, 413, 413, 413, 414, 414, 414, 414, 415, 415, 80 415, 415, 416, 416, 416, 416, 417, 417, 417, 417, 418, 418, 418, 418, 419, 81 419, 419, 419, 419, 419, 420, 420, 420, 420, 421, 421, 422, 422, 422, 422, 82 423, 423, 423, 423, 424, 424, 424, 425, 425, 425, 425, 426, 426, 426, 426, 83 427, 427, 427, 427, 428, 428, 428, 429, 429, 429, 429, 429, 429, 430, 430, 84 430, 430, 431, 431, 432, 432, 432, 433, 433, 433, 433, 434, 434, 434, 435, 85 435, 435, 435, 436, 436, 436, 436, 437, 437, 437, 438, 438, 438, 438, 439, 86 439, 439, 439, 439, 440, 440, 440, 441, 441, 442, 442, 442, 443, 443, 443, 87 443, 444, 444, 444, 445, 445, 445, 446, 446, 446, 446, 447, 447, 447, 448, 88 448, 448, 449, 449, 449, 449, 449, 450, 450, 450, 451, 451, 452, 452, 452, 89 453, 453, 453, 454, 454, 454, 455, 455, 455, 456, 456, 456, 457, 457, 457, 90 458, 458, 458, 459, 459, 459, 459, 460, 460, 460, 461, 461, 462, 462, 462, 91 463, 463, 463, 464, 464, 465, 465, 465, 466, 466, 466, 467, 467, 467, 468, 92 468, 469, 469, 469, 469, 470, 470, 470, 471, 472, 472, 472, 473, 473, 474, 93 474, 474, 475, 475, 476, 476, 476, 477, 477, 478, 478, 478, 479, 479, 479, 94 480, 480, 480, 481, 482, 482, 483, 483, 484, 484, 484, 485, 485, 486, 486, 95 487, 487, 488, 488, 488, 489, 489, 489, 490, 490, 491, 492, 492, 493, 493, 96 494, 494, 495, 495, 496, 496, 497, 497, 498, 498, 499, 499, 499, 500, 501, 97 502, 502, 503, 503, 504, 504, 505, 505, 506, 507, 507, 508, 508, 509, 509, 98 510, 510, 511, 512, 513, 513, 514, 515, 515, 516, 517, 517, 518, 519, 519, 99 519, 520, 521, 522, 523, 524, 524, 525, 526, 526, 527, 528, 529, 529, 530, 100 531, 532, 533, 534, 535, 535, 536, 537, 538, 539, 539, 540, 542, 543, 544, 101 545, 546, 547, 548, 549, 549, 550, 552, 553, 554, 555, 556, 558, 559, 559, 102 561, 562, 564, 565, 566, 568, 569, 570, 572, 574, 575, 577, 578, 579, 582, 103 583, 585, 587, 589, 590, 593, 595, 597, 599, 602, 604, 607, 609, 612, 615, 104 618, 620, 624, 628, 631, 635, 639, 644, 649, 654, 659, 666, 673, 680, 690, 105 700, 714 }; 106 107 private final String mAutoFixShader = 108 "precision mediump float;\n" + 109 "uniform sampler2D tex_sampler_0;\n" + 110 "uniform sampler2D tex_sampler_1;\n" + 111 "uniform sampler2D tex_sampler_2;\n" + 112 "uniform float scale;\n" + 113 "uniform float shift_scale;\n" + 114 "uniform float hist_offset;\n" + 115 "uniform float hist_scale;\n" + 116 "uniform float density_offset;\n" + 117 "uniform float density_scale;\n" + 118 "varying vec2 v_texcoord;\n" + 119 "void main() {\n" + 120 " const vec3 weights = vec3(0.33333, 0.33333, 0.33333);\n" + 121 " vec4 color = texture2D(tex_sampler_0, v_texcoord);\n" + 122 " float energy = dot(color.rgb, weights);\n" + 123 " float mask_value = energy - 0.5;\n" + 124 " float alpha;\n" + 125 " if (mask_value > 0.0) {\n" + 126 " alpha = (pow(2.0 * mask_value, 1.5) - 1.0) * scale + 1.0;\n" + 127 " } else { \n" + 128 " alpha = (pow(2.0 * mask_value, 2.0) - 1.0) * scale + 1.0;\n" + 129 " }\n" + 130 " float index = energy * hist_scale + hist_offset;\n" + 131 " vec4 temp = texture2D(tex_sampler_1, vec2(index, 0.5));\n" + 132 " float value = temp.g + temp.r * shift_scale;\n" + 133 " index = value * density_scale + density_offset;\n" + 134 " temp = texture2D(tex_sampler_2, vec2(index, 0.5));\n" + 135 " value = temp.g + temp.r * shift_scale;\n" + 136 " float dst_energy = energy * alpha + value * (1.0 - alpha);\n" + 137 " float max_energy = energy / max(color.r, max(color.g, color.b));\n" + 138 " if (dst_energy > max_energy) {\n" + 139 " dst_energy = max_energy;\n" + 140 " }\n" + 141 " if (energy == 0.0) {\n" + 142 " gl_FragColor = color;\n" + 143 " } else {\n" + 144 " gl_FragColor = vec4(color.rgb * dst_energy / energy, color.a);\n" + 145 " }\n" + 146 "}\n"; 147 148 private Program mShaderProgram; 149 private Program mNativeProgram; 150 151 private int mWidth = 0; 152 private int mHeight = 0; 153 private int mTarget = FrameFormat.TARGET_UNSPECIFIED; 154 155 private Frame mHistFrame; 156 private Frame mDensityFrame; 157 158 public AutoFixFilter(String name) { 159 super(name); 160 } 161 162 @Override 163 public void setupPorts() { 164 addMaskedInputPort("image", ImageFormat.create(ImageFormat.COLORSPACE_RGBA)); 165 addOutputBasedOnInput("image", "image"); 166 } 167 168 @Override 169 public FrameFormat getOutputFormat(String portName, FrameFormat inputFormat) { 170 return inputFormat; 171 } 172 173 public void initProgram(FilterContext context, int target) { 174 switch (target) { 175 case FrameFormat.TARGET_GPU: 176 ShaderProgram shaderProgram = new ShaderProgram(context, mAutoFixShader); 177 shaderProgram.setMaximumTileSize(mTileSize); 178 mShaderProgram = shaderProgram; 179 break; 180 181 default: 182 throw new RuntimeException("Filter Sharpen does not support frames of " + 183 "target " + target + "!"); 184 } 185 mTarget = target; 186 } 187 188 private void initParameters() { 189 mShaderProgram.setHostValue("shift_scale", 1.0f / 256f); 190 mShaderProgram.setHostValue("hist_offset", 0.5f / 766f); 191 mShaderProgram.setHostValue("hist_scale", 765f / 766f); 192 mShaderProgram.setHostValue("density_offset", 0.5f / 1024f); 193 mShaderProgram.setHostValue("density_scale", 1023f / 1024f); 194 mShaderProgram.setHostValue("scale", mScale); 195 } 196 197 @Override 198 protected void prepare(FilterContext context) { 199 int densityDim = 1024; 200 int histDim = 255 * 3 + 1; 201 long precision = (256l * 256l - 1l); 202 203 int[] densityTable = new int[densityDim]; 204 for (int i = 0; i < densityDim; ++i) { 205 long temp = normal_cdf[i] * precision / histDim; 206 densityTable[i] = (int) temp; 207 } 208 209 FrameFormat densityFormat = ImageFormat.create(densityDim, 1, 210 ImageFormat.COLORSPACE_RGBA, 211 FrameFormat.TARGET_GPU); 212 mDensityFrame = context.getFrameManager().newFrame(densityFormat); 213 mDensityFrame.setInts(densityTable); 214 } 215 216 @Override 217 public void tearDown(FilterContext context) { 218 if (mDensityFrame != null) { 219 mDensityFrame.release(); 220 mDensityFrame = null; 221 } 222 223 if (mHistFrame != null) { 224 mHistFrame.release(); 225 mHistFrame = null; 226 } 227 } 228 229 @Override 230 public void fieldPortValueUpdated(String name, FilterContext context) { 231 if (mShaderProgram != null) { 232 mShaderProgram.setHostValue("scale", mScale); 233 } 234 } 235 236 @Override 237 public void process(FilterContext context) { 238 // Get input frame 239 Frame input = pullInput("image"); 240 FrameFormat inputFormat = input.getFormat(); 241 242 // Create program if not created already 243 if (mShaderProgram == null || inputFormat.getTarget() != mTarget) { 244 initProgram(context, inputFormat.getTarget()); 245 initParameters(); 246 } 247 248 // Check if the frame size has changed 249 if (inputFormat.getWidth() != mWidth || inputFormat.getHeight() != mHeight) { 250 mWidth = inputFormat.getWidth(); 251 mHeight = inputFormat.getHeight(); 252 createHistogramFrame(context, mWidth, mHeight, input.getInts()); 253 } 254 255 // Create output frame 256 Frame output = context.getFrameManager().newFrame(inputFormat); 257 258 // Process 259 Frame[] inputs = {input, mHistFrame, mDensityFrame}; 260 mShaderProgram.process(inputs, output); 261 262 // Push output 263 pushOutput("image", output); 264 265 // Release pushed frame 266 output.release(); 267 } 268 269 private void createHistogramFrame(FilterContext context, int width, int height, int[] data) { 270 int histDims = 255 * 3 + 1; 271 int[] histArray = new int[histDims]; 272 273 float border_thickness_ratio = 0.05f; 274 int y_border_thickness = (int) (height * border_thickness_ratio); 275 int x_border_thickness = (int) (width * border_thickness_ratio); 276 int pixels = (width - 2 * x_border_thickness) * (height - 2 * y_border_thickness); 277 278 float count = 0f; 279 for (int y = y_border_thickness; y < height - y_border_thickness; ++y) { 280 for (int x = x_border_thickness; x < width - x_border_thickness; ++x) { 281 int index = y * width + x; 282 int energy = (data[index] & 0xFF) + ((data[index] >> 8) & 0xFF) + 283 ((data[index] >> 16) & 0xFF); 284 histArray[energy] ++; 285 } 286 } 287 288 for (int i = 1; i < histDims; i++) { 289 histArray[i] += histArray[i-1]; 290 } 291 292 for (int i = 0; i < histDims; i++) { 293 long temp = (256 * 256 - 1l) * histArray[i] / pixels; 294 histArray[i] = (int) temp; 295 } 296 297 FrameFormat shaderHistFormat = ImageFormat.create(histDims, 1, 298 ImageFormat.COLORSPACE_RGBA, 299 FrameFormat.TARGET_GPU); 300 if (mHistFrame != null) 301 mHistFrame.release(); 302 303 mHistFrame = context.getFrameManager().newFrame(shaderHistFormat); 304 mHistFrame.setInts(histArray); 305 } 306} 307