1package com.android.rs.refocus;
2
3import android.graphics.Bitmap;
4import android.support.v8.renderscript.RenderScript;
5import android.util.Log;
6
7/**
8 * An abstract class that implements refocus filtering using Render Script. The
9 * main function is {@code compute}. All other functions and data structures are
10 * supporting this main function. Subclasses need to implement individual steps
11 * based on pixel representation, e.g., uint8 or float32.
12 *
13 * @param  pixel representation, which can be float of byte.
14 *
15 * @author zhl@google.com (Li Zhang)
16 */
17public abstract class RefocusFilter<ScriptType> {
18  //private static final Log.Tag TAG = new Log.Tag("RefocusFilter");
19  protected static final String TAG = "RefocusFilter";
20  // Render Script context.
21  protected RenderScript renderScript;
22
23  // Script functions in .rs file.
24  protected ScriptType scriptC;
25
26  /*
27   * A constructor that initializes the class.
28   *
29   * @param rs the Render Script context.
30   */
31  public RefocusFilter(RenderScript rs) {
32    renderScript = rs;
33  }
34
35  /*
36   * A function that implements refocus filtering using Render Script.
37   *
38   * @param inputImage an RGBD image. RGB channels of the input image form the
39   * color image. The D channel has a range of [1,BlurStack.MAX_DEPTH], where 0
40   * is reserved for invalid padded pixels. Depth here refers to inverse depth
41   * (i.e., disparity), where larger depths are closer to the camera.
42   *
43   * @param blurStack an object that has all the parameters for refocus
44   * filtering, including: the number of blending layers, the depth levels in
45   * each blending layer, focal depth, etc. For details, please refer to the
46   * definition of {@code BlurStack}.
47   *
48   * @return a {@code Bitmap} of the filtering result
49   */
50  /*
51    Commented out for now to define in derived classes
52    so that images after each stage could be extracted
53   */
54  protected Bitmap compute(Bitmap inputImage, BlurStack blurStack) {
55
56    // Initializes {@code scriptC} and allocates required memory buffers
57    // (defined in subclasses) that interface between Java and Render Script.
58    initializeScriptAndBuffers(inputImage,
59        blurStack.getLayerInfo(blurStack.getFocusLayer()));
60
61    // Processes layers from back-most to focal depth (including the focal
62    // depth).
63    if (!processLayersFromBackToFocus(blurStack)) {
64      return null;
65    }
66
67    // Processes layers from front-most to focal depth (excluding the focal
68    // depth).
69    if (!processLayersFromFrontToFocus(blurStack)) {
70      return null;
71    }
72
73    // Extracts the result from .rs file to Java.
74    Bitmap resultImage = extractResultImage();
75    renderScript.finish();
76
77    Log.d(TAG, "filterAndBlendAllLayersUsingKernel is finished");
78    return resultImage;
79  }
80
81  /*
82   * Process layers from back-most to focal depth (including the focal depth).
83   */
84  protected boolean processLayersFromBackToFocus(BlurStack blurStack) {
85    for (int targetLayer = blurStack.getNumLayers() - 1;
86        targetLayer >= blurStack.getFocusLayer(); --targetLayer) {
87      // Sets up target layer info in Render Script.
88      LayerInfo layerInfo = blurStack.getLayerInfo(targetLayer);
89      setTargetLayer(layerInfo);
90
91      // For a layer that is behind the focal depth, its back depth has the
92      // largest blur kernel radius. Uses the kernel radius as dilation radius
93      // of this layer.
94      int dilationRadius = getKernelRadius(layerInfo.backDepth, blurStack);
95      setBlendInfo(dilationRadius);
96
97      // Sends blur kernel matrix data to Render Script.
98      setKernelData(targetLayer, blurStack);
99
100      // Marks active pixels (pixels that on this layer).
101      // Marks pixels that are close enough (within dilationRadius) to the
102      // active pixels.
103      // Computes distance transform of the active pixels in their neighborhood
104      // and use the distance value as matte for layer blending later.
105      computeLayerMatteBehindFocalDepth();
106
107      // Computes filtering for pixels on the target layer and saves the
108      // filtering result in a buffer {@code g_fuzzy_image} in .rs file.
109      filterLayerBehindFocalDepth();
110
111      // Replaces active pixels in {@code g_sharp_image} with the filtering
112      // result saved in {@code g_fuzzy_image}. The replacement is soft,
113      // blending {@code g_sharp_image} and {@code g_fuzzy_image} using the
114      // computed matte. Uses the blending result as the sharp input image for
115      // the next iteration.
116      updateSharpImageUsingFuzzyImage();
117    }
118    return true;
119  }
120
121  /*
122   * Processes layers from front-most to focal depth (excluding the focal depth)
123   */
124  protected boolean processLayersFromFrontToFocus(BlurStack blurStack) {
125    // At this point, the input image {@code g_sharp_image} has been updated by
126    // the first pass from back-most layer to focus layer {@code
127    // processLayersFromBackToFocus}.
128    for (int targetLayer = 0; targetLayer < blurStack.getFocusLayer();
129        ++targetLayer) {
130      // Sets up target layer info in Render Script.
131      LayerInfo layerInfo = blurStack.getLayerInfo(targetLayer);
132      setTargetLayer(layerInfo);
133
134      // For a layer that is in front of the focal depth, its front depth has
135      // the largest blur kernel radius. Uses the kernel radius as dilation
136      // radius of this layer.
137      int dilationRadius = getKernelRadius(layerInfo.frontDepth, blurStack);
138      setBlendInfo(dilationRadius);
139
140      // Sends blur kernel matrix data to Render Script.
141      setKernelData(targetLayer, blurStack);
142
143      // Marks active pixels (pixels that on this layer).
144      // Marks pixels that are close enough (within dilationRadius) to the
145      // active pixels.
146      // Computes distance transform of the active pixels in their neighborhood
147      // and use the distance value as matte for layer blending later.
148      computeLayerMatteInFrontOfFocalDepth();
149
150      // Computes filtering for pixels on the target layer and accumulates the
151      // filtering result to an buffer {@code g_fuzzy_image} in .rs file.
152      // The accumulating operation is soft, using the computed matte values.
153      filterLayerInFrontOfFocalDepth();
154    }
155
156    // Fills in the pixels on or behind the focal depth in {@code g_fuzzy_image}
157    // using pixels in {@code g_sharp_image}. Does the filling in a soft way by
158    // blending using the matte. Uses the blending result (saved in {@code
159    // g_fuzzy_image}) as the final output image.
160    finalizeFuzzyImageUsingSharpImage();
161    return true;
162  }
163
164  private static int getKernelRadius(int depth, BlurStack blurStack) {
165    int kernelRadius = KernelDataForRenderScript
166        .computeKernelRadiusFromDiskRadius(blurStack.getDiskRadius(depth));
167    return kernelRadius;
168  }
169
170  // ///////////////////////////////////////////////////////////////////////////
171  //
172  // The following abstract functions must be implemented in a subclass.
173  //
174  // ///////////////////////////////////////////////////////////////////////////
175
176  // Initializes the member {@code scriptC} and allocate memory buffers (defined
177  // in a subclass) that interface between Java and .rs file.
178  protected abstract void initializeScriptAndBuffers(Bitmap inputImage,
179      LayerInfo focalLayer);
180
181  // Extracts the result image from memory buffer.
182  protected abstract Bitmap extractResultImage();
183
184  // Sets target layer info in .rs file.
185  protected abstract void setTargetLayer(LayerInfo layerInfo);
186
187  // Sets dilation radius in .rs file for blending target layer.
188  protected abstract void setBlendInfo(int dilationRadius);
189
190  /*
191   * A function that prepares the blur kernels for the target layer and passes
192   * them to the Render Script. Each depth value in the layer has a kernel. The
193   * kernels are packed in a memory buffer. Auxiliary information for parsing
194   * the memory buffer is also prepared and passed to Render Script.
195   *
196   * @param targetLayer the index of a target layer
197   *
198   * @param blurStack a BlurStack object that has the layer structure of the
199   * refocus filter task in Java
200   */
201  protected abstract void setKernelData(int targetLayer, BlurStack blurStack);
202
203  protected abstract void computeLayerMatteBehindFocalDepth();
204
205  protected abstract void filterLayerBehindFocalDepth();
206
207  protected abstract void updateSharpImageUsingFuzzyImage();
208
209  protected abstract void computeLayerMatteInFrontOfFocalDepth();
210
211  protected abstract void filterLayerInFrontOfFocalDepth();
212
213  protected abstract void finalizeFuzzyImageUsingSharpImage();
214}
215