14cead8034aab0e20e402baee87cbe9129db00192Stephen Hinespackage com.android.rs.refocus; 2de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 3de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wangimport android.graphics.Bitmap; 4de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wangimport android.support.v8.renderscript.RenderScript; 5de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wangimport android.util.Log; 6de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 7de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang/** 8de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * An abstract class that implements refocus filtering using Render Script. The 9de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * main function is {@code compute}. All other functions and data structures are 10de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * supporting this main function. Subclasses need to implement individual steps 11de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * based on pixel representation, e.g., uint8 or float32. 12de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * 13de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * @parampixel representation, which can be float of byte. 14de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * 15de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * @author zhl@google.com (Li Zhang) 16de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang */ 17de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wangpublic abstract class RefocusFilter<ScriptType> { 18de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang //private static final Log.Tag TAG = new Log.Tag("RefocusFilter"); 19de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected static final String TAG = "RefocusFilter"; 20de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Render Script context. 21de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected RenderScript renderScript; 22de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 23de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Script functions in .rs file. 24de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected ScriptType scriptC; 25de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 26de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang /* 27de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * A constructor that initializes the class. 28de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * 29de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * @param rs the Render Script context. 30de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang */ 31de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang public RefocusFilter(RenderScript rs) { 32de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang renderScript = rs; 33de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang } 34de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 35de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang /* 36de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * A function that implements refocus filtering using Render Script. 37de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * 38de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * @param inputImage an RGBD image. RGB channels of the input image form the 39de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * color image. The D channel has a range of [1,BlurStack.MAX_DEPTH], where 0 40de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * is reserved for invalid padded pixels. Depth here refers to inverse depth 41de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * (i.e., disparity), where larger depths are closer to the camera. 42de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * 43de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * @param blurStack an object that has all the parameters for refocus 44de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * filtering, including: the number of blending layers, the depth levels in 45de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * each blending layer, focal depth, etc. For details, please refer to the 46de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * definition of {@code BlurStack}. 47de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * 48de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * @return a {@code Bitmap} of the filtering result 49de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang */ 50de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang /* 51de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang Commented out for now to define in derived classes 52de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang so that images after each stage could be extracted 53de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang */ 54de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected Bitmap compute(Bitmap inputImage, BlurStack blurStack) { 55de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 56de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Initializes {@code scriptC} and allocates required memory buffers 57de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // (defined in subclasses) that interface between Java and Render Script. 58de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang initializeScriptAndBuffers(inputImage, 59de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang blurStack.getLayerInfo(blurStack.getFocusLayer())); 60de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 61de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Processes layers from back-most to focal depth (including the focal 62de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // depth). 63de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang if (!processLayersFromBackToFocus(blurStack)) { 64de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang return null; 65de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang } 66de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 67de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Processes layers from front-most to focal depth (excluding the focal 68de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // depth). 69de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang if (!processLayersFromFrontToFocus(blurStack)) { 70de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang return null; 71de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang } 72de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 73de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Extracts the result from .rs file to Java. 74de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang Bitmap resultImage = extractResultImage(); 75de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang renderScript.finish(); 76de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 77de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang Log.d(TAG, "filterAndBlendAllLayersUsingKernel is finished"); 78de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang return resultImage; 79de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang } 80de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 81de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang /* 82de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * Process layers from back-most to focal depth (including the focal depth). 83de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang */ 84de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected boolean processLayersFromBackToFocus(BlurStack blurStack) { 85de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang for (int targetLayer = blurStack.getNumLayers() - 1; 86de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang targetLayer >= blurStack.getFocusLayer(); --targetLayer) { 87de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Sets up target layer info in Render Script. 88de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang LayerInfo layerInfo = blurStack.getLayerInfo(targetLayer); 89de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang setTargetLayer(layerInfo); 90de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 91de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // For a layer that is behind the focal depth, its back depth has the 92de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // largest blur kernel radius. Uses the kernel radius as dilation radius 93de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // of this layer. 94de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang int dilationRadius = getKernelRadius(layerInfo.backDepth, blurStack); 95de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang setBlendInfo(dilationRadius); 96de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 97de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Sends blur kernel matrix data to Render Script. 98de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang setKernelData(targetLayer, blurStack); 99de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 100de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Marks active pixels (pixels that on this layer). 101de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Marks pixels that are close enough (within dilationRadius) to the 102de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // active pixels. 103de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Computes distance transform of the active pixels in their neighborhood 104de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // and use the distance value as matte for layer blending later. 105de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang computeLayerMatteBehindFocalDepth(); 106de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 107de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Computes filtering for pixels on the target layer and saves the 108de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // filtering result in a buffer {@code g_fuzzy_image} in .rs file. 109de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang filterLayerBehindFocalDepth(); 110de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 111de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Replaces active pixels in {@code g_sharp_image} with the filtering 112de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // result saved in {@code g_fuzzy_image}. The replacement is soft, 113de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // blending {@code g_sharp_image} and {@code g_fuzzy_image} using the 114de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // computed matte. Uses the blending result as the sharp input image for 115de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // the next iteration. 116de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang updateSharpImageUsingFuzzyImage(); 117de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang } 118de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang return true; 119de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang } 120de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 121de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang /* 122de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * Processes layers from front-most to focal depth (excluding the focal depth) 123de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang */ 124de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected boolean processLayersFromFrontToFocus(BlurStack blurStack) { 125de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // At this point, the input image {@code g_sharp_image} has been updated by 126de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // the first pass from back-most layer to focus layer {@code 127de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // processLayersFromBackToFocus}. 128de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang for (int targetLayer = 0; targetLayer < blurStack.getFocusLayer(); 129de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang ++targetLayer) { 130de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Sets up target layer info in Render Script. 131de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang LayerInfo layerInfo = blurStack.getLayerInfo(targetLayer); 132de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang setTargetLayer(layerInfo); 133de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 134de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // For a layer that is in front of the focal depth, its front depth has 135de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // the largest blur kernel radius. Uses the kernel radius as dilation 136de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // radius of this layer. 137de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang int dilationRadius = getKernelRadius(layerInfo.frontDepth, blurStack); 138de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang setBlendInfo(dilationRadius); 139de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 140de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Sends blur kernel matrix data to Render Script. 141de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang setKernelData(targetLayer, blurStack); 142de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 143de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Marks active pixels (pixels that on this layer). 144de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Marks pixels that are close enough (within dilationRadius) to the 145de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // active pixels. 146de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Computes distance transform of the active pixels in their neighborhood 147de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // and use the distance value as matte for layer blending later. 148de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang computeLayerMatteInFrontOfFocalDepth(); 149de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 150de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Computes filtering for pixels on the target layer and accumulates the 151de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // filtering result to an buffer {@code g_fuzzy_image} in .rs file. 152de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // The accumulating operation is soft, using the computed matte values. 153de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang filterLayerInFrontOfFocalDepth(); 154de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang } 155de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 156de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Fills in the pixels on or behind the focal depth in {@code g_fuzzy_image} 157de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // using pixels in {@code g_sharp_image}. Does the filling in a soft way by 158de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // blending using the matte. Uses the blending result (saved in {@code 159de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // g_fuzzy_image}) as the final output image. 160de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang finalizeFuzzyImageUsingSharpImage(); 161de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang return true; 162de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang } 163de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 164de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang private static int getKernelRadius(int depth, BlurStack blurStack) { 165de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang int kernelRadius = KernelDataForRenderScript 166de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang .computeKernelRadiusFromDiskRadius(blurStack.getDiskRadius(depth)); 167de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang return kernelRadius; 168de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang } 169de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 170de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // /////////////////////////////////////////////////////////////////////////// 171de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // 172de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // The following abstract functions must be implemented in a subclass. 173de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // 174de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // /////////////////////////////////////////////////////////////////////////// 175de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 176de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Initializes the member {@code scriptC} and allocate memory buffers (defined 177de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // in a subclass) that interface between Java and .rs file. 178de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected abstract void initializeScriptAndBuffers(Bitmap inputImage, 179de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang LayerInfo focalLayer); 180de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 181de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Extracts the result image from memory buffer. 182de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected abstract Bitmap extractResultImage(); 183de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 184de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Sets target layer info in .rs file. 185de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected abstract void setTargetLayer(LayerInfo layerInfo); 186de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 187de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang // Sets dilation radius in .rs file for blending target layer. 188de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected abstract void setBlendInfo(int dilationRadius); 189de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 190de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang /* 191de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * A function that prepares the blur kernels for the target layer and passes 192de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * them to the Render Script. Each depth value in the layer has a kernel. The 193de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * kernels are packed in a memory buffer. Auxiliary information for parsing 194de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * the memory buffer is also prepared and passed to Render Script. 195de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * 196de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * @param targetLayer the index of a target layer 197de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * 198de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * @param blurStack a BlurStack object that has the layer structure of the 199de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang * refocus filter task in Java 200de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang */ 201de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected abstract void setKernelData(int targetLayer, BlurStack blurStack); 202de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 203de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected abstract void computeLayerMatteBehindFocalDepth(); 204de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 205de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected abstract void filterLayerBehindFocalDepth(); 206de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 207de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected abstract void updateSharpImageUsingFuzzyImage(); 208de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 209de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected abstract void computeLayerMatteInFrontOfFocalDepth(); 210de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 211de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected abstract void filterLayerInFrontOfFocalDepth(); 212de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang 213de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang protected abstract void finalizeFuzzyImageUsingSharpImage(); 214de2f182fdb522689e05280e449a39ec2c1b53e1fCindy Wang} 215