filterpacks/videoproc/BackDropperFilter.java

/*
 * Copyright (C) 2011 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.filterpacks.videoproc;

import android.filterfw.core.Filter;
import android.filterfw.core.FilterContext;
import android.filterfw.core.GenerateFieldPort;
import android.filterfw.core.GenerateFinalPort;
import android.filterfw.core.Frame;
import android.filterfw.core.GLFrame;
import android.filterfw.core.FrameFormat;
import android.filterfw.core.MutableFrameFormat;
import android.filterfw.core.ShaderProgram;
import android.filterfw.format.ImageFormat;
import android.opengl.GLES20;
import android.os.SystemClock;
import android.os.SystemProperties;
import android.util.Log;

import java.lang.Math;
import java.util.Arrays;
import java.nio.ByteBuffer;

/**
 * @hide
 */
public class BackDropperFilter extends Filter {
    /** User-visible parameters */

    private final int BACKGROUND_STRETCH   = 0;
    private final int BACKGROUND_FIT       = 1;
    private final int BACKGROUND_FILL_CROP = 2;

    @GenerateFieldPort(name = "backgroundFitMode", hasDefault = true)
    private int mBackgroundFitMode = BACKGROUND_FILL_CROP;
    @GenerateFieldPort(name = "learningDuration", hasDefault = true)
    private int mLearningDuration = DEFAULT_LEARNING_DURATION;
    @GenerateFieldPort(name = "learningVerifyDuration", hasDefault = true)
    private int mLearningVerifyDuration = DEFAULT_LEARNING_VERIFY_DURATION;
    @GenerateFieldPort(name = "acceptStddev", hasDefault = true)
    private float mAcceptStddev = DEFAULT_ACCEPT_STDDEV;
    @GenerateFieldPort(name = "hierLrgScale", hasDefault = true)
    private float mHierarchyLrgScale = DEFAULT_HIER_LRG_SCALE;
    @GenerateFieldPort(name = "hierMidScale", hasDefault = true)
    private float mHierarchyMidScale = DEFAULT_HIER_MID_SCALE;
    @GenerateFieldPort(name = "hierSmlScale", hasDefault = true)
    private float mHierarchySmlScale = DEFAULT_HIER_SML_SCALE;

    // Dimensions of foreground / background mask. Optimum value should take into account only
    // image contents, NOT dimensions of input video stream.
    @GenerateFieldPort(name = "maskWidthExp", hasDefault = true)
    private int mMaskWidthExp = DEFAULT_MASK_WIDTH_EXPONENT;
    @GenerateFieldPort(name = "maskHeightExp", hasDefault = true)
    private int mMaskHeightExp = DEFAULT_MASK_HEIGHT_EXPONENT;

    // Levels at which to compute foreground / background decision. Think of them as are deltas
    // SUBTRACTED from maskWidthExp and maskHeightExp.
    @GenerateFieldPort(name = "hierLrgExp", hasDefault = true)
    private int mHierarchyLrgExp = DEFAULT_HIER_LRG_EXPONENT;
    @GenerateFieldPort(name = "hierMidExp", hasDefault = true)
    private int mHierarchyMidExp = DEFAULT_HIER_MID_EXPONENT;
    @GenerateFieldPort(name = "hierSmlExp", hasDefault = true)
    private int mHierarchySmlExp = DEFAULT_HIER_SML_EXPONENT;

    @GenerateFieldPort(name = "lumScale", hasDefault = true)
    private float mLumScale = DEFAULT_Y_SCALE_FACTOR;
    @GenerateFieldPort(name = "chromaScale", hasDefault = true)
    private float mChromaScale = DEFAULT_UV_SCALE_FACTOR;
    @GenerateFieldPort(name = "maskBg", hasDefault = true)
    private float mMaskBg = DEFAULT_MASK_BLEND_BG;
    @GenerateFieldPort(name = "maskFg", hasDefault = true)
    private float mMaskFg = DEFAULT_MASK_BLEND_FG;
    @GenerateFieldPort(name = "exposureChange", hasDefault = true)
    private float mExposureChange = DEFAULT_EXPOSURE_CHANGE;
    @GenerateFieldPort(name = "whitebalanceredChange", hasDefault = true)
    private float mWhiteBalanceRedChange = DEFAULT_WHITE_BALANCE_RED_CHANGE;
    @GenerateFieldPort(name = "whitebalanceblueChange", hasDefault = true)
    private float mWhiteBalanceBlueChange = DEFAULT_WHITE_BALANCE_BLUE_CHANGE;
    @GenerateFieldPort(name = "autowbToggle", hasDefault = true)
    private int mAutoWBToggle = DEFAULT_WHITE_BALANCE_TOGGLE;

    // TODO: These are not updatable:
    @GenerateFieldPort(name = "learningAdaptRate", hasDefault = true)
    private float mAdaptRateLearning = DEFAULT_LEARNING_ADAPT_RATE;
    @GenerateFieldPort(name = "adaptRateBg", hasDefault = true)
    private float mAdaptRateBg = DEFAULT_ADAPT_RATE_BG;
    @GenerateFieldPort(name = "adaptRateFg", hasDefault = true)
    private float mAdaptRateFg = DEFAULT_ADAPT_RATE_FG;
    @GenerateFieldPort(name = "maskVerifyRate", hasDefault = true)
    private float mVerifyRate = DEFAULT_MASK_VERIFY_RATE;
    @GenerateFieldPort(name = "learningDoneListener", hasDefault = true)
    private LearningDoneListener mLearningDoneListener = null;

    @GenerateFieldPort(name = "useTheForce", hasDefault = true)
    private boolean mUseTheForce = false;

    @GenerateFinalPort(name = "provideDebugOutputs", hasDefault = true)
    private boolean mProvideDebugOutputs = false;

    // Whether to mirror the background or not. For ex, the Camera app
    // would mirror the preview for the front camera
    @GenerateFieldPort(name = "mirrorBg", hasDefault = true)
    private boolean mMirrorBg = false;

    // The orientation of the display. This will change the flipping
    // coordinates, if we were to mirror the background
    @GenerateFieldPort(name = "orientation", hasDefault = true)
    private int mOrientation = 0;

    /** Default algorithm parameter values, for non-shader use */

    // Frame count for learning bg model
    private static final int DEFAULT_LEARNING_DURATION = 40;
    // Frame count for learning verification
    private static final int DEFAULT_LEARNING_VERIFY_DURATION = 10;
    // Maximum distance (in standard deviations) for considering a pixel as background
    private static final float DEFAULT_ACCEPT_STDDEV = 0.85f;
    // Variance threshold scale factor for large scale of hierarchy
    private static final float DEFAULT_HIER_LRG_SCALE = 0.7f;
    // Variance threshold scale factor for medium scale of hierarchy
    private static final float DEFAULT_HIER_MID_SCALE = 0.6f;
    // Variance threshold scale factor for small scale of hierarchy
    private static final float DEFAULT_HIER_SML_SCALE = 0.5f;
    // Width of foreground / background mask.
    private static final int DEFAULT_MASK_WIDTH_EXPONENT = 8;
    // Height of foreground / background mask.
    private static final int DEFAULT_MASK_HEIGHT_EXPONENT = 8;
    // Area over which to average for large scale (length in pixels = 2^HIERARCHY_*_EXPONENT)
    private static final int DEFAULT_HIER_LRG_EXPONENT = 3;
    // Area over which to average for medium scale
    private static final int DEFAULT_HIER_MID_EXPONENT = 2;
    // Area over which to average for small scale
    private static final int DEFAULT_HIER_SML_EXPONENT = 0;
    // Scale factor for luminance channel in distance calculations (larger = more significant)
    private static final float DEFAULT_Y_SCALE_FACTOR = 0.40f;
    // Scale factor for chroma channels in distance calculations
    private static final float DEFAULT_UV_SCALE_FACTOR = 1.35f;
    // Mask value to start blending away from background
    private static final float DEFAULT_MASK_BLEND_BG = 0.65f;
    // Mask value to start blending away from foreground
    private static final float DEFAULT_MASK_BLEND_FG = 0.95f;
    // Exposure stop number to change the brightness of foreground
    private static final float DEFAULT_EXPOSURE_CHANGE = 1.0f;
    // White balance change in Red channel for foreground
    private static final float DEFAULT_WHITE_BALANCE_RED_CHANGE = 0.0f;
    // White balance change in Blue channel for foreground
    private static final float DEFAULT_WHITE_BALANCE_BLUE_CHANGE = 0.0f;
    // Variable to control automatic white balance effect
    // 0.f -> Auto WB is off; 1.f-> Auto WB is on
    private static final int DEFAULT_WHITE_BALANCE_TOGGLE = 0;

    // Default rate at which to learn bg model during learning period
    private static final float DEFAULT_LEARNING_ADAPT_RATE = 0.2f;
    // Default rate at which to learn bg model from new background pixels
    private static final float DEFAULT_ADAPT_RATE_BG = 0.0f;
    // Default rate at which to learn bg model from new foreground pixels
    private static final float DEFAULT_ADAPT_RATE_FG = 0.0f;
    // Default rate at which to verify whether background is stable
    private static final float DEFAULT_MASK_VERIFY_RATE = 0.25f;
    // Default rate at which to verify whether background is stable
    private static final int   DEFAULT_LEARNING_DONE_THRESHOLD = 20;

    // Default 3x3 matrix, column major, for fitting background 1:1
    private static final float[] DEFAULT_BG_FIT_TRANSFORM = new float[] {
        1.0f, 0.0f, 0.0f,
        0.0f, 1.0f, 0.0f,
        0.0f, 0.0f, 1.0f
    };

    /** Default algorithm parameter values, for shader use */

    // Area over which to blur binary mask values (length in pixels = 2^MASK_SMOOTH_EXPONENT)
    private static final String MASK_SMOOTH_EXPONENT = "2.0";
    // Scale value for mapping variance distance to fit nicely to 0-1, 8-bit
    private static final String DISTANCE_STORAGE_SCALE = "0.6";
    // Scale value for mapping variance to fit nicely to 0-1, 8-bit
    private static final String VARIANCE_STORAGE_SCALE = "5.0";
    // Default scale of auto white balance parameters
    private static final String DEFAULT_AUTO_WB_SCALE = "0.25";
    // Minimum variance (0-255 scale)
    private static final String MIN_VARIANCE = "3.0";
    // Column-major array for 4x4 matrix converting RGB to YCbCr, JPEG definition (no pedestal)
    private static final String RGB_TO_YUV_MATRIX = "0.299, -0.168736,  0.5,      0.000, " +
                                                    "0.587, -0.331264, -0.418688, 0.000, " +
                                                    "0.114,  0.5,      -0.081312, 0.000, " +
                                                    "0.000,  0.5,       0.5,      1.000 ";
    /** Stream names */

    private static final String[] mInputNames = {"video",
                                                 "background"};

    private static final String[] mOutputNames = {"video"};

    private static final String[] mDebugOutputNames = {"debug1",
                                                       "debug2"};

    /** Other private variables */

    private FrameFormat mOutputFormat;
    private MutableFrameFormat mMemoryFormat;
    private MutableFrameFormat mMaskFormat;
    private MutableFrameFormat mAverageFormat;

    private final boolean mLogVerbose;
    private static final String TAG = "BackDropperFilter";

    /** Shader source code */

    // Shared uniforms and utility functions
    private static String mSharedUtilShader =
            "precision mediump float;\n" +
            "uniform float fg_adapt_rate;\n" +
            "uniform float bg_adapt_rate;\n" +
            "const mat4 coeff_yuv = mat4(" + RGB_TO_YUV_MATRIX + ");\n" +
            "const float dist_scale = " + DISTANCE_STORAGE_SCALE + ";\n" +
            "const float inv_dist_scale = 1. / dist_scale;\n" +
            "const float var_scale=" + VARIANCE_STORAGE_SCALE + ";\n" +
            "const float inv_var_scale = 1. / var_scale;\n" +
            "const float min_variance = inv_var_scale *" + MIN_VARIANCE + "/ 256.;\n" +
            "const float auto_wb_scale = " + DEFAULT_AUTO_WB_SCALE + ";\n" +
            "\n" +
            // Variance distance in luminance between current pixel and background model
            "float gauss_dist_y(float y, float mean, float variance) {\n" +
            "  float dist = (y - mean) * (y - mean) / variance;\n" +
            "  return dist;\n" +
            "}\n" +
            // Sum of variance distances in chroma between current pixel and background
            // model
            "float gauss_dist_uv(vec2 uv, vec2 mean, vec2 variance) {\n" +
            "  vec2 dist = (uv - mean) * (uv - mean) / variance;\n" +
            "  return dist.r + dist.g;\n" +
            "}\n" +
            // Select learning rate for pixel based on smoothed decision mask alpha
            "float local_adapt_rate(float alpha) {\n" +
            "  return mix(bg_adapt_rate, fg_adapt_rate, alpha);\n" +
            "}\n" +
            "\n";

    // Distance calculation shader. Calculates a distance metric between the foreground and the
    //   current background model, in both luminance and in chroma (yuv space).  Distance is
    //   measured in variances from the mean background value. For chroma, the distance is the sum
    //   of the two individual color channel distances. The distances are output on the b and alpha
    //   channels, r and g are for debug information.
    // Inputs:
    //   tex_sampler_0: Mip-map for foreground (live) video frame.
    //   tex_sampler_1: Background mean mask.
    //   tex_sampler_2: Background variance mask.
    //   subsample_level: Level on foreground frame's mip-map.
    private static final String mBgDistanceShader =
            "uniform sampler2D tex_sampler_0;\n" +
            "uniform sampler2D tex_sampler_1;\n" +
            "uniform sampler2D tex_sampler_2;\n" +
            "uniform float subsample_level;\n" +
            "varying vec2 v_texcoord;\n" +
            "void main() {\n" +
            "  vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" +
            "  vec4 fg = coeff_yuv * vec4(fg_rgb.rgb, 1.);\n" +
            "  vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" +
            "  vec4 variance = inv_var_scale * texture2D(tex_sampler_2, v_texcoord);\n" +
            "\n" +
            "  float dist_y = gauss_dist_y(fg.r, mean.r, variance.r);\n" +
            "  float dist_uv = gauss_dist_uv(fg.gb, mean.gb, variance.gb);\n" +
            "  gl_FragColor = vec4(0.5*fg.rg, dist_scale*dist_y, dist_scale*dist_uv);\n" +
            "}\n";

    // Foreground/background mask decision shader. Decides whether a frame is in the foreground or
    //   the background using a hierarchical threshold on the distance. Binary foreground/background
    //   mask is placed in the alpha channel. The RGB channels contain debug information.
    private static final String mBgMaskShader =
            "uniform sampler2D tex_sampler_0;\n" +
            "uniform float accept_variance;\n" +
            "uniform vec2 yuv_weights;\n" +
            "uniform float scale_lrg;\n" +
            "uniform float scale_mid;\n" +
            "uniform float scale_sml;\n" +
            "uniform float exp_lrg;\n" +
            "uniform float exp_mid;\n" +
            "uniform float exp_sml;\n" +
            "varying vec2 v_texcoord;\n" +
            // Decide whether pixel is foreground or background based on Y and UV
            //   distance and maximum acceptable variance.
            // yuv_weights.x is smaller than yuv_weights.y to discount the influence of shadow
            "bool is_fg(vec2 dist_yc, float accept_variance) {\n" +
            "  return ( dot(yuv_weights, dist_yc) >= accept_variance );\n" +
            "}\n" +
            "void main() {\n" +
            "  vec4 dist_lrg_sc = texture2D(tex_sampler_0, v_texcoord, exp_lrg);\n" +
            "  vec4 dist_mid_sc = texture2D(tex_sampler_0, v_texcoord, exp_mid);\n" +
            "  vec4 dist_sml_sc = texture2D(tex_sampler_0, v_texcoord, exp_sml);\n" +
            "  vec2 dist_lrg = inv_dist_scale * dist_lrg_sc.ba;\n" +
            "  vec2 dist_mid = inv_dist_scale * dist_mid_sc.ba;\n" +
            "  vec2 dist_sml = inv_dist_scale * dist_sml_sc.ba;\n" +
            "  vec2 norm_dist = 0.75 * dist_sml / accept_variance;\n" + // For debug viz
            "  bool is_fg_lrg = is_fg(dist_lrg, accept_variance * scale_lrg);\n" +
            "  bool is_fg_mid = is_fg_lrg || is_fg(dist_mid, accept_variance * scale_mid);\n" +
            "  float is_fg_sml =\n" +
            "      float(is_fg_mid || is_fg(dist_sml, accept_variance * scale_sml));\n" +
            "  float alpha = 0.5 * is_fg_sml + 0.3 * float(is_fg_mid) + 0.2 * float(is_fg_lrg);\n" +
            "  gl_FragColor = vec4(alpha, norm_dist, is_fg_sml);\n" +
            "}\n";

    // Automatic White Balance parameter decision shader
    // Use the Gray World assumption that in a white balance corrected image, the average of R, G, B
    //   channel will be a common gray value.
    // To match the white balance of foreground and background, the average of R, G, B channel of
    //   two videos should match.
    // Inputs:
    //   tex_sampler_0: Mip-map for foreground (live) video frame.
    //   tex_sampler_1: Mip-map for background (playback) video frame.
    //   pyramid_depth: Depth of input frames' mip-maps.
    private static final String mAutomaticWhiteBalance =
            "uniform sampler2D tex_sampler_0;\n" +
            "uniform sampler2D tex_sampler_1;\n" +
            "uniform float pyramid_depth;\n" +
            "uniform bool autowb_toggle;\n" +
            "varying vec2 v_texcoord;\n" +
            "void main() {\n" +
            "   vec4 mean_video = texture2D(tex_sampler_0, v_texcoord, pyramid_depth);\n"+
            "   vec4 mean_bg = texture2D(tex_sampler_1, v_texcoord, pyramid_depth);\n" +
            // If Auto WB is toggled off, the return texture will be a unicolor texture of value 1
            // If Auto WB is toggled on, the return texture will be a unicolor texture with
            //   adjustment parameters for R and B channels stored in the corresponding channel
            "   float green_normalizer = mean_video.g / mean_bg.g;\n"+
            "   vec4 adjusted_value = vec4(mean_bg.r / mean_video.r * green_normalizer, 1., \n" +
            "                         mean_bg.b / mean_video.b * green_normalizer, 1.) * auto_wb_scale; \n" +
            "   gl_FragColor = autowb_toggle ? adjusted_value : vec4(auto_wb_scale);\n" +
            "}\n";


    // Background subtraction shader. Uses a mipmap of the binary mask map to blend smoothly between
    //   foreground and background
    // Inputs:
    //   tex_sampler_0: Foreground (live) video frame.
    //   tex_sampler_1: Background (playback) video frame.
    //   tex_sampler_2: Foreground/background mask.
    //   tex_sampler_3: Auto white-balance factors.
    private static final String mBgSubtractShader =
            "uniform mat3 bg_fit_transform;\n" +
            "uniform float mask_blend_bg;\n" +
            "uniform float mask_blend_fg;\n" +
            "uniform float exposure_change;\n" +
            "uniform float whitebalancered_change;\n" +
            "uniform float whitebalanceblue_change;\n" +
            "uniform sampler2D tex_sampler_0;\n" +
            "uniform sampler2D tex_sampler_1;\n" +
            "uniform sampler2D tex_sampler_2;\n" +
            "uniform sampler2D tex_sampler_3;\n" +
            "varying vec2 v_texcoord;\n" +
            "void main() {\n" +
            "  vec2 bg_texcoord = (bg_fit_transform * vec3(v_texcoord, 1.)).xy;\n" +
            "  vec4 bg_rgb = texture2D(tex_sampler_1, bg_texcoord);\n" +
            // The foreground texture is modified by multiplying both manual and auto white balance changes in R and B
            //   channel and multiplying exposure change in all R, G, B channels.
            "  vec4 wb_auto_scale = texture2D(tex_sampler_3, v_texcoord) * exposure_change / auto_wb_scale;\n" +
            "  vec4 wb_manual_scale = vec4(1. + whitebalancered_change, 1., 1. + whitebalanceblue_change, 1.);\n" +
            "  vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord);\n" +
            "  vec4 fg_adjusted = fg_rgb * wb_manual_scale * wb_auto_scale;\n"+
            "  vec4 mask = texture2D(tex_sampler_2, v_texcoord, \n" +
            "                      " + MASK_SMOOTH_EXPONENT + ");\n" +
            "  float alpha = smoothstep(mask_blend_bg, mask_blend_fg, mask.a);\n" +
            "  gl_FragColor = mix(bg_rgb, fg_adjusted, alpha);\n";

    // May the Force... Makes the foreground object translucent blue, with a bright
    // blue-white outline
    private static final String mBgSubtractForceShader =
            "  vec4 ghost_rgb = (fg_adjusted * 0.7 + vec4(0.3,0.3,0.4,0.))*0.65 + \n" +
            "                   0.35*bg_rgb;\n" +
            "  float glow_start = 0.75 * mask_blend_bg; \n"+
            "  float glow_max   = mask_blend_bg; \n"+
            "  gl_FragColor = mask.a < glow_start ? bg_rgb : \n" +
            "                 mask.a < glow_max ? mix(bg_rgb, vec4(0.9,0.9,1.0,1.0), \n" +
            "                                     (mask.a - glow_start) / (glow_max - glow_start) ) : \n" +
            "                 mask.a < mask_blend_fg ? mix(vec4(0.9,0.9,1.0,1.0), ghost_rgb, \n" +
            "                                    (mask.a - glow_max) / (mask_blend_fg - glow_max) ) : \n" +
            "                 ghost_rgb;\n" +
            "}\n";

    // Background model mean update shader. Skews the current model mean toward the most recent pixel
    //   value for a pixel, weighted by the learning rate and by whether the pixel is classified as
    //   foreground or background.
    // Inputs:
    //   tex_sampler_0: Mip-map for foreground (live) video frame.
    //   tex_sampler_1: Background mean mask.
    //   tex_sampler_2: Foreground/background mask.
    //   subsample_level: Level on foreground frame's mip-map.
    private static final String mUpdateBgModelMeanShader =
            "uniform sampler2D tex_sampler_0;\n" +
            "uniform sampler2D tex_sampler_1;\n" +
            "uniform sampler2D tex_sampler_2;\n" +
            "uniform float subsample_level;\n" +
            "varying vec2 v_texcoord;\n" +
            "void main() {\n" +
            "  vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" +
            "  vec4 fg = coeff_yuv * vec4(fg_rgb.rgb, 1.);\n" +
            "  vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" +
            "  vec4 mask = texture2D(tex_sampler_2, v_texcoord, \n" +
            "                      " + MASK_SMOOTH_EXPONENT + ");\n" +
            "\n" +
            "  float alpha = local_adapt_rate(mask.a);\n" +
            "  vec4 new_mean = mix(mean, fg, alpha);\n" +
            "  gl_FragColor = new_mean;\n" +
            "}\n";

    // Background model variance update shader. Skews the current model variance toward the most
    //   recent variance for the pixel, weighted by the learning rate and by whether the pixel is
    //   classified as foreground or background.
    // Inputs:
    //   tex_sampler_0: Mip-map for foreground (live) video frame.
    //   tex_sampler_1: Background mean mask.
    //   tex_sampler_2: Background variance mask.
    //   tex_sampler_3: Foreground/background mask.
    //   subsample_level: Level on foreground frame's mip-map.
    // TODO: to improve efficiency, use single mark for mean + variance, then merge this into
    // mUpdateBgModelMeanShader.
    private static final String mUpdateBgModelVarianceShader =
            "uniform sampler2D tex_sampler_0;\n" +
            "uniform sampler2D tex_sampler_1;\n" +
            "uniform sampler2D tex_sampler_2;\n" +
            "uniform sampler2D tex_sampler_3;\n" +
            "uniform float subsample_level;\n" +
            "varying vec2 v_texcoord;\n" +
            "void main() {\n" +
            "  vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" +
            "  vec4 fg = coeff_yuv * vec4(fg_rgb.rgb, 1.);\n" +
            "  vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" +
            "  vec4 variance = inv_var_scale * texture2D(tex_sampler_2, v_texcoord);\n" +
            "  vec4 mask = texture2D(tex_sampler_3, v_texcoord, \n" +
            "                      " + MASK_SMOOTH_EXPONENT + ");\n" +
            "\n" +
            "  float alpha = local_adapt_rate(mask.a);\n" +
            "  vec4 cur_variance = (fg-mean)*(fg-mean);\n" +
            "  vec4 new_variance = mix(variance, cur_variance, alpha);\n" +
            "  new_variance = max(new_variance, vec4(min_variance));\n" +
            "  gl_FragColor = var_scale * new_variance;\n" +
            "}\n";

    // Background verification shader. Skews the current background verification mask towards the
    //   most recent frame, weighted by the learning rate.
    private static final String mMaskVerifyShader =
            "uniform sampler2D tex_sampler_0;\n" +
            "uniform sampler2D tex_sampler_1;\n" +
            "uniform float verify_rate;\n" +
            "varying vec2 v_texcoord;\n" +
            "void main() {\n" +
            "  vec4 lastmask = texture2D(tex_sampler_0, v_texcoord);\n" +
            "  vec4 mask = texture2D(tex_sampler_1, v_texcoord);\n" +
            "  float newmask = mix(lastmask.a, mask.a, verify_rate);\n" +
            "  gl_FragColor = vec4(0., 0., 0., newmask);\n" +
            "}\n";

    /** Shader program objects */

    private ShaderProgram mBgDistProgram;
    private ShaderProgram mBgMaskProgram;
    private ShaderProgram mBgSubtractProgram;
    private ShaderProgram mBgUpdateMeanProgram;
    private ShaderProgram mBgUpdateVarianceProgram;
    private ShaderProgram mCopyOutProgram;
    private ShaderProgram mAutomaticWhiteBalanceProgram;
    private ShaderProgram mMaskVerifyProgram;
    private ShaderProgram copyShaderProgram;

    /** Background model storage */

    private boolean mPingPong;
    private GLFrame mBgMean[];
    private GLFrame mBgVariance[];
    private GLFrame mMaskVerify[];
    private GLFrame mDistance;
    private GLFrame mAutoWB;
    private GLFrame mMask;
    private GLFrame mVideoInput;
    private GLFrame mBgInput;
    private GLFrame mMaskAverage;

    /** Overall filter state */

    private boolean isOpen;
    private int mFrameCount;
    private boolean mStartLearning;
    private boolean mBackgroundFitModeChanged;
    private float mRelativeAspect;
    private int mPyramidDepth;
    private int mSubsampleLevel;

    /** Learning listener object */

    public interface LearningDoneListener {
        public void onLearningDone(BackDropperFilter filter);
    }

    /** Public Filter methods */

    public BackDropperFilter(String name) {
        super(name);

        mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);

        String adjStr = SystemProperties.get("ro.media.effect.bgdropper.adj");
        if (adjStr.length() > 0) {
            try {
                mAcceptStddev += Float.parseFloat(adjStr);
                if (mLogVerbose) {
                    Log.v(TAG, "Adjusting accept threshold by " + adjStr +
                            ", now " + mAcceptStddev);
                }
            } catch (NumberFormatException e) {
                Log.e(TAG,
                        "Badly formatted property ro.media.effect.bgdropper.adj: " + adjStr);
            }
        }
    }

    @Override
    public void setupPorts() {
        // Inputs.
        // TODO: Target should be GPU, but relaxed for now.
        FrameFormat imageFormat = ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
                                                     FrameFormat.TARGET_UNSPECIFIED);
        for (String inputName : mInputNames) {
            addMaskedInputPort(inputName, imageFormat);
        }
        // Normal outputs
        for (String outputName : mOutputNames) {
            addOutputBasedOnInput(outputName, "video");
        }

        // Debug outputs
        if (mProvideDebugOutputs) {
            for (String outputName : mDebugOutputNames) {
                addOutputBasedOnInput(outputName, "video");
            }
        }
    }

    @Override
    public FrameFormat getOutputFormat(String portName, FrameFormat inputFormat) {
        // Create memory format based on video input.
        MutableFrameFormat format = inputFormat.mutableCopy();
        // Is this a debug output port? If so, leave dimensions unspecified.
        if (!Arrays.asList(mOutputNames).contains(portName)) {
            format.setDimensions(FrameFormat.SIZE_UNSPECIFIED, FrameFormat.SIZE_UNSPECIFIED);
        }
        return format;
    }

    private boolean createMemoryFormat(FrameFormat inputFormat) {
        // We can't resize because that would require re-learning.
        if (mMemoryFormat != null) {
            return false;
        }

        if (inputFormat.getWidth() == FrameFormat.SIZE_UNSPECIFIED ||
            inputFormat.getHeight() == FrameFormat.SIZE_UNSPECIFIED) {
            throw new RuntimeException("Attempting to process input frame with unknown size");
        }

        mMaskFormat = inputFormat.mutableCopy();
        int maskWidth = (int)Math.pow(2, mMaskWidthExp);
        int maskHeight = (int)Math.pow(2, mMaskHeightExp);
        mMaskFormat.setDimensions(maskWidth, maskHeight);

        mPyramidDepth = Math.max(mMaskWidthExp, mMaskHeightExp);
        mMemoryFormat = mMaskFormat.mutableCopy();
        int widthExp = Math.max(mMaskWidthExp, pyramidLevel(inputFormat.getWidth()));
        int heightExp = Math.max(mMaskHeightExp, pyramidLevel(inputFormat.getHeight()));
        mPyramidDepth = Math.max(widthExp, heightExp);
        int memWidth = Math.max(maskWidth, (int)Math.pow(2, widthExp));
        int memHeight = Math.max(maskHeight, (int)Math.pow(2, heightExp));
        mMemoryFormat.setDimensions(memWidth, memHeight);
        mSubsampleLevel = mPyramidDepth - Math.max(mMaskWidthExp, mMaskHeightExp);

        if (mLogVerbose) {
            Log.v(TAG, "Mask frames size " + maskWidth + " x " + maskHeight);
            Log.v(TAG, "Pyramid levels " + widthExp + " x " + heightExp);
            Log.v(TAG, "Memory frames size " + memWidth + " x " + memHeight);
        }

        mAverageFormat = inputFormat.mutableCopy();
        mAverageFormat.setDimensions(1,1);
        return true;
    }

    public void prepare(FilterContext context){
        if (mLogVerbose) Log.v(TAG, "Preparing BackDropperFilter!");

        mBgMean = new GLFrame[2];
        mBgVariance = new GLFrame[2];
        mMaskVerify = new GLFrame[2];
        copyShaderProgram = ShaderProgram.createIdentity(context);
    }

    private void allocateFrames(FrameFormat inputFormat, FilterContext context) {
        if (!createMemoryFormat(inputFormat)) {
            return;  // All set.
        }
        if (mLogVerbose) Log.v(TAG, "Allocating BackDropperFilter frames");

        // Create initial background model values
        int numBytes = mMaskFormat.getSize();
        byte[] initialBgMean = new byte[numBytes];
        byte[] initialBgVariance = new byte[numBytes];
        byte[] initialMaskVerify = new byte[numBytes];
        for (int i = 0; i < numBytes; i++) {
            initialBgMean[i] = (byte)128;
            initialBgVariance[i] = (byte)10;
            initialMaskVerify[i] = (byte)0;
        }

        // Get frames to store background model in
        for (int i = 0; i < 2; i++) {
            mBgMean[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat);
            mBgMean[i].setData(initialBgMean, 0, numBytes);

            mBgVariance[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat);
            mBgVariance[i].setData(initialBgVariance, 0, numBytes);

            mMaskVerify[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat);
            mMaskVerify[i].setData(initialMaskVerify, 0, numBytes);
        }

        // Get frames to store other textures in
        if (mLogVerbose) Log.v(TAG, "Done allocating texture for Mean and Variance objects!");

        mDistance = (GLFrame)context.getFrameManager().newFrame(mMaskFormat);
        mMask = (GLFrame)context.getFrameManager().newFrame(mMaskFormat);
        mAutoWB = (GLFrame)context.getFrameManager().newFrame(mAverageFormat);
        mVideoInput = (GLFrame)context.getFrameManager().newFrame(mMemoryFormat);
        mBgInput = (GLFrame)context.getFrameManager().newFrame(mMemoryFormat);
        mMaskAverage = (GLFrame)context.getFrameManager().newFrame(mAverageFormat);

        // Create shader programs
        mBgDistProgram = new ShaderProgram(context, mSharedUtilShader + mBgDistanceShader);
        mBgDistProgram.setHostValue("subsample_level", (float)mSubsampleLevel);

        mBgMaskProgram = new ShaderProgram(context, mSharedUtilShader + mBgMaskShader);
        mBgMaskProgram.setHostValue("accept_variance", mAcceptStddev * mAcceptStddev);
        float[] yuvWeights = { mLumScale, mChromaScale };
        mBgMaskProgram.setHostValue("yuv_weights", yuvWeights );
        mBgMaskProgram.setHostValue("scale_lrg", mHierarchyLrgScale);
        mBgMaskProgram.setHostValue("scale_mid", mHierarchyMidScale);
        mBgMaskProgram.setHostValue("scale_sml", mHierarchySmlScale);
        mBgMaskProgram.setHostValue("exp_lrg", (float)(mSubsampleLevel + mHierarchyLrgExp));
        mBgMaskProgram.setHostValue("exp_mid", (float)(mSubsampleLevel + mHierarchyMidExp));
        mBgMaskProgram.setHostValue("exp_sml", (float)(mSubsampleLevel + mHierarchySmlExp));

        if (mUseTheForce) {
            mBgSubtractProgram = new ShaderProgram(context, mSharedUtilShader + mBgSubtractShader + mBgSubtractForceShader);
        } else {
            mBgSubtractProgram = new ShaderProgram(context, mSharedUtilShader + mBgSubtractShader + "}\n");
        }
        mBgSubtractProgram.setHostValue("bg_fit_transform", DEFAULT_BG_FIT_TRANSFORM);
        mBgSubtractProgram.setHostValue("mask_blend_bg", mMaskBg);
        mBgSubtractProgram.setHostValue("mask_blend_fg", mMaskFg);
        mBgSubtractProgram.setHostValue("exposure_change", mExposureChange);
        mBgSubtractProgram.setHostValue("whitebalanceblue_change", mWhiteBalanceBlueChange);
        mBgSubtractProgram.setHostValue("whitebalancered_change", mWhiteBalanceRedChange);


        mBgUpdateMeanProgram = new ShaderProgram(context, mSharedUtilShader + mUpdateBgModelMeanShader);
        mBgUpdateMeanProgram.setHostValue("subsample_level", (float)mSubsampleLevel);

        mBgUpdateVarianceProgram = new ShaderProgram(context, mSharedUtilShader + mUpdateBgModelVarianceShader);
        mBgUpdateVarianceProgram.setHostValue("subsample_level", (float)mSubsampleLevel);

        mCopyOutProgram = ShaderProgram.createIdentity(context);

        mAutomaticWhiteBalanceProgram = new ShaderProgram(context, mSharedUtilShader + mAutomaticWhiteBalance);
        mAutomaticWhiteBalanceProgram.setHostValue("pyramid_depth", (float)mPyramidDepth);
        mAutomaticWhiteBalanceProgram.setHostValue("autowb_toggle", mAutoWBToggle);

        mMaskVerifyProgram = new ShaderProgram(context, mSharedUtilShader + mMaskVerifyShader);
        mMaskVerifyProgram.setHostValue("verify_rate", mVerifyRate);

        if (mLogVerbose) Log.v(TAG, "Shader width set to " + mMemoryFormat.getWidth());

        mRelativeAspect = 1.f;

        mFrameCount = 0;
        mStartLearning = true;
    }

    public void process(FilterContext context) {
        // Grab inputs and ready intermediate frames and outputs.
        Frame video = pullInput("video");
        Frame background = pullInput("background");
        allocateFrames(video.getFormat(), context);

        // Update learning rate after initial learning period
        if (mStartLearning) {
            if (mLogVerbose) Log.v(TAG, "Starting learning");
            mBgUpdateMeanProgram.setHostValue("bg_adapt_rate", mAdaptRateLearning);
            mBgUpdateMeanProgram.setHostValue("fg_adapt_rate", mAdaptRateLearning);
            mBgUpdateVarianceProgram.setHostValue("bg_adapt_rate", mAdaptRateLearning);
            mBgUpdateVarianceProgram.setHostValue("fg_adapt_rate", mAdaptRateLearning);
            mFrameCount = 0;
        }

        // Select correct pingpong buffers
        int inputIndex = mPingPong ? 0 : 1;
        int outputIndex = mPingPong ? 1 : 0;
        mPingPong = !mPingPong;

        // Check relative aspect ratios
        updateBgScaling(video, background, mBackgroundFitModeChanged);
        mBackgroundFitModeChanged = false;

        // Make copies for input frames to GLFrames

        copyShaderProgram.process(video, mVideoInput);
        copyShaderProgram.process(background, mBgInput);

        mVideoInput.generateMipMap();
        mVideoInput.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER,
                                        GLES20.GL_LINEAR_MIPMAP_NEAREST);

        mBgInput.generateMipMap();
        mBgInput.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER,
                                     GLES20.GL_LINEAR_MIPMAP_NEAREST);

        if (mStartLearning) {
            copyShaderProgram.process(mVideoInput, mBgMean[inputIndex]);
            mStartLearning = false;
        }

        // Process shaders
        Frame[] distInputs = { mVideoInput, mBgMean[inputIndex], mBgVariance[inputIndex] };
        mBgDistProgram.process(distInputs, mDistance);
        mDistance.generateMipMap();
        mDistance.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER,
                                      GLES20.GL_LINEAR_MIPMAP_NEAREST);

        mBgMaskProgram.process(mDistance, mMask);
        mMask.generateMipMap();
        mMask.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER,
                                  GLES20.GL_LINEAR_MIPMAP_NEAREST);

        Frame[] autoWBInputs = { mVideoInput, mBgInput };
        mAutomaticWhiteBalanceProgram.process(autoWBInputs, mAutoWB);

        if (mFrameCount <= mLearningDuration) {
            // During learning
            pushOutput("video", video);

            if (mFrameCount == mLearningDuration - mLearningVerifyDuration) {
                copyShaderProgram.process(mMask, mMaskVerify[outputIndex]);

                mBgUpdateMeanProgram.setHostValue("bg_adapt_rate", mAdaptRateBg);
                mBgUpdateMeanProgram.setHostValue("fg_adapt_rate", mAdaptRateFg);
                mBgUpdateVarianceProgram.setHostValue("bg_adapt_rate", mAdaptRateBg);
                mBgUpdateVarianceProgram.setHostValue("fg_adapt_rate", mAdaptRateFg);


            } else if (mFrameCount > mLearningDuration - mLearningVerifyDuration) {
                // In the learning verification stage, compute background masks and a weighted average
                //   with weights grow exponentially with time
                Frame[] maskVerifyInputs = {mMaskVerify[inputIndex], mMask};
                mMaskVerifyProgram.process(maskVerifyInputs, mMaskVerify[outputIndex]);
                mMaskVerify[outputIndex].generateMipMap();
                mMaskVerify[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER,
                                                             GLES20.GL_LINEAR_MIPMAP_NEAREST);
            }

            if (mFrameCount == mLearningDuration) {
                // In the last verification frame, verify if the verification mask is almost blank
                // If not, restart learning
                copyShaderProgram.process(mMaskVerify[outputIndex], mMaskAverage);
                ByteBuffer mMaskAverageByteBuffer = mMaskAverage.getData();
                byte[] mask_average = mMaskAverageByteBuffer.array();
                int bi = (int)(mask_average[3] & 0xFF);

                if (mLogVerbose) {
                    Log.v(TAG,
                            String.format("Mask_average is %d, threshold is %d",
                                    bi, DEFAULT_LEARNING_DONE_THRESHOLD));
                }

                if (bi >= DEFAULT_LEARNING_DONE_THRESHOLD) {
                    mStartLearning = true;                                      // Restart learning
                } else {
                  if (mLogVerbose) Log.v(TAG, "Learning done");
                  if (mLearningDoneListener != null) {
                      mLearningDoneListener.onLearningDone(this);
                   }
                }
            }
        } else {
            Frame output = context.getFrameManager().newFrame(video.getFormat());
            Frame[] subtractInputs = { video, background, mMask, mAutoWB };
            mBgSubtractProgram.process(subtractInputs, output);
            pushOutput("video", output);
            output.release();
        }

        // Compute mean and variance of the background
        if (mFrameCount < mLearningDuration - mLearningVerifyDuration ||
            mAdaptRateBg > 0.0 || mAdaptRateFg > 0.0) {
            Frame[] meanUpdateInputs = { mVideoInput, mBgMean[inputIndex], mMask };
            mBgUpdateMeanProgram.process(meanUpdateInputs, mBgMean[outputIndex]);
            mBgMean[outputIndex].generateMipMap();
            mBgMean[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER,
                                                     GLES20.GL_LINEAR_MIPMAP_NEAREST);

            Frame[] varianceUpdateInputs = {
              mVideoInput, mBgMean[inputIndex], mBgVariance[inputIndex], mMask
            };
            mBgUpdateVarianceProgram.process(varianceUpdateInputs, mBgVariance[outputIndex]);
            mBgVariance[outputIndex].generateMipMap();
            mBgVariance[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER,
                                                         GLES20.GL_LINEAR_MIPMAP_NEAREST);
        }

        // Provide debug output to two smaller viewers
        if (mProvideDebugOutputs) {
            Frame dbg1 = context.getFrameManager().newFrame(video.getFormat());
            mCopyOutProgram.process(video, dbg1);
            pushOutput("debug1", dbg1);
            dbg1.release();

            Frame dbg2 = context.getFrameManager().newFrame(mMemoryFormat);
            mCopyOutProgram.process(mMask, dbg2);
            pushOutput("debug2", dbg2);
            dbg2.release();
        }

        mFrameCount++;

        if (mLogVerbose) {
            if (mFrameCount % 30 == 0) {
                if (startTime == -1) {
                    context.getGLEnvironment().activate();
                    GLES20.glFinish();
                    startTime = SystemClock.elapsedRealtime();
                } else {
                    context.getGLEnvironment().activate();
                    GLES20.glFinish();
                    long endTime = SystemClock.elapsedRealtime();
                    Log.v(TAG, "Avg. frame duration: " + String.format("%.2f",(endTime-startTime)/30.) +
                          " ms. Avg. fps: " + String.format("%.2f", 1000./((endTime-startTime)/30.)) );
                    startTime = endTime;
                }
            }
        }
    }

    private long startTime = -1;

    public void close(FilterContext context) {
        if (mMemoryFormat == null) {
            return;
        }

        if (mLogVerbose) Log.v(TAG, "Filter Closing!");
        for (int i = 0; i < 2; i++) {
            mBgMean[i].release();
            mBgVariance[i].release();
            mMaskVerify[i].release();
        }
        mDistance.release();
        mMask.release();
        mAutoWB.release();
        mVideoInput.release();
        mBgInput.release();
        mMaskAverage.release();

        mMemoryFormat = null;
    }

    // Relearn background model
    synchronized public void relearn() {
        // Let the processing thread know about learning restart
        mStartLearning = true;
    }

    @Override
    public void fieldPortValueUpdated(String name, FilterContext context) {
        // TODO: Many of these can be made ProgramPorts!
        if (name.equals("backgroundFitMode")) {
            mBackgroundFitModeChanged = true;
        } else if (name.equals("acceptStddev")) {
            mBgMaskProgram.setHostValue("accept_variance", mAcceptStddev * mAcceptStddev);
        } else if (name.equals("hierLrgScale")) {
            mBgMaskProgram.setHostValue("scale_lrg", mHierarchyLrgScale);
        } else if (name.equals("hierMidScale")) {
            mBgMaskProgram.setHostValue("scale_mid", mHierarchyMidScale);
        } else if (name.equals("hierSmlScale")) {
            mBgMaskProgram.setHostValue("scale_sml", mHierarchySmlScale);
        } else if (name.equals("hierLrgExp")) {
            mBgMaskProgram.setHostValue("exp_lrg", (float)(mSubsampleLevel + mHierarchyLrgExp));
        } else if (name.equals("hierMidExp")) {
            mBgMaskProgram.setHostValue("exp_mid", (float)(mSubsampleLevel + mHierarchyMidExp));
        } else if (name.equals("hierSmlExp")) {
            mBgMaskProgram.setHostValue("exp_sml", (float)(mSubsampleLevel + mHierarchySmlExp));
        } else if (name.equals("lumScale") || name.equals("chromaScale")) {
            float[] yuvWeights = { mLumScale, mChromaScale };
            mBgMaskProgram.setHostValue("yuv_weights", yuvWeights );
        } else if (name.equals("maskBg")) {
            mBgSubtractProgram.setHostValue("mask_blend_bg", mMaskBg);
        } else if (name.equals("maskFg")) {
            mBgSubtractProgram.setHostValue("mask_blend_fg", mMaskFg);
        } else if (name.equals("exposureChange")) {
            mBgSubtractProgram.setHostValue("exposure_change", mExposureChange);
        } else if (name.equals("whitebalanceredChange")) {
            mBgSubtractProgram.setHostValue("whitebalancered_change", mWhiteBalanceRedChange);
        } else if (name.equals("whitebalanceblueChange")) {
            mBgSubtractProgram.setHostValue("whitebalanceblue_change", mWhiteBalanceBlueChange);
        } else if (name.equals("autowbToggle")){
            mAutomaticWhiteBalanceProgram.setHostValue("autowb_toggle", mAutoWBToggle);
        }
    }

    private void updateBgScaling(Frame video, Frame background, boolean fitModeChanged) {
        float foregroundAspect = (float)video.getFormat().getWidth() / video.getFormat().getHeight();
        float backgroundAspect = (float)background.getFormat().getWidth() / background.getFormat().getHeight();
        float currentRelativeAspect = foregroundAspect/backgroundAspect;
        if (currentRelativeAspect != mRelativeAspect || fitModeChanged) {
            mRelativeAspect = currentRelativeAspect;
            float xMin = 0.f, xWidth = 1.f, yMin = 0.f, yWidth = 1.f;
            switch (mBackgroundFitMode) {
                case BACKGROUND_STRETCH:
                    // Just map 1:1
                    break;
                case BACKGROUND_FIT:
                    if (mRelativeAspect > 1.0f) {
                        // Foreground is wider than background, scale down
                        // background in X
                        xMin = 0.5f - 0.5f * mRelativeAspect;
                        xWidth = 1.f * mRelativeAspect;
                    } else {
                        // Foreground is taller than background, scale down
                        // background in Y
                        yMin = 0.5f - 0.5f / mRelativeAspect;
                        yWidth = 1 / mRelativeAspect;
                    }
                    break;
                case BACKGROUND_FILL_CROP:
                    if (mRelativeAspect > 1.0f) {
                        // Foreground is wider than background, crop
                        // background in Y
                        yMin = 0.5f - 0.5f / mRelativeAspect;
                        yWidth = 1.f / mRelativeAspect;
                    } else {
                        // Foreground is taller than background, crop
                        // background in X
                        xMin = 0.5f - 0.5f * mRelativeAspect;
                        xWidth = mRelativeAspect;
                    }
                    break;
            }
            // If mirroring is required (for ex. the camera mirrors the preview
            // in the front camera)
            // TODO: Backdropper does not attempt to apply any other transformation
            // than just flipping. However, in the current state, it's "x-axis" is always aligned
            // with the Camera's width. Hence, we need to define the mirroring based on the camera
            // orientation. In the future, a cleaner design would be to cast away all the rotation
            // in a separate place.
            if (mMirrorBg) {
                if (mLogVerbose) Log.v(TAG, "Mirroring the background!");
                // Mirroring in portrait
                if (mOrientation == 0 || mOrientation == 180) {
                    xWidth = -xWidth;
                    xMin = 1.0f - xMin;
                } else {
                    // Mirroring in landscape
                    yWidth = -yWidth;
                    yMin = 1.0f - yMin;
                }
            }
            if (mLogVerbose) Log.v(TAG, "bgTransform: xMin, yMin, xWidth, yWidth : " +
                    xMin + ", " + yMin + ", " + xWidth + ", " + yWidth +
                    ", mRelAspRatio = " + mRelativeAspect);
            // The following matrix is the transpose of the actual matrix
            float[] bgTransform = {xWidth, 0.f, 0.f,
                                   0.f, yWidth, 0.f,
                                   xMin, yMin,  1.f};
            mBgSubtractProgram.setHostValue("bg_fit_transform", bgTransform);
        }
    }

    private int pyramidLevel(int size) {
        return (int)Math.floor(Math.log10(size) / Math.log10(2)) - 1;
    }

}