165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn/* 265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * Copyright (C) 2011 The Android Open Source Project 365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * 465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * Licensed under the Apache License, Version 2.0 (the "License"); 565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * you may not use this file except in compliance with the License. 665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * You may obtain a copy of the License at 765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * 865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * http://www.apache.org/licenses/LICENSE-2.0 965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * 1065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * Unless required by applicable law or agreed to in writing, software 1165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * distributed under the License is distributed on an "AS IS" BASIS, 1265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * See the License for the specific language governing permissions and 1465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * limitations under the License. 1565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn */ 1665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 1765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennpackage android.filterpacks.videoproc; 1865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 1965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.Filter; 2065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.FilterContext; 2165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.GenerateFieldPort; 2265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.GenerateFinalPort; 2365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.Frame; 2465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.GLFrame; 2565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.FrameFormat; 2665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.MutableFrameFormat; 2765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.Program; 2865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.core.ShaderProgram; 2965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.filterfw.format.ImageFormat; 3065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.opengl.GLES20; 3165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.os.SystemClock; 324239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvalaimport android.os.SystemProperties; 3365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport android.util.Log; 3465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 3565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport java.lang.ArrayIndexOutOfBoundsException; 3665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport java.lang.Math; 3765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport java.util.Arrays; 3865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennimport java.nio.ByteBuffer; 3965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 4065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn/** 4165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn * @hide 4265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn */ 4365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Rennpublic class BackDropperFilter extends Filter { 4465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** User-visible parameters */ 4565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 4665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private final int BACKGROUND_STRETCH = 0; 4765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private final int BACKGROUND_FIT = 1; 4865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private final int BACKGROUND_FILL_CROP = 2; 4965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 5065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "backgroundFitMode", hasDefault = true) 5165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mBackgroundFitMode = BACKGROUND_FILL_CROP; 5265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "learningDuration", hasDefault = true) 5365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mLearningDuration = DEFAULT_LEARNING_DURATION; 5465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "learningVerifyDuration", hasDefault = true) 5565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mLearningVerifyDuration = DEFAULT_LEARNING_VERIFY_DURATION; 5665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "acceptStddev", hasDefault = true) 5765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mAcceptStddev = DEFAULT_ACCEPT_STDDEV; 5865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "hierLrgScale", hasDefault = true) 5965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mHierarchyLrgScale = DEFAULT_HIER_LRG_SCALE; 6065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "hierMidScale", hasDefault = true) 6165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mHierarchyMidScale = DEFAULT_HIER_MID_SCALE; 6265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "hierSmlScale", hasDefault = true) 6365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mHierarchySmlScale = DEFAULT_HIER_SML_SCALE; 6465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 6565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Dimensions of foreground / background mask. Optimum value should take into account only 6665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // image contents, NOT dimensions of input video stream. 6765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "maskWidthExp", hasDefault = true) 6865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mMaskWidthExp = DEFAULT_MASK_WIDTH_EXPONENT; 6965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "maskHeightExp", hasDefault = true) 7065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mMaskHeightExp = DEFAULT_MASK_HEIGHT_EXPONENT; 7165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 7265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Levels at which to compute foreground / background decision. Think of them as are deltas 7365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // SUBTRACTED from maskWidthExp and maskHeightExp. 7465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "hierLrgExp", hasDefault = true) 7565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mHierarchyLrgExp = DEFAULT_HIER_LRG_EXPONENT; 7665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "hierMidExp", hasDefault = true) 7765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mHierarchyMidExp = DEFAULT_HIER_MID_EXPONENT; 7865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "hierSmlExp", hasDefault = true) 7965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mHierarchySmlExp = DEFAULT_HIER_SML_EXPONENT; 8065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 8165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "lumScale", hasDefault = true) 8265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mLumScale = DEFAULT_Y_SCALE_FACTOR; 8365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "chromaScale", hasDefault = true) 8465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mChromaScale = DEFAULT_UV_SCALE_FACTOR; 8565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "maskBg", hasDefault = true) 8665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mMaskBg = DEFAULT_MASK_BLEND_BG; 8765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "maskFg", hasDefault = true) 8865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mMaskFg = DEFAULT_MASK_BLEND_FG; 8965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "exposureChange", hasDefault = true) 9065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mExposureChange = DEFAULT_EXPOSURE_CHANGE; 9165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "whitebalanceredChange", hasDefault = true) 9265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mWhiteBalanceRedChange = DEFAULT_WHITE_BALANCE_RED_CHANGE; 9365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "whitebalanceblueChange", hasDefault = true) 9465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mWhiteBalanceBlueChange = DEFAULT_WHITE_BALANCE_BLUE_CHANGE; 9565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "autowbToggle", hasDefault = true) 9665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mAutoWBToggle = DEFAULT_WHITE_BALANCE_TOGGLE; 9765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 9865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // TODO: These are not updatable: 9965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "learningAdaptRate", hasDefault = true) 10065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mAdaptRateLearning = DEFAULT_LEARNING_ADAPT_RATE; 10165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "adaptRateBg", hasDefault = true) 10265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mAdaptRateBg = DEFAULT_ADAPT_RATE_BG; 10365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "adaptRateFg", hasDefault = true) 10465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mAdaptRateFg = DEFAULT_ADAPT_RATE_FG; 10565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "maskVerifyRate", hasDefault = true) 10665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mVerifyRate = DEFAULT_MASK_VERIFY_RATE; 10765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "learningDoneListener", hasDefault = true) 10865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private LearningDoneListener mLearningDoneListener = null; 10965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 11065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "useTheForce", hasDefault = true) 11165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private boolean mUseTheForce = false; 11265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 11365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFinalPort(name = "provideDebugOutputs", hasDefault = true) 11465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private boolean mProvideDebugOutputs = false; 11565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 11665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Whether to mirror the background or not. For ex, the Camera app 11765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // would mirror the preview for the front camera 11865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "mirrorBg", hasDefault = true) 11965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private boolean mMirrorBg = false; 12065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 12165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // The orientation of the display. This will change the flipping 12265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // coordinates, if we were to mirror the background 12365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @GenerateFieldPort(name = "orientation", hasDefault = true) 12465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mOrientation = 0; 12565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 12665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Default algorithm parameter values, for non-shader use */ 12765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 12865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Frame count for learning bg model 12965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final int DEFAULT_LEARNING_DURATION = 40; 13065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Frame count for learning verification 13165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final int DEFAULT_LEARNING_VERIFY_DURATION = 10; 13265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Maximum distance (in standard deviations) for considering a pixel as background 13365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_ACCEPT_STDDEV = 0.85f; 13465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Variance threshold scale factor for large scale of hierarchy 13565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_HIER_LRG_SCALE = 0.7f; 13665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Variance threshold scale factor for medium scale of hierarchy 13765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_HIER_MID_SCALE = 0.6f; 13865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Variance threshold scale factor for small scale of hierarchy 13965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_HIER_SML_SCALE = 0.5f; 14065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Width of foreground / background mask. 14165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final int DEFAULT_MASK_WIDTH_EXPONENT = 8; 14265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Height of foreground / background mask. 14365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final int DEFAULT_MASK_HEIGHT_EXPONENT = 8; 14465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Area over which to average for large scale (length in pixels = 2^HIERARCHY_*_EXPONENT) 14565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final int DEFAULT_HIER_LRG_EXPONENT = 3; 14665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Area over which to average for medium scale 14765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final int DEFAULT_HIER_MID_EXPONENT = 2; 14865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Area over which to average for small scale 14965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final int DEFAULT_HIER_SML_EXPONENT = 0; 15065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Scale factor for luminance channel in distance calculations (larger = more significant) 15165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_Y_SCALE_FACTOR = 0.40f; 15265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Scale factor for chroma channels in distance calculations 15365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_UV_SCALE_FACTOR = 1.35f; 15465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Mask value to start blending away from background 15565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_MASK_BLEND_BG = 0.65f; 15665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Mask value to start blending away from foreground 15765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_MASK_BLEND_FG = 0.95f; 15865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Exposure stop number to change the brightness of foreground 15965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_EXPOSURE_CHANGE = 1.0f; 16065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // White balance change in Red channel for foreground 16165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_WHITE_BALANCE_RED_CHANGE = 0.0f; 16265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // White balance change in Blue channel for foreground 16365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_WHITE_BALANCE_BLUE_CHANGE = 0.0f; 16465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Variable to control automatic white balance effect 16565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // 0.f -> Auto WB is off; 1.f-> Auto WB is on 16665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final int DEFAULT_WHITE_BALANCE_TOGGLE = 0; 16765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 16865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Default rate at which to learn bg model during learning period 16965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_LEARNING_ADAPT_RATE = 0.2f; 17065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Default rate at which to learn bg model from new background pixels 17165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_ADAPT_RATE_BG = 0.0f; 17265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Default rate at which to learn bg model from new foreground pixels 17365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_ADAPT_RATE_FG = 0.0f; 17465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Default rate at which to verify whether background is stable 17565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float DEFAULT_MASK_VERIFY_RATE = 0.25f; 17665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Default rate at which to verify whether background is stable 17765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final int DEFAULT_LEARNING_DONE_THRESHOLD = 20; 17865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 17965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Default 3x3 matrix, column major, for fitting background 1:1 18065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final float[] DEFAULT_BG_FIT_TRANSFORM = new float[] { 18165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 1.0f, 0.0f, 0.0f, 18265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 0.0f, 1.0f, 0.0f, 18365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 0.0f, 0.0f, 1.0f 18465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn }; 18565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 18665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Default algorithm parameter values, for shader use */ 18765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 18865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Area over which to blur binary mask values (length in pixels = 2^MASK_SMOOTH_EXPONENT) 18965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String MASK_SMOOTH_EXPONENT = "2.0"; 19065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Scale value for mapping variance distance to fit nicely to 0-1, 8-bit 19165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String DISTANCE_STORAGE_SCALE = "0.6"; 19265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Scale value for mapping variance to fit nicely to 0-1, 8-bit 19365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String VARIANCE_STORAGE_SCALE = "5.0"; 19465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Default scale of auto white balance parameters 19565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String DEFAULT_AUTO_WB_SCALE = "0.25"; 19665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Minimum variance (0-255 scale) 19765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String MIN_VARIANCE = "3.0"; 19865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Column-major array for 4x4 matrix converting RGB to YCbCr, JPEG definition (no pedestal) 19965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String RGB_TO_YUV_MATRIX = "0.299, -0.168736, 0.5, 0.000, " + 20065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "0.587, -0.331264, -0.418688, 0.000, " + 20165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "0.114, 0.5, -0.081312, 0.000, " + 20265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "0.000, 0.5, 0.5, 1.000 "; 20365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Stream names */ 20465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 20565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String[] mInputNames = {"video", 20665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "background"}; 20765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 20865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String[] mOutputNames = {"video"}; 20965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 21065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String[] mDebugOutputNames = {"debug1", 21165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "debug2"}; 21265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 21365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Other private variables */ 21465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 21565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private FrameFormat mOutputFormat; 21665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private MutableFrameFormat mMemoryFormat; 21765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private MutableFrameFormat mMaskFormat; 21865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private MutableFrameFormat mAverageFormat; 21965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 22065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private final boolean mLogVerbose; 22165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String TAG = "BackDropperFilter"; 22265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 22365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Shader source code */ 22465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 22565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Shared uniforms and utility functions 22665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static String mSharedUtilShader = 22765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "precision mediump float;\n" + 22865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float fg_adapt_rate;\n" + 22965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float bg_adapt_rate;\n" + 23065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "const mat4 coeff_yuv = mat4(" + RGB_TO_YUV_MATRIX + ");\n" + 23165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "const float dist_scale = " + DISTANCE_STORAGE_SCALE + ";\n" + 23265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "const float inv_dist_scale = 1. / dist_scale;\n" + 23365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "const float var_scale=" + VARIANCE_STORAGE_SCALE + ";\n" + 23465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "const float inv_var_scale = 1. / var_scale;\n" + 23565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "const float min_variance = inv_var_scale *" + MIN_VARIANCE + "/ 256.;\n" + 23665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "const float auto_wb_scale = " + DEFAULT_AUTO_WB_SCALE + ";\n" + 23765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "\n" + 23865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Variance distance in luminance between current pixel and background model 23965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "float gauss_dist_y(float y, float mean, float variance) {\n" + 24065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float dist = (y - mean) * (y - mean) / variance;\n" + 24165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " return dist;\n" + 24265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "}\n" + 24365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Sum of variance distances in chroma between current pixel and background 24465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // model 24565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "float gauss_dist_uv(vec2 uv, vec2 mean, vec2 variance) {\n" + 24665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec2 dist = (uv - mean) * (uv - mean) / variance;\n" + 24765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " return dist.r + dist.g;\n" + 24865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "}\n" + 24965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Select learning rate for pixel based on smoothed decision mask alpha 25065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "float local_adapt_rate(float alpha) {\n" + 25165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " return mix(bg_adapt_rate, fg_adapt_rate, alpha);\n" + 25265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "}\n" + 25365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "\n"; 25465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 25565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Distance calculation shader. Calculates a distance metric between the foreground and the 25665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // current background model, in both luminance and in chroma (yuv space). Distance is 25765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // measured in variances from the mean background value. For chroma, the distance is the sum 25865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // of the two individual color channel distances. The distances are output on the b and alpha 25965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // channels, r and g are for debug information. 26065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Inputs: 26165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_0: Mip-map for foreground (live) video frame. 26265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_1: Background mean mask. 26365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_2: Background variance mask. 26465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // subsample_level: Level on foreground frame's mip-map. 26565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String mBgDistanceShader = 26665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_0;\n" + 26765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_1;\n" + 26865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_2;\n" + 26965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float subsample_level;\n" + 27065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "varying vec2 v_texcoord;\n" + 27165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "void main() {\n" + 27265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" + 27365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 fg = coeff_yuv * vec4(fg_rgb.rgb, 1.);\n" + 27465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" + 27565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 variance = inv_var_scale * texture2D(tex_sampler_2, v_texcoord);\n" + 27665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "\n" + 27765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float dist_y = gauss_dist_y(fg.r, mean.r, variance.r);\n" + 27865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float dist_uv = gauss_dist_uv(fg.gb, mean.gb, variance.gb);\n" + 27965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " gl_FragColor = vec4(0.5*fg.rg, dist_scale*dist_y, dist_scale*dist_uv);\n" + 28065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "}\n"; 28165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 28265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Foreground/background mask decision shader. Decides whether a frame is in the foreground or 28365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // the background using a hierarchical threshold on the distance. Binary foreground/background 28465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // mask is placed in the alpha channel. The RGB channels contain debug information. 28565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String mBgMaskShader = 28665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_0;\n" + 28765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float accept_variance;\n" + 28865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform vec2 yuv_weights;\n" + 28965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float scale_lrg;\n" + 29065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float scale_mid;\n" + 29165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float scale_sml;\n" + 29265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float exp_lrg;\n" + 29365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float exp_mid;\n" + 29465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float exp_sml;\n" + 29565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "varying vec2 v_texcoord;\n" + 29665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Decide whether pixel is foreground or background based on Y and UV 29765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // distance and maximum acceptable variance. 29865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // yuv_weights.x is smaller than yuv_weights.y to discount the influence of shadow 29965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "bool is_fg(vec2 dist_yc, float accept_variance) {\n" + 30065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " return ( dot(yuv_weights, dist_yc) >= accept_variance );\n" + 30165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "}\n" + 30265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "void main() {\n" + 30365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 dist_lrg_sc = texture2D(tex_sampler_0, v_texcoord, exp_lrg);\n" + 30465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 dist_mid_sc = texture2D(tex_sampler_0, v_texcoord, exp_mid);\n" + 30565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 dist_sml_sc = texture2D(tex_sampler_0, v_texcoord, exp_sml);\n" + 30665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec2 dist_lrg = inv_dist_scale * dist_lrg_sc.ba;\n" + 30765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec2 dist_mid = inv_dist_scale * dist_mid_sc.ba;\n" + 30865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec2 dist_sml = inv_dist_scale * dist_sml_sc.ba;\n" + 30965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec2 norm_dist = 0.75 * dist_sml / accept_variance;\n" + // For debug viz 31065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " bool is_fg_lrg = is_fg(dist_lrg, accept_variance * scale_lrg);\n" + 31165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " bool is_fg_mid = is_fg_lrg || is_fg(dist_mid, accept_variance * scale_mid);\n" + 31265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float is_fg_sml =\n" + 31365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float(is_fg_mid || is_fg(dist_sml, accept_variance * scale_sml));\n" + 31465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float alpha = 0.5 * is_fg_sml + 0.3 * float(is_fg_mid) + 0.2 * float(is_fg_lrg);\n" + 31565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " gl_FragColor = vec4(alpha, norm_dist, is_fg_sml);\n" + 31665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "}\n"; 31765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 31865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Automatic White Balance parameter decision shader 31965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Use the Gray World assumption that in a white balance corrected image, the average of R, G, B 32065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // channel will be a common gray value. 32165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // To match the white balance of foreground and background, the average of R, G, B channel of 32265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // two videos should match. 32365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Inputs: 32465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_0: Mip-map for foreground (live) video frame. 32565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_1: Mip-map for background (playback) video frame. 32665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // pyramid_depth: Depth of input frames' mip-maps. 32765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String mAutomaticWhiteBalance = 32865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_0;\n" + 32965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_1;\n" + 33065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float pyramid_depth;\n" + 33165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform bool autowb_toggle;\n" + 33265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "varying vec2 v_texcoord;\n" + 33365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "void main() {\n" + 33465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 mean_video = texture2D(tex_sampler_0, v_texcoord, pyramid_depth);\n"+ 33565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 mean_bg = texture2D(tex_sampler_1, v_texcoord, pyramid_depth);\n" + 33665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // If Auto WB is toggled off, the return texture will be a unicolor texture of value 1 33765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // If Auto WB is toggled on, the return texture will be a unicolor texture with 33865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // adjustment parameters for R and B channels stored in the corresponding channel 33965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float green_normalizer = mean_video.g / mean_bg.g;\n"+ 34065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 adjusted_value = vec4(mean_bg.r / mean_video.r * green_normalizer, 1., \n" + 34165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " mean_bg.b / mean_video.b * green_normalizer, 1.) * auto_wb_scale; \n" + 34265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " gl_FragColor = autowb_toggle ? adjusted_value : vec4(auto_wb_scale);\n" + 34365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "}\n"; 34465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 34565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 34665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Background subtraction shader. Uses a mipmap of the binary mask map to blend smoothly between 34765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // foreground and background 34865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Inputs: 34965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_0: Foreground (live) video frame. 35065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_1: Background (playback) video frame. 35165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_2: Foreground/background mask. 35265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_3: Auto white-balance factors. 35365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String mBgSubtractShader = 35465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform mat3 bg_fit_transform;\n" + 35565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float mask_blend_bg;\n" + 35665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float mask_blend_fg;\n" + 35765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float exposure_change;\n" + 35865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float whitebalancered_change;\n" + 35965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float whitebalanceblue_change;\n" + 36065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_0;\n" + 36165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_1;\n" + 36265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_2;\n" + 36365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_3;\n" + 36465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "varying vec2 v_texcoord;\n" + 36565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "void main() {\n" + 36665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec2 bg_texcoord = (bg_fit_transform * vec3(v_texcoord, 1.)).xy;\n" + 36765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 bg_rgb = texture2D(tex_sampler_1, bg_texcoord);\n" + 36865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // The foreground texture is modified by multiplying both manual and auto white balance changes in R and B 36965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // channel and multiplying exposure change in all R, G, B channels. 37065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 wb_auto_scale = texture2D(tex_sampler_3, v_texcoord) * exposure_change / auto_wb_scale;\n" + 37165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 wb_manual_scale = vec4(1. + whitebalancered_change, 1., 1. + whitebalanceblue_change, 1.);\n" + 37265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord);\n" + 37365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 fg_adjusted = fg_rgb * wb_manual_scale * wb_auto_scale;\n"+ 37465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 mask = texture2D(tex_sampler_2, v_texcoord, \n" + 37565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " " + MASK_SMOOTH_EXPONENT + ");\n" + 37665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float alpha = smoothstep(mask_blend_bg, mask_blend_fg, mask.a);\n" + 37765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " gl_FragColor = mix(bg_rgb, fg_adjusted, alpha);\n"; 37865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 37965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // May the Force... Makes the foreground object translucent blue, with a bright 38065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // blue-white outline 38165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String mBgSubtractForceShader = 38265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 ghost_rgb = (fg_adjusted * 0.7 + vec4(0.3,0.3,0.4,0.))*0.65 + \n" + 38365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " 0.35*bg_rgb;\n" + 38465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float glow_start = 0.75 * mask_blend_bg; \n"+ 38565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float glow_max = mask_blend_bg; \n"+ 38665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " gl_FragColor = mask.a < glow_start ? bg_rgb : \n" + 38765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " mask.a < glow_max ? mix(bg_rgb, vec4(0.9,0.9,1.0,1.0), \n" + 38865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " (mask.a - glow_start) / (glow_max - glow_start) ) : \n" + 38965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " mask.a < mask_blend_fg ? mix(vec4(0.9,0.9,1.0,1.0), ghost_rgb, \n" + 39065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " (mask.a - glow_max) / (mask_blend_fg - glow_max) ) : \n" + 39165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " ghost_rgb;\n" + 39265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "}\n"; 39365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 39465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Background model mean update shader. Skews the current model mean toward the most recent pixel 39565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // value for a pixel, weighted by the learning rate and by whether the pixel is classified as 39665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // foreground or background. 39765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Inputs: 39865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_0: Mip-map for foreground (live) video frame. 39965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_1: Background mean mask. 40065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_2: Foreground/background mask. 40165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // subsample_level: Level on foreground frame's mip-map. 40265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String mUpdateBgModelMeanShader = 40365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_0;\n" + 40465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_1;\n" + 40565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_2;\n" + 40665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float subsample_level;\n" + 40765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "varying vec2 v_texcoord;\n" + 40865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "void main() {\n" + 40965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" + 41065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 fg = coeff_yuv * vec4(fg_rgb.rgb, 1.);\n" + 41165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" + 41265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 mask = texture2D(tex_sampler_2, v_texcoord, \n" + 41365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " " + MASK_SMOOTH_EXPONENT + ");\n" + 41465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "\n" + 41565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float alpha = local_adapt_rate(mask.a);\n" + 41665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 new_mean = mix(mean, fg, alpha);\n" + 41765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " gl_FragColor = new_mean;\n" + 41865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "}\n"; 41965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 42065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Background model variance update shader. Skews the current model variance toward the most 42165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // recent variance for the pixel, weighted by the learning rate and by whether the pixel is 42265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // classified as foreground or background. 42365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Inputs: 42465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_0: Mip-map for foreground (live) video frame. 42565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_1: Background mean mask. 42665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_2: Background variance mask. 42765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // tex_sampler_3: Foreground/background mask. 42865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // subsample_level: Level on foreground frame's mip-map. 42965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // TODO: to improve efficiency, use single mark for mean + variance, then merge this into 43065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // mUpdateBgModelMeanShader. 43165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String mUpdateBgModelVarianceShader = 43265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_0;\n" + 43365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_1;\n" + 43465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_2;\n" + 43565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_3;\n" + 43665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float subsample_level;\n" + 43765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "varying vec2 v_texcoord;\n" + 43865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "void main() {\n" + 43965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" + 44065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 fg = coeff_yuv * vec4(fg_rgb.rgb, 1.);\n" + 44165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" + 44265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 variance = inv_var_scale * texture2D(tex_sampler_2, v_texcoord);\n" + 44365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 mask = texture2D(tex_sampler_3, v_texcoord, \n" + 44465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " " + MASK_SMOOTH_EXPONENT + ");\n" + 44565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "\n" + 44665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float alpha = local_adapt_rate(mask.a);\n" + 44765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 cur_variance = (fg-mean)*(fg-mean);\n" + 44865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 new_variance = mix(variance, cur_variance, alpha);\n" + 44965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " new_variance = max(new_variance, vec4(min_variance));\n" + 45065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " gl_FragColor = var_scale * new_variance;\n" + 45165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "}\n"; 45265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 45365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Background verification shader. Skews the current background verification mask towards the 45465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // most recent frame, weighted by the learning rate. 45565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private static final String mMaskVerifyShader = 45665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_0;\n" + 45765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform sampler2D tex_sampler_1;\n" + 45865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "uniform float verify_rate;\n" + 45965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "varying vec2 v_texcoord;\n" + 46065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "void main() {\n" + 46165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 lastmask = texture2D(tex_sampler_0, v_texcoord);\n" + 46265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " vec4 mask = texture2D(tex_sampler_1, v_texcoord);\n" + 46365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " float newmask = mix(lastmask.a, mask.a, verify_rate);\n" + 46465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " gl_FragColor = vec4(0., 0., 0., newmask);\n" + 46565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn "}\n"; 46665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 46765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Shader program objects */ 46865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 46965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private ShaderProgram mBgDistProgram; 47065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private ShaderProgram mBgMaskProgram; 47165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private ShaderProgram mBgSubtractProgram; 47265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private ShaderProgram mBgUpdateMeanProgram; 47365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private ShaderProgram mBgUpdateVarianceProgram; 47465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private ShaderProgram mCopyOutProgram; 47565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private ShaderProgram mAutomaticWhiteBalanceProgram; 47665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private ShaderProgram mMaskVerifyProgram; 47765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private ShaderProgram copyShaderProgram; 47865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 47965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Background model storage */ 48065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 48165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private boolean mPingPong; 48265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private GLFrame mBgMean[]; 48365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private GLFrame mBgVariance[]; 48465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private GLFrame mMaskVerify[]; 48565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private GLFrame mDistance; 48665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private GLFrame mAutoWB; 48765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private GLFrame mMask; 48865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private GLFrame mVideoInput; 48965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private GLFrame mBgInput; 49065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private GLFrame mMaskAverage; 49165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 49265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Overall filter state */ 49365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 49465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private boolean isOpen; 49565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mFrameCount; 49665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private boolean mStartLearning; 49765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private boolean mBackgroundFitModeChanged; 49865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private float mRelativeAspect; 49965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mPyramidDepth; 50065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int mSubsampleLevel; 50165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 50265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Learning listener object */ 50365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 50465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public interface LearningDoneListener { 50565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void onLearningDone(BackDropperFilter filter); 50665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 50765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 50865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn /** Public Filter methods */ 50965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 51065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public BackDropperFilter(String name) { 51165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn super(name); 51265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 51365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 5144239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala 5154239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala String adjStr = SystemProperties.get("ro.media.effect.bgdropper.adj"); 5164239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala if (adjStr.length() > 0) { 5174239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala try { 5184239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala mAcceptStddev += Float.parseFloat(adjStr); 5194239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala if (mLogVerbose) { 5204239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala Log.v(TAG, "Adjusting accept threshold by " + adjStr + 5214239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala ", now " + mAcceptStddev); 5224239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala } 5234239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala } catch (NumberFormatException e) { 5244239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala Log.e(TAG, 5254239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala "Badly formatted property ro.media.effect.bgdropper.adj: " + adjStr); 5264239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala } 5274239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala } 52865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 52965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 53065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 53165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void setupPorts() { 53265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Inputs. 53365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // TODO: Target should be GPU, but relaxed for now. 53465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn FrameFormat imageFormat = ImageFormat.create(ImageFormat.COLORSPACE_RGBA, 53565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn FrameFormat.TARGET_UNSPECIFIED); 53665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn for (String inputName : mInputNames) { 53765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn addMaskedInputPort(inputName, imageFormat); 53865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 53965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Normal outputs 54065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn for (String outputName : mOutputNames) { 54165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn addOutputBasedOnInput(outputName, "video"); 54265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 54365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 54465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Debug outputs 54565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mProvideDebugOutputs) { 54665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn for (String outputName : mDebugOutputNames) { 54765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn addOutputBasedOnInput(outputName, "video"); 54865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 54965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 55065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 55165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 55265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 55365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public FrameFormat getOutputFormat(String portName, FrameFormat inputFormat) { 55465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Create memory format based on video input. 55565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn MutableFrameFormat format = inputFormat.mutableCopy(); 55665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Is this a debug output port? If so, leave dimensions unspecified. 55765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (!Arrays.asList(mOutputNames).contains(portName)) { 55865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn format.setDimensions(FrameFormat.SIZE_UNSPECIFIED, FrameFormat.SIZE_UNSPECIFIED); 55965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 56065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn return format; 56165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 56265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 56365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private boolean createMemoryFormat(FrameFormat inputFormat) { 56465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // We can't resize because that would require re-learning. 56565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mMemoryFormat != null) { 56665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn return false; 56765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 56865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 56965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (inputFormat.getWidth() == FrameFormat.SIZE_UNSPECIFIED || 57065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn inputFormat.getHeight() == FrameFormat.SIZE_UNSPECIFIED) { 57165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn throw new RuntimeException("Attempting to process input frame with unknown size"); 57265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 57365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 57465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskFormat = inputFormat.mutableCopy(); 57565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn int maskWidth = (int)Math.pow(2, mMaskWidthExp); 57665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn int maskHeight = (int)Math.pow(2, mMaskHeightExp); 57765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskFormat.setDimensions(maskWidth, maskHeight); 57865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 57965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mPyramidDepth = Math.max(mMaskWidthExp, mMaskHeightExp); 58065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMemoryFormat = mMaskFormat.mutableCopy(); 58165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn int widthExp = Math.max(mMaskWidthExp, pyramidLevel(inputFormat.getWidth())); 58265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn int heightExp = Math.max(mMaskHeightExp, pyramidLevel(inputFormat.getHeight())); 58365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mPyramidDepth = Math.max(widthExp, heightExp); 58465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn int memWidth = Math.max(maskWidth, (int)Math.pow(2, widthExp)); 58565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn int memHeight = Math.max(maskHeight, (int)Math.pow(2, heightExp)); 58665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMemoryFormat.setDimensions(memWidth, memHeight); 58765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mSubsampleLevel = mPyramidDepth - Math.max(mMaskWidthExp, mMaskHeightExp); 58865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 58965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) { 59065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Log.v(TAG, "Mask frames size " + maskWidth + " x " + maskHeight); 59165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Log.v(TAG, "Pyramid levels " + widthExp + " x " + heightExp); 59265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Log.v(TAG, "Memory frames size " + memWidth + " x " + memHeight); 59365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 59465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 59565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mAverageFormat = inputFormat.mutableCopy(); 59665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mAverageFormat.setDimensions(1,1); 59765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn return true; 59865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 59965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 60065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void prepare(FilterContext context){ 60165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "Preparing BackDropperFilter!"); 60265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 60365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMean = new GLFrame[2]; 60465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgVariance = new GLFrame[2]; 60565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskVerify = new GLFrame[2]; 60665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn copyShaderProgram = ShaderProgram.createIdentity(context); 60765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 60865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 60965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private void allocateFrames(FrameFormat inputFormat, FilterContext context) { 61065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (!createMemoryFormat(inputFormat)) { 61165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn return; // All set. 61265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 61365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "Allocating BackDropperFilter frames"); 61465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 61565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Create initial background model values 61665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn int numBytes = mMaskFormat.getSize(); 61765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn byte[] initialBgMean = new byte[numBytes]; 61865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn byte[] initialBgVariance = new byte[numBytes]; 61965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn byte[] initialMaskVerify = new byte[numBytes]; 62065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn for (int i = 0; i < numBytes; i++) { 62165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn initialBgMean[i] = (byte)128; 62265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn initialBgVariance[i] = (byte)10; 62365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn initialMaskVerify[i] = (byte)0; 62465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 62565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 62665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Get frames to store background model in 62765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn for (int i = 0; i < 2; i++) { 62865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMean[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 62965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMean[i].setData(initialBgMean, 0, numBytes); 63065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 63165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgVariance[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 63265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgVariance[i].setData(initialBgVariance, 0, numBytes); 63365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 63465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskVerify[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 63565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskVerify[i].setData(initialMaskVerify, 0, numBytes); 63665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 63765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 63865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Get frames to store other textures in 63965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "Done allocating texture for Mean and Variance objects!"); 64065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 64165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mDistance = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 64265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMask = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 64365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mAutoWB = (GLFrame)context.getFrameManager().newFrame(mAverageFormat); 64465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mVideoInput = (GLFrame)context.getFrameManager().newFrame(mMemoryFormat); 64565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgInput = (GLFrame)context.getFrameManager().newFrame(mMemoryFormat); 64665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskAverage = (GLFrame)context.getFrameManager().newFrame(mAverageFormat); 64765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 64865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Create shader programs 64965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgDistProgram = new ShaderProgram(context, mSharedUtilShader + mBgDistanceShader); 65065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgDistProgram.setHostValue("subsample_level", (float)mSubsampleLevel); 65165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 65265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram = new ShaderProgram(context, mSharedUtilShader + mBgMaskShader); 65365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("accept_variance", mAcceptStddev * mAcceptStddev); 65465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn float[] yuvWeights = { mLumScale, mChromaScale }; 65565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("yuv_weights", yuvWeights ); 65665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("scale_lrg", mHierarchyLrgScale); 65765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("scale_mid", mHierarchyMidScale); 65865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("scale_sml", mHierarchySmlScale); 65965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("exp_lrg", (float)(mSubsampleLevel + mHierarchyLrgExp)); 66065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("exp_mid", (float)(mSubsampleLevel + mHierarchyMidExp)); 66165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("exp_sml", (float)(mSubsampleLevel + mHierarchySmlExp)); 66265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 66365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mUseTheForce) { 66465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram = new ShaderProgram(context, mSharedUtilShader + mBgSubtractShader + mBgSubtractForceShader); 66565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 66665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram = new ShaderProgram(context, mSharedUtilShader + mBgSubtractShader + "}\n"); 66765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 66865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.setHostValue("bg_fit_transform", DEFAULT_BG_FIT_TRANSFORM); 66965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.setHostValue("mask_blend_bg", mMaskBg); 67065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.setHostValue("mask_blend_fg", mMaskFg); 67165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.setHostValue("exposure_change", mExposureChange); 67265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.setHostValue("whitebalanceblue_change", mWhiteBalanceBlueChange); 67365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.setHostValue("whitebalancered_change", mWhiteBalanceRedChange); 67465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 67565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 67665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateMeanProgram = new ShaderProgram(context, mSharedUtilShader + mUpdateBgModelMeanShader); 67765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateMeanProgram.setHostValue("subsample_level", (float)mSubsampleLevel); 67865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 67965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateVarianceProgram = new ShaderProgram(context, mSharedUtilShader + mUpdateBgModelVarianceShader); 68065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateVarianceProgram.setHostValue("subsample_level", (float)mSubsampleLevel); 68165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 68265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mCopyOutProgram = ShaderProgram.createIdentity(context); 68365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 68465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mAutomaticWhiteBalanceProgram = new ShaderProgram(context, mSharedUtilShader + mAutomaticWhiteBalance); 68565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mAutomaticWhiteBalanceProgram.setHostValue("pyramid_depth", (float)mPyramidDepth); 68665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mAutomaticWhiteBalanceProgram.setHostValue("autowb_toggle", mAutoWBToggle); 68765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 68865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskVerifyProgram = new ShaderProgram(context, mSharedUtilShader + mMaskVerifyShader); 68965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskVerifyProgram.setHostValue("verify_rate", mVerifyRate); 69065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 69165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "Shader width set to " + mMemoryFormat.getWidth()); 69265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 69365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mRelativeAspect = 1.f; 69465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 69565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mFrameCount = 0; 69665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mStartLearning = true; 69765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 69865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 69965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void process(FilterContext context) { 70065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Grab inputs and ready intermediate frames and outputs. 70165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame video = pullInput("video"); 70265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame background = pullInput("background"); 70365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn allocateFrames(video.getFormat(), context); 70465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 70565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Update learning rate after initial learning period 70665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mStartLearning) { 70765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "Starting learning"); 70865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateMeanProgram.setHostValue("bg_adapt_rate", mAdaptRateLearning); 70965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateMeanProgram.setHostValue("fg_adapt_rate", mAdaptRateLearning); 71065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateVarianceProgram.setHostValue("bg_adapt_rate", mAdaptRateLearning); 71165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateVarianceProgram.setHostValue("fg_adapt_rate", mAdaptRateLearning); 71265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mFrameCount = 0; 71365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 71465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 71565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Select correct pingpong buffers 71665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn int inputIndex = mPingPong ? 0 : 1; 71765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn int outputIndex = mPingPong ? 1 : 0; 71865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mPingPong = !mPingPong; 71965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 72065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Check relative aspect ratios 72165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn updateBgScaling(video, background, mBackgroundFitModeChanged); 72265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBackgroundFitModeChanged = false; 72365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 72465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Make copies for input frames to GLFrames 72565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 72665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn copyShaderProgram.process(video, mVideoInput); 72765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn copyShaderProgram.process(background, mBgInput); 72865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 72965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mVideoInput.generateMipMap(); 73065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mVideoInput.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 73165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn GLES20.GL_LINEAR_MIPMAP_NEAREST); 73265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 73365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgInput.generateMipMap(); 73465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgInput.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 73565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn GLES20.GL_LINEAR_MIPMAP_NEAREST); 73665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 7374239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala if (mStartLearning) { 7384239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala copyShaderProgram.process(mVideoInput, mBgMean[inputIndex]); 7394239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala mStartLearning = false; 7404239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala } 7414239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala 74265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Process shaders 74365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame[] distInputs = { mVideoInput, mBgMean[inputIndex], mBgVariance[inputIndex] }; 74465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgDistProgram.process(distInputs, mDistance); 74565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mDistance.generateMipMap(); 74665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mDistance.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 74765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn GLES20.GL_LINEAR_MIPMAP_NEAREST); 74865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 74965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.process(mDistance, mMask); 75065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMask.generateMipMap(); 75165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMask.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 75265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn GLES20.GL_LINEAR_MIPMAP_NEAREST); 75365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 75465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame[] autoWBInputs = { mVideoInput, mBgInput }; 75565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mAutomaticWhiteBalanceProgram.process(autoWBInputs, mAutoWB); 75665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 75765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mFrameCount <= mLearningDuration) { 75865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // During learning 75965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn pushOutput("video", video); 76065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 76165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mFrameCount == mLearningDuration - mLearningVerifyDuration) { 76265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn copyShaderProgram.process(mMask, mMaskVerify[outputIndex]); 76365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 76465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateMeanProgram.setHostValue("bg_adapt_rate", mAdaptRateBg); 76565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateMeanProgram.setHostValue("fg_adapt_rate", mAdaptRateFg); 76665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateVarianceProgram.setHostValue("bg_adapt_rate", mAdaptRateBg); 76765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateVarianceProgram.setHostValue("fg_adapt_rate", mAdaptRateFg); 76865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 76965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 77065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (mFrameCount > mLearningDuration - mLearningVerifyDuration) { 77165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // In the learning verification stage, compute background masks and a weighted average 77265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // with weights grow exponentially with time 77365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame[] maskVerifyInputs = {mMaskVerify[inputIndex], mMask}; 77465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskVerifyProgram.process(maskVerifyInputs, mMaskVerify[outputIndex]); 77565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskVerify[outputIndex].generateMipMap(); 77665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskVerify[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 77765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn GLES20.GL_LINEAR_MIPMAP_NEAREST); 77865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 77965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 78065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mFrameCount == mLearningDuration) { 78165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // In the last verification frame, verify if the verification mask is almost blank 78265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // If not, restart learning 78365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn copyShaderProgram.process(mMaskVerify[outputIndex], mMaskAverage); 78465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn ByteBuffer mMaskAverageByteBuffer = mMaskAverage.getData(); 78565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn byte[] mask_average = mMaskAverageByteBuffer.array(); 78665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn int bi = (int)(mask_average[3] & 0xFF); 7874239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala 7884239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala if (mLogVerbose) { 7894239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala Log.v(TAG, 7904239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala String.format("Mask_average is %d, threshold is %d", 7914239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala bi, DEFAULT_LEARNING_DONE_THRESHOLD)); 7924239373aedb5f95e7edcc3c75920eb3e265b667cEino-Ville Talvala } 79365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 79465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (bi >= DEFAULT_LEARNING_DONE_THRESHOLD) { 79565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mStartLearning = true; // Restart learning 79665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 79765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "Learning done"); 79865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLearningDoneListener != null) { 79965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mLearningDoneListener.onLearningDone(this); 80065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 80165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 80265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 80365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 80465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame output = context.getFrameManager().newFrame(video.getFormat()); 80565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame[] subtractInputs = { video, background, mMask, mAutoWB }; 80665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.process(subtractInputs, output); 80765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn pushOutput("video", output); 80865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn output.release(); 80965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 81065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 81165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Compute mean and variance of the background 81265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mFrameCount < mLearningDuration - mLearningVerifyDuration || 81365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mAdaptRateBg > 0.0 || mAdaptRateFg > 0.0) { 81465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame[] meanUpdateInputs = { mVideoInput, mBgMean[inputIndex], mMask }; 81565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateMeanProgram.process(meanUpdateInputs, mBgMean[outputIndex]); 81665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMean[outputIndex].generateMipMap(); 81765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMean[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 81865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn GLES20.GL_LINEAR_MIPMAP_NEAREST); 81965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 82065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame[] varianceUpdateInputs = { 82165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mVideoInput, mBgMean[inputIndex], mBgVariance[inputIndex], mMask 82265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn }; 82365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgUpdateVarianceProgram.process(varianceUpdateInputs, mBgVariance[outputIndex]); 82465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgVariance[outputIndex].generateMipMap(); 82565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgVariance[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 82665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn GLES20.GL_LINEAR_MIPMAP_NEAREST); 82765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 82865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 82965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Provide debug output to two smaller viewers 83065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mProvideDebugOutputs) { 83165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame dbg1 = context.getFrameManager().newFrame(video.getFormat()); 83265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mCopyOutProgram.process(video, dbg1); 83365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn pushOutput("debug1", dbg1); 83465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn dbg1.release(); 83565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 83665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Frame dbg2 = context.getFrameManager().newFrame(mMemoryFormat); 83765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mCopyOutProgram.process(mMask, dbg2); 83865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn pushOutput("debug2", dbg2); 83965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn dbg2.release(); 84065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 84165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 84265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mFrameCount++; 84365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 84465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) { 84565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mFrameCount % 30 == 0) { 84665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (startTime == -1) { 84765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn context.getGLEnvironment().activate(); 84865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn GLES20.glFinish(); 84965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn startTime = SystemClock.elapsedRealtime(); 85065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 85165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn context.getGLEnvironment().activate(); 85265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn GLES20.glFinish(); 85365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn long endTime = SystemClock.elapsedRealtime(); 85465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn Log.v(TAG, "Avg. frame duration: " + String.format("%.2f",(endTime-startTime)/30.) + 85565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn " ms. Avg. fps: " + String.format("%.2f", 1000./((endTime-startTime)/30.)) ); 85665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn startTime = endTime; 85765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 85865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 85965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 86065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 86165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 86265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private long startTime = -1; 86365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 86465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void close(FilterContext context) { 86565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mMemoryFormat == null) { 86665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn return; 86765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 86865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 86965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "Filter Closing!"); 87065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn for (int i = 0; i < 2; i++) { 87165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMean[i].release(); 87265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgVariance[i].release(); 87365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskVerify[i].release(); 87465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 87565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mDistance.release(); 87665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMask.release(); 87765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mAutoWB.release(); 87865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mVideoInput.release(); 87965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgInput.release(); 88065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMaskAverage.release(); 88165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 88265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mMemoryFormat = null; 88365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 88465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 88565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Relearn background model 88665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn synchronized public void relearn() { 88765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Let the processing thread know about learning restart 88865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mStartLearning = true; 88965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 89065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 89165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn @Override 89265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn public void fieldPortValueUpdated(String name, FilterContext context) { 89365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // TODO: Many of these can be made ProgramPorts! 89465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (name.equals("backgroundFitMode")) { 89565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBackgroundFitModeChanged = true; 89665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("acceptStddev")) { 89765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("accept_variance", mAcceptStddev * mAcceptStddev); 89865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("hierLrgScale")) { 89965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("scale_lrg", mHierarchyLrgScale); 90065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("hierMidScale")) { 90165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("scale_mid", mHierarchyMidScale); 90265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("hierSmlScale")) { 90365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("scale_sml", mHierarchySmlScale); 90465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("hierLrgExp")) { 90565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("exp_lrg", (float)(mSubsampleLevel + mHierarchyLrgExp)); 90665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("hierMidExp")) { 90765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("exp_mid", (float)(mSubsampleLevel + mHierarchyMidExp)); 90865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("hierSmlExp")) { 90965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("exp_sml", (float)(mSubsampleLevel + mHierarchySmlExp)); 91065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("lumScale") || name.equals("chromaScale")) { 91165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn float[] yuvWeights = { mLumScale, mChromaScale }; 91265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgMaskProgram.setHostValue("yuv_weights", yuvWeights ); 91365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("maskBg")) { 91465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.setHostValue("mask_blend_bg", mMaskBg); 91565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("maskFg")) { 91665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.setHostValue("mask_blend_fg", mMaskFg); 91765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("exposureChange")) { 91865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.setHostValue("exposure_change", mExposureChange); 91965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("whitebalanceredChange")) { 92065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.setHostValue("whitebalancered_change", mWhiteBalanceRedChange); 92165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("whitebalanceblueChange")) { 92265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.setHostValue("whitebalanceblue_change", mWhiteBalanceBlueChange); 92365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else if (name.equals("autowbToggle")){ 92465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mAutomaticWhiteBalanceProgram.setHostValue("autowb_toggle", mAutoWBToggle); 92565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 92665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 92765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 92865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private void updateBgScaling(Frame video, Frame background, boolean fitModeChanged) { 92965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn float foregroundAspect = (float)video.getFormat().getWidth() / video.getFormat().getHeight(); 93065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn float backgroundAspect = (float)background.getFormat().getWidth() / background.getFormat().getHeight(); 93165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn float currentRelativeAspect = foregroundAspect/backgroundAspect; 93265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (currentRelativeAspect != mRelativeAspect || fitModeChanged) { 93365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mRelativeAspect = currentRelativeAspect; 93465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn float xMin = 0.f, xWidth = 1.f, yMin = 0.f, yWidth = 1.f; 93565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn switch (mBackgroundFitMode) { 93665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn case BACKGROUND_STRETCH: 93765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Just map 1:1 93865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn break; 93965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn case BACKGROUND_FIT: 94065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mRelativeAspect > 1.0f) { 94165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Foreground is wider than background, scale down 94265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // background in X 94365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn xMin = 0.5f - 0.5f * mRelativeAspect; 94465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn xWidth = 1.f * mRelativeAspect; 94565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 94665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Foreground is taller than background, scale down 94765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // background in Y 94865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn yMin = 0.5f - 0.5f / mRelativeAspect; 94965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn yWidth = 1 / mRelativeAspect; 95065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 95165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn break; 95265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn case BACKGROUND_FILL_CROP: 95365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mRelativeAspect > 1.0f) { 95465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Foreground is wider than background, crop 95565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // background in Y 95665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn yMin = 0.5f - 0.5f / mRelativeAspect; 95765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn yWidth = 1.f / mRelativeAspect; 95865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 95965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Foreground is taller than background, crop 96065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // background in X 96165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn xMin = 0.5f - 0.5f * mRelativeAspect; 96265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn xWidth = mRelativeAspect; 96365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 96465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn break; 96565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 96665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // If mirroring is required (for ex. the camera mirrors the preview 96765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // in the front camera) 96865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // TODO: Backdropper does not attempt to apply any other transformation 96965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // than just flipping. However, in the current state, it's "x-axis" is always aligned 97065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // with the Camera's width. Hence, we need to define the mirroring based on the camera 97165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // orientation. In the future, a cleaner design would be to cast away all the rotation 97265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // in a separate place. 97365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mMirrorBg) { 97465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "Mirroring the background!"); 97565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Mirroring in portrait 97665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mOrientation == 0 || mOrientation == 180) { 97765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn xWidth = -xWidth; 97865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn xMin = 1.0f - xMin; 97965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } else { 98065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // Mirroring in landscape 98165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn yWidth = -yWidth; 98265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn yMin = 1.0f - yMin; 98365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 98465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 98565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn if (mLogVerbose) Log.v(TAG, "bgTransform: xMin, yMin, xWidth, yWidth : " + 98665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn xMin + ", " + yMin + ", " + xWidth + ", " + yWidth + 98765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn ", mRelAspRatio = " + mRelativeAspect); 98865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn // The following matrix is the transpose of the actual matrix 98965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn float[] bgTransform = {xWidth, 0.f, 0.f, 99065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 0.f, yWidth, 0.f, 99165953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn xMin, yMin, 1.f}; 99265953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn mBgSubtractProgram.setHostValue("bg_fit_transform", bgTransform); 99365953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 99465953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 99565953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 99665953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn private int pyramidLevel(int size) { 99765953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn return (int)Math.floor(Math.log10(size) / Math.log10(2)) - 1; 99865953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn } 99965953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn 100065953da4636fbf5f0a24b8f5f2b5fa7d76ff13d9Marius Renn} 1001