xref: /frameworks/base/tests/Camera2Tests/SmartCamera/SimpleCamera/src/androidx/media/filterfw/samples/simplecamera/ImageGoodnessFilter.java

/*
 * Copyright 2013 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
// Takes sharpness score, rates the image good if above 10, bad otherwise

package androidx.media.filterfw.samples.simplecamera;

import android.graphics.Bitmap;
import android.os.AsyncTask;
import android.util.Log;
import android.widget.ImageView;
import androidx.media.filterfw.Filter;
import androidx.media.filterfw.FrameImage2D;
import androidx.media.filterfw.FrameType;
import androidx.media.filterfw.FrameValue;
import androidx.media.filterfw.MffContext;
import androidx.media.filterfw.OutputPort;
import androidx.media.filterfw.Signature;

public class ImageGoodnessFilter extends Filter {

    private static final String TAG = "ImageGoodnessFilter";
    private static boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);

    private final static String GREAT = "Great Picture!";
    private final static String GOOD = "Good Picture!";
    private final static String OK = "Ok Picture";
    private final static String BAD = "Bad Picture";
    private final static String AWFUL = "Awful Picture";
    private final static float SMALL_SCORE_INC = 0.25f;
    private final static float BIG_SCORE_INC = 0.5f;
    private final static float LOW_VARIANCE = 0.1f;
    private final static float MEDIUM_VARIANCE = 10;
    private final static float HIGH_VARIANCE = 100;
    private float sharpnessMean = 0;
    private float sharpnessVar = 0;
    private float underExposureMean = 0;
    private float underExposureVar = 0;
    private float overExposureMean = 0;
    private float overExposureVar = 0;
    private float contrastMean = 0;
    private float contrastVar = 0;
    private float colorfulnessMean = 0;
    private float colorfulnessVar = 0;
    private float brightnessMean = 0;
    private float brightnessVar = 0;

    private float motionMean = 0;
    private float scoreMean = 0;
    private static final float DECAY = 0.03f;
    /**
     * @param context
     * @param name
     */
    public ImageGoodnessFilter(MffContext context, String name) {
        super(context, name);
    }

    @Override
    public Signature getSignature() {
        FrameType floatT = FrameType.single(float.class);
        FrameType imageIn = FrameType.image2D(FrameType.ELEMENT_RGBA8888, FrameType.READ_GPU);

        return new Signature()
                .addInputPort("sharpness", Signature.PORT_REQUIRED, floatT)
                .addInputPort("overExposure", Signature.PORT_REQUIRED, floatT)
                .addInputPort("underExposure", Signature.PORT_REQUIRED, floatT)
                .addInputPort("colorfulness", Signature.PORT_REQUIRED, floatT)
                .addInputPort("contrastRating", Signature.PORT_REQUIRED, floatT)
                .addInputPort("motionValues", Signature.PORT_REQUIRED, FrameType.array(float.class))
                .addInputPort("brightness", Signature.PORT_REQUIRED, floatT)
                .addInputPort("capturing", Signature.PORT_REQUIRED, FrameType.single(boolean.class))
                .addInputPort("image", Signature.PORT_REQUIRED, imageIn)
                .addOutputPort("goodOrBadPic", Signature.PORT_REQUIRED,
                        FrameType.single(String.class))
                .addOutputPort("score", Signature.PORT_OPTIONAL, floatT)
                .disallowOtherPorts();
    }

    /**
     * @see androidx.media.filterfw.Filter#onProcess()
     */
    @Override
    protected void onProcess() {
        FrameValue sharpnessFrameValue =
                getConnectedInputPort("sharpness").pullFrame().asFrameValue();
        float sharpness = ((Float)sharpnessFrameValue.getValue()).floatValue();

        FrameValue overExposureFrameValue =
                getConnectedInputPort("overExposure").pullFrame().asFrameValue();
        float overExposure = ((Float)overExposureFrameValue.getValue()).floatValue();

        FrameValue underExposureFrameValue =
                getConnectedInputPort("underExposure").pullFrame().asFrameValue();
        float underExposure = ((Float)underExposureFrameValue.getValue()).floatValue();

        FrameValue colorfulnessFrameValue =
                getConnectedInputPort("colorfulness").pullFrame().asFrameValue();
        float colorfulness = ((Float)colorfulnessFrameValue.getValue()).floatValue();

        FrameValue contrastRatingFrameValue =
                getConnectedInputPort("contrastRating").pullFrame().asFrameValue();
        float contrastRating = ((Float)contrastRatingFrameValue.getValue()).floatValue();

        FrameValue brightnessFrameValue =
                getConnectedInputPort("brightness").pullFrame().asFrameValue();
        float brightness = ((Float)brightnessFrameValue.getValue()).floatValue();

        FrameValue motionValuesFrameValue =
                getConnectedInputPort("motionValues").pullFrame().asFrameValue();
        float[] motionValues = (float[]) motionValuesFrameValue.getValue();


        float vectorAccel = (float) Math.sqrt(Math.pow(motionValues[0], 2) +
                Math.pow(motionValues[1], 2) + Math.pow(motionValues[2], 2));
        String outStr;

        FrameValue capturingFrameValue =
                getConnectedInputPort("capturing").pullFrame().asFrameValue();
        boolean capturing = (Boolean) capturingFrameValue.getValue();

        FrameImage2D inputImage = getConnectedInputPort("image").pullFrame().asFrameImage2D();


        // TODO: get rid of magic numbers
        float score = 0.0f;
        score = computePictureScore(vectorAccel, sharpness, underExposure, overExposure,
                    contrastRating, colorfulness, brightness);
        if (scoreMean == 0) scoreMean = score;
        else scoreMean = scoreMean * (1 - DECAY) + score * DECAY;

        if (motionMean == 0) motionMean = vectorAccel;
        else motionMean = motionMean * (1 - DECAY) + vectorAccel * DECAY;

        float classifierScore = classifierComputeScore(vectorAccel, sharpness, underExposure,
                colorfulness, contrastRating, score);

//        Log.v(TAG, "ClassifierScore:: " + classifierScore);
        final float GREAT_SCORE = 3.5f;
        final float GOOD_SCORE = 2.5f;
        final float OK_SCORE = 1.5f;
        final float BAD_SCORE = 0.5f;

        if (score >= GREAT_SCORE) {
            outStr = GREAT;
        } else if (score >= GOOD_SCORE) {
            outStr = GOOD;
        } else if (score >= OK_SCORE) {
            outStr = OK;
        } else if (score >= BAD_SCORE) {
            outStr = BAD;
        } else {
            outStr = AWFUL;
        }

        if(capturing) {
            if (outStr.equals(GREAT)) {
                // take a picture
                Bitmap bitmap = inputImage.toBitmap();

                new AsyncOperation().execute(bitmap);
                final float RESET_FEATURES = 0.01f;
                sharpnessMean = RESET_FEATURES;
                underExposureMean = RESET_FEATURES;
                overExposureMean = RESET_FEATURES;
                contrastMean = RESET_FEATURES;
                colorfulnessMean = RESET_FEATURES;
                brightnessMean = RESET_FEATURES;
            }
        }

        OutputPort outPort = getConnectedOutputPort("goodOrBadPic");
        FrameValue stringFrame = outPort.fetchAvailableFrame(null).asFrameValue();
        stringFrame.setValue(outStr);
        outPort.pushFrame(stringFrame);

        OutputPort scoreOutPort = getConnectedOutputPort("score");
        FrameValue scoreFrame = scoreOutPort.fetchAvailableFrame(null).asFrameValue();
        scoreFrame.setValue(score);
        scoreOutPort.pushFrame(scoreFrame);

    }

    private class AsyncOperation extends AsyncTask<Bitmap, Void, String> {
        private Bitmap b;
        protected void onPostExecute(String result) {
            ImageView view = SmartCamera.getImageView();
            view.setImageBitmap(b);
        }

        @Override
        protected String doInBackground(Bitmap... params) {
            // TODO Auto-generated method stub
            b = params[0];
            return null;
        }

    }
    // Returns a number between -1 and 1
    private float classifierComputeScore(float vectorAccel, float sharpness, float underExposure,
           float colorfulness, float contrast, float score) {
        float result = (-0.0223f * sharpness + -0.0563f * underExposure + 0.0137f * colorfulness
                + 0.3102f * contrast + 0.0314f * vectorAccel + -0.0094f * score + 0.0227f *
                sharpnessMean + 0.0459f * underExposureMean + -0.3934f * contrastMean +
                -0.0697f * motionMean + 0.0091f * scoreMean + -0.0152f);
        return result;
    }

    // Returns a number between -1 and 4 representing the score for this picture
    private float computePictureScore(float vector_accel, float sharpness,
            float underExposure, float overExposure, float contrastRating, float colorfulness,
            float brightness) {
        final float ACCELERATION_THRESHOLD_VERY_STEADY = 0.1f;
        final float ACCELERATION_THRESHOLD_STEADY = 0.3f;
        final float ACCELERATION_THRESHOLD_MOTION = 2f;

        float score = 0.0f;
        if (vector_accel > ACCELERATION_THRESHOLD_MOTION) {
            score -= (BIG_SCORE_INC + BIG_SCORE_INC); // set score to -1, bad pic
        } else if (vector_accel > ACCELERATION_THRESHOLD_STEADY) {
            score -= BIG_SCORE_INC;
            score = subComputeScore(sharpness, underExposure, overExposure, contrastRating,
                    colorfulness, brightness, score);
        } else if (vector_accel < ACCELERATION_THRESHOLD_VERY_STEADY) {
            score += BIG_SCORE_INC;
            score = subComputeScore(sharpness, underExposure, overExposure, contrastRating,
                    colorfulness, brightness, score);
        } else {
            score = subComputeScore(sharpness, underExposure, overExposure, contrastRating,
                    colorfulness, brightness, score);
        }
        return score;
    }

    // Changes the score by at most +/- 3.5
    private float subComputeScore(float sharpness, float underExposure, float overExposure,
                float contrastRating, float colorfulness, float brightness, float score) {
        // The score methods return values -0.5 to 0.5
        final float SHARPNESS_WEIGHT = 2;
        score += SHARPNESS_WEIGHT * sharpnessScore(sharpness);
        score += underExposureScore(underExposure);
        score += overExposureScore(overExposure);
        score += contrastScore(contrastRating);
        score += colorfulnessScore(colorfulness);
        score += brightnessScore(brightness);
        return score;
    }

    private float sharpnessScore(float sharpness) {
        if (sharpnessMean == 0) {
            sharpnessMean = sharpness;
            sharpnessVar = 0;
            return 0;
        } else {
            sharpnessMean = sharpnessMean * (1 - DECAY) + sharpness * DECAY;
            sharpnessVar = sharpnessVar * (1 - DECAY) + (sharpness - sharpnessMean) *
                    (sharpness - sharpnessMean) * DECAY;
            if (sharpnessVar < LOW_VARIANCE) {
                return BIG_SCORE_INC;
            } else if (sharpness < sharpnessMean && sharpnessVar > MEDIUM_VARIANCE) {
                return -BIG_SCORE_INC;
            } else if (sharpness < sharpnessMean) {
                return -SMALL_SCORE_INC;
            } else if (sharpness > sharpnessMean && sharpnessVar > HIGH_VARIANCE) {
                return 0;
            } else if (sharpness > sharpnessMean && sharpnessVar > MEDIUM_VARIANCE) {
                return SMALL_SCORE_INC;
            } else  {
                return BIG_SCORE_INC; // low variance, sharpness above the mean
            }
        }
    }

    private float underExposureScore(float underExposure) {
        if (underExposureMean == 0) {
            underExposureMean = underExposure;
            underExposureVar = 0;
            return 0;
        } else {
            underExposureMean = underExposureMean * (1 - DECAY) + underExposure * DECAY;
            underExposureVar = underExposureVar * (1 - DECAY) + (underExposure - underExposureMean)
                    * (underExposure - underExposureMean) * DECAY;
            if (underExposureVar < LOW_VARIANCE) {
                return BIG_SCORE_INC;
            } else if (underExposure > underExposureMean && underExposureVar > MEDIUM_VARIANCE) {
                return -BIG_SCORE_INC;
            } else if (underExposure > underExposureMean) {
                return -SMALL_SCORE_INC;
            } else if (underExposure < underExposureMean && underExposureVar > HIGH_VARIANCE) {
                return 0;
            } else if (underExposure < underExposureMean && underExposureVar > MEDIUM_VARIANCE) {
                return SMALL_SCORE_INC;
            } else {
                return BIG_SCORE_INC; // low variance, underExposure below the mean
            }
        }
    }

    private float overExposureScore(float overExposure) {
        if (overExposureMean == 0) {
            overExposureMean = overExposure;
            overExposureVar = 0;
            return 0;
        } else {
            overExposureMean = overExposureMean * (1 - DECAY) + overExposure * DECAY;
            overExposureVar = overExposureVar * (1 - DECAY) + (overExposure - overExposureMean) *
                    (overExposure - overExposureMean) * DECAY;
            if (overExposureVar < LOW_VARIANCE) {
                return BIG_SCORE_INC;
            } else if (overExposure > overExposureMean && overExposureVar > MEDIUM_VARIANCE) {
                return -BIG_SCORE_INC;
            } else if (overExposure > overExposureMean) {
                return -SMALL_SCORE_INC;
            } else if (overExposure < overExposureMean && overExposureVar > HIGH_VARIANCE) {
                return 0;
            } else if (overExposure < overExposureMean && overExposureVar > MEDIUM_VARIANCE) {
                return SMALL_SCORE_INC;
            } else {
                return BIG_SCORE_INC; // low variance, overExposure below the mean
            }
        }
    }

    private float contrastScore(float contrast) {
        if (contrastMean == 0) {
            contrastMean = contrast;
            contrastVar = 0;
            return 0;
        } else {
            contrastMean = contrastMean * (1 - DECAY) + contrast * DECAY;
            contrastVar = contrastVar * (1 - DECAY) + (contrast - contrastMean) *
                    (contrast - contrastMean) * DECAY;
            if (contrastVar < LOW_VARIANCE) {
                return BIG_SCORE_INC;
            } else if (contrast < contrastMean && contrastVar > MEDIUM_VARIANCE) {
                return -BIG_SCORE_INC;
            } else if (contrast < contrastMean) {
                return -SMALL_SCORE_INC;
            } else if (contrast > contrastMean && contrastVar > 100) {
                return 0;
            } else if (contrast > contrastMean && contrastVar > MEDIUM_VARIANCE) {
                return SMALL_SCORE_INC;
            } else {
                return BIG_SCORE_INC; // low variance, contrast above the mean
            }
        }
    }

    private float colorfulnessScore(float colorfulness) {
        if (colorfulnessMean == 0) {
            colorfulnessMean = colorfulness;
            colorfulnessVar = 0;
            return 0;
        } else {
            colorfulnessMean = colorfulnessMean * (1 - DECAY) + colorfulness * DECAY;
            colorfulnessVar = colorfulnessVar * (1 - DECAY) + (colorfulness - colorfulnessMean) *
                    (colorfulness - colorfulnessMean) * DECAY;
            if (colorfulnessVar < LOW_VARIANCE) {
                return BIG_SCORE_INC;
            } else if (colorfulness < colorfulnessMean && colorfulnessVar > MEDIUM_VARIANCE) {
                return -BIG_SCORE_INC;
            } else if (colorfulness < colorfulnessMean) {
                return -SMALL_SCORE_INC;
            } else if (colorfulness > colorfulnessMean && colorfulnessVar > 100) {
                return 0;
            } else if (colorfulness > colorfulnessMean && colorfulnessVar > MEDIUM_VARIANCE) {
                return SMALL_SCORE_INC;
            } else {
                return BIG_SCORE_INC; // low variance, colorfulness above the mean
            }
        }
    }

    private float brightnessScore(float brightness) {
        if (brightnessMean == 0) {
            brightnessMean = brightness;
            brightnessVar = 0;
            return 0;
        } else {
            brightnessMean = brightnessMean * (1 - DECAY) + brightness * DECAY;
            brightnessVar = brightnessVar * (1 - DECAY) + (brightness - brightnessMean) *
                    (brightness - brightnessMean) * DECAY;
            if (brightnessVar < LOW_VARIANCE) {
                return BIG_SCORE_INC;
            } else if (brightness < brightnessMean && brightnessVar > MEDIUM_VARIANCE) {
                return -BIG_SCORE_INC;
            } else if (brightness < brightnessMean) {
                return -SMALL_SCORE_INC;
            } else if (brightness > brightnessMean && brightnessVar > 100) {
                return 0;
            } else if (brightness > brightnessMean && brightnessVar > MEDIUM_VARIANCE) {
                return SMALL_SCORE_INC;
            } else {
                return BIG_SCORE_INC; // low variance, brightness above the mean
            }
        }
    }
}