1/*
2 * Copyright 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16// Takes sharpness score, rates the image good if above 10, bad otherwise
17
18package androidx.media.filterfw.samples.simplecamera;
19
20import android.graphics.Bitmap;
21import android.os.AsyncTask;
22import android.util.Log;
23import android.widget.ImageView;
24import androidx.media.filterfw.Filter;
25import androidx.media.filterfw.FrameImage2D;
26import androidx.media.filterfw.FrameType;
27import androidx.media.filterfw.FrameValue;
28import androidx.media.filterfw.MffContext;
29import androidx.media.filterfw.OutputPort;
30import androidx.media.filterfw.Signature;
31
32public class ImageGoodnessFilter extends Filter {
33
34    private static final String TAG = "ImageGoodnessFilter";
35    private static boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
36
37    private final static String GREAT = "Great Picture!";
38    private final static String GOOD = "Good Picture!";
39    private final static String OK = "Ok Picture";
40    private final static String BAD = "Bad Picture";
41    private final static String AWFUL = "Awful Picture";
42    private final static float SMALL_SCORE_INC = 0.25f;
43    private final static float BIG_SCORE_INC = 0.5f;
44    private final static float LOW_VARIANCE = 0.1f;
45    private final static float MEDIUM_VARIANCE = 10;
46    private final static float HIGH_VARIANCE = 100;
47    private float sharpnessMean = 0;
48    private float sharpnessVar = 0;
49    private float underExposureMean = 0;
50    private float underExposureVar = 0;
51    private float overExposureMean = 0;
52    private float overExposureVar = 0;
53    private float contrastMean = 0;
54    private float contrastVar = 0;
55    private float colorfulnessMean = 0;
56    private float colorfulnessVar = 0;
57    private float brightnessMean = 0;
58    private float brightnessVar = 0;
59
60    private float motionMean = 0;
61    private float scoreMean = 0;
62    private static final float DECAY = 0.03f;
63    /**
64     * @param context
65     * @param name
66     */
67    public ImageGoodnessFilter(MffContext context, String name) {
68        super(context, name);
69    }
70
71    @Override
72    public Signature getSignature() {
73        FrameType floatT = FrameType.single(float.class);
74        FrameType imageIn = FrameType.image2D(FrameType.ELEMENT_RGBA8888, FrameType.READ_GPU);
75
76        return new Signature()
77                .addInputPort("sharpness", Signature.PORT_REQUIRED, floatT)
78                .addInputPort("overExposure", Signature.PORT_REQUIRED, floatT)
79                .addInputPort("underExposure", Signature.PORT_REQUIRED, floatT)
80                .addInputPort("colorfulness", Signature.PORT_REQUIRED, floatT)
81                .addInputPort("contrastRating", Signature.PORT_REQUIRED, floatT)
82                .addInputPort("motionValues", Signature.PORT_REQUIRED, FrameType.array(float.class))
83                .addInputPort("brightness", Signature.PORT_REQUIRED, floatT)
84                .addInputPort("capturing", Signature.PORT_REQUIRED, FrameType.single(boolean.class))
85                .addInputPort("image", Signature.PORT_REQUIRED, imageIn)
86                .addOutputPort("goodOrBadPic", Signature.PORT_REQUIRED,
87                        FrameType.single(String.class))
88                .addOutputPort("score", Signature.PORT_OPTIONAL, floatT)
89                .disallowOtherPorts();
90    }
91
92    /**
93     * @see androidx.media.filterfw.Filter#onProcess()
94     */
95    @Override
96    protected void onProcess() {
97        FrameValue sharpnessFrameValue =
98                getConnectedInputPort("sharpness").pullFrame().asFrameValue();
99        float sharpness = ((Float)sharpnessFrameValue.getValue()).floatValue();
100
101        FrameValue overExposureFrameValue =
102                getConnectedInputPort("overExposure").pullFrame().asFrameValue();
103        float overExposure = ((Float)overExposureFrameValue.getValue()).floatValue();
104
105        FrameValue underExposureFrameValue =
106                getConnectedInputPort("underExposure").pullFrame().asFrameValue();
107        float underExposure = ((Float)underExposureFrameValue.getValue()).floatValue();
108
109        FrameValue colorfulnessFrameValue =
110                getConnectedInputPort("colorfulness").pullFrame().asFrameValue();
111        float colorfulness = ((Float)colorfulnessFrameValue.getValue()).floatValue();
112
113        FrameValue contrastRatingFrameValue =
114                getConnectedInputPort("contrastRating").pullFrame().asFrameValue();
115        float contrastRating = ((Float)contrastRatingFrameValue.getValue()).floatValue();
116
117        FrameValue brightnessFrameValue =
118                getConnectedInputPort("brightness").pullFrame().asFrameValue();
119        float brightness = ((Float)brightnessFrameValue.getValue()).floatValue();
120
121        FrameValue motionValuesFrameValue =
122                getConnectedInputPort("motionValues").pullFrame().asFrameValue();
123        float[] motionValues = (float[]) motionValuesFrameValue.getValue();
124
125
126        float vectorAccel = (float) Math.sqrt(Math.pow(motionValues[0], 2) +
127                Math.pow(motionValues[1], 2) + Math.pow(motionValues[2], 2));
128        String outStr;
129
130        FrameValue capturingFrameValue =
131                getConnectedInputPort("capturing").pullFrame().asFrameValue();
132        boolean capturing = (Boolean) capturingFrameValue.getValue();
133
134        FrameImage2D inputImage = getConnectedInputPort("image").pullFrame().asFrameImage2D();
135
136
137        // TODO: get rid of magic numbers
138        float score = 0.0f;
139        score = computePictureScore(vectorAccel, sharpness, underExposure, overExposure,
140                    contrastRating, colorfulness, brightness);
141        if (scoreMean == 0) scoreMean = score;
142        else scoreMean = scoreMean * (1 - DECAY) + score * DECAY;
143
144        if (motionMean == 0) motionMean = vectorAccel;
145        else motionMean = motionMean * (1 - DECAY) + vectorAccel * DECAY;
146
147        float classifierScore = classifierComputeScore(vectorAccel, sharpness, underExposure,
148                colorfulness, contrastRating, score);
149
150//        Log.v(TAG, "ClassifierScore:: " + classifierScore);
151        final float GREAT_SCORE = 3.5f;
152        final float GOOD_SCORE = 2.5f;
153        final float OK_SCORE = 1.5f;
154        final float BAD_SCORE = 0.5f;
155
156        if (score >= GREAT_SCORE) {
157            outStr = GREAT;
158        } else if (score >= GOOD_SCORE) {
159            outStr = GOOD;
160        } else if (score >= OK_SCORE) {
161            outStr = OK;
162        } else if (score >= BAD_SCORE) {
163            outStr = BAD;
164        } else {
165            outStr = AWFUL;
166        }
167
168        if(capturing) {
169            if (outStr.equals(GREAT)) {
170                // take a picture
171                Bitmap bitmap = inputImage.toBitmap();
172
173                new AsyncOperation().execute(bitmap);
174                final float RESET_FEATURES = 0.01f;
175                sharpnessMean = RESET_FEATURES;
176                underExposureMean = RESET_FEATURES;
177                overExposureMean = RESET_FEATURES;
178                contrastMean = RESET_FEATURES;
179                colorfulnessMean = RESET_FEATURES;
180                brightnessMean = RESET_FEATURES;
181            }
182        }
183
184        OutputPort outPort = getConnectedOutputPort("goodOrBadPic");
185        FrameValue stringFrame = outPort.fetchAvailableFrame(null).asFrameValue();
186        stringFrame.setValue(outStr);
187        outPort.pushFrame(stringFrame);
188
189        OutputPort scoreOutPort = getConnectedOutputPort("score");
190        FrameValue scoreFrame = scoreOutPort.fetchAvailableFrame(null).asFrameValue();
191        scoreFrame.setValue(score);
192        scoreOutPort.pushFrame(scoreFrame);
193
194    }
195
196    private class AsyncOperation extends AsyncTask<Bitmap, Void, String> {
197        private Bitmap b;
198        protected void onPostExecute(String result) {
199            ImageView view = SmartCamera.getImageView();
200            view.setImageBitmap(b);
201        }
202
203        @Override
204        protected String doInBackground(Bitmap... params) {
205            // TODO Auto-generated method stub
206            b = params[0];
207            return null;
208        }
209
210    }
211    // Returns a number between -1 and 1
212    private float classifierComputeScore(float vectorAccel, float sharpness, float underExposure,
213           float colorfulness, float contrast, float score) {
214        float result = (-0.0223f * sharpness + -0.0563f * underExposure + 0.0137f * colorfulness
215                + 0.3102f * contrast + 0.0314f * vectorAccel + -0.0094f * score + 0.0227f *
216                sharpnessMean + 0.0459f * underExposureMean + -0.3934f * contrastMean +
217                -0.0697f * motionMean + 0.0091f * scoreMean + -0.0152f);
218        return result;
219    }
220
221    // Returns a number between -1 and 4 representing the score for this picture
222    private float computePictureScore(float vector_accel, float sharpness,
223            float underExposure, float overExposure, float contrastRating, float colorfulness,
224            float brightness) {
225        final float ACCELERATION_THRESHOLD_VERY_STEADY = 0.1f;
226        final float ACCELERATION_THRESHOLD_STEADY = 0.3f;
227        final float ACCELERATION_THRESHOLD_MOTION = 2f;
228
229        float score = 0.0f;
230        if (vector_accel > ACCELERATION_THRESHOLD_MOTION) {
231            score -= (BIG_SCORE_INC + BIG_SCORE_INC); // set score to -1, bad pic
232        } else if (vector_accel > ACCELERATION_THRESHOLD_STEADY) {
233            score -= BIG_SCORE_INC;
234            score = subComputeScore(sharpness, underExposure, overExposure, contrastRating,
235                    colorfulness, brightness, score);
236        } else if (vector_accel < ACCELERATION_THRESHOLD_VERY_STEADY) {
237            score += BIG_SCORE_INC;
238            score = subComputeScore(sharpness, underExposure, overExposure, contrastRating,
239                    colorfulness, brightness, score);
240        } else {
241            score = subComputeScore(sharpness, underExposure, overExposure, contrastRating,
242                    colorfulness, brightness, score);
243        }
244        return score;
245    }
246
247    // Changes the score by at most +/- 3.5
248    private float subComputeScore(float sharpness, float underExposure, float overExposure,
249                float contrastRating, float colorfulness, float brightness, float score) {
250        // The score methods return values -0.5 to 0.5
251        final float SHARPNESS_WEIGHT = 2;
252        score += SHARPNESS_WEIGHT * sharpnessScore(sharpness);
253        score += underExposureScore(underExposure);
254        score += overExposureScore(overExposure);
255        score += contrastScore(contrastRating);
256        score += colorfulnessScore(colorfulness);
257        score += brightnessScore(brightness);
258        return score;
259    }
260
261    private float sharpnessScore(float sharpness) {
262        if (sharpnessMean == 0) {
263            sharpnessMean = sharpness;
264            sharpnessVar = 0;
265            return 0;
266        } else {
267            sharpnessMean = sharpnessMean * (1 - DECAY) + sharpness * DECAY;
268            sharpnessVar = sharpnessVar * (1 - DECAY) + (sharpness - sharpnessMean) *
269                    (sharpness - sharpnessMean) * DECAY;
270            if (sharpnessVar < LOW_VARIANCE) {
271                return BIG_SCORE_INC;
272            } else if (sharpness < sharpnessMean && sharpnessVar > MEDIUM_VARIANCE) {
273                return -BIG_SCORE_INC;
274            } else if (sharpness < sharpnessMean) {
275                return -SMALL_SCORE_INC;
276            } else if (sharpness > sharpnessMean && sharpnessVar > HIGH_VARIANCE) {
277                return 0;
278            } else if (sharpness > sharpnessMean && sharpnessVar > MEDIUM_VARIANCE) {
279                return SMALL_SCORE_INC;
280            } else  {
281                return BIG_SCORE_INC; // low variance, sharpness above the mean
282            }
283        }
284    }
285
286    private float underExposureScore(float underExposure) {
287        if (underExposureMean == 0) {
288            underExposureMean = underExposure;
289            underExposureVar = 0;
290            return 0;
291        } else {
292            underExposureMean = underExposureMean * (1 - DECAY) + underExposure * DECAY;
293            underExposureVar = underExposureVar * (1 - DECAY) + (underExposure - underExposureMean)
294                    * (underExposure - underExposureMean) * DECAY;
295            if (underExposureVar < LOW_VARIANCE) {
296                return BIG_SCORE_INC;
297            } else if (underExposure > underExposureMean && underExposureVar > MEDIUM_VARIANCE) {
298                return -BIG_SCORE_INC;
299            } else if (underExposure > underExposureMean) {
300                return -SMALL_SCORE_INC;
301            } else if (underExposure < underExposureMean && underExposureVar > HIGH_VARIANCE) {
302                return 0;
303            } else if (underExposure < underExposureMean && underExposureVar > MEDIUM_VARIANCE) {
304                return SMALL_SCORE_INC;
305            } else {
306                return BIG_SCORE_INC; // low variance, underExposure below the mean
307            }
308        }
309    }
310
311    private float overExposureScore(float overExposure) {
312        if (overExposureMean == 0) {
313            overExposureMean = overExposure;
314            overExposureVar = 0;
315            return 0;
316        } else {
317            overExposureMean = overExposureMean * (1 - DECAY) + overExposure * DECAY;
318            overExposureVar = overExposureVar * (1 - DECAY) + (overExposure - overExposureMean) *
319                    (overExposure - overExposureMean) * DECAY;
320            if (overExposureVar < LOW_VARIANCE) {
321                return BIG_SCORE_INC;
322            } else if (overExposure > overExposureMean && overExposureVar > MEDIUM_VARIANCE) {
323                return -BIG_SCORE_INC;
324            } else if (overExposure > overExposureMean) {
325                return -SMALL_SCORE_INC;
326            } else if (overExposure < overExposureMean && overExposureVar > HIGH_VARIANCE) {
327                return 0;
328            } else if (overExposure < overExposureMean && overExposureVar > MEDIUM_VARIANCE) {
329                return SMALL_SCORE_INC;
330            } else {
331                return BIG_SCORE_INC; // low variance, overExposure below the mean
332            }
333        }
334    }
335
336    private float contrastScore(float contrast) {
337        if (contrastMean == 0) {
338            contrastMean = contrast;
339            contrastVar = 0;
340            return 0;
341        } else {
342            contrastMean = contrastMean * (1 - DECAY) + contrast * DECAY;
343            contrastVar = contrastVar * (1 - DECAY) + (contrast - contrastMean) *
344                    (contrast - contrastMean) * DECAY;
345            if (contrastVar < LOW_VARIANCE) {
346                return BIG_SCORE_INC;
347            } else if (contrast < contrastMean && contrastVar > MEDIUM_VARIANCE) {
348                return -BIG_SCORE_INC;
349            } else if (contrast < contrastMean) {
350                return -SMALL_SCORE_INC;
351            } else if (contrast > contrastMean && contrastVar > 100) {
352                return 0;
353            } else if (contrast > contrastMean && contrastVar > MEDIUM_VARIANCE) {
354                return SMALL_SCORE_INC;
355            } else {
356                return BIG_SCORE_INC; // low variance, contrast above the mean
357            }
358        }
359    }
360
361    private float colorfulnessScore(float colorfulness) {
362        if (colorfulnessMean == 0) {
363            colorfulnessMean = colorfulness;
364            colorfulnessVar = 0;
365            return 0;
366        } else {
367            colorfulnessMean = colorfulnessMean * (1 - DECAY) + colorfulness * DECAY;
368            colorfulnessVar = colorfulnessVar * (1 - DECAY) + (colorfulness - colorfulnessMean) *
369                    (colorfulness - colorfulnessMean) * DECAY;
370            if (colorfulnessVar < LOW_VARIANCE) {
371                return BIG_SCORE_INC;
372            } else if (colorfulness < colorfulnessMean && colorfulnessVar > MEDIUM_VARIANCE) {
373                return -BIG_SCORE_INC;
374            } else if (colorfulness < colorfulnessMean) {
375                return -SMALL_SCORE_INC;
376            } else if (colorfulness > colorfulnessMean && colorfulnessVar > 100) {
377                return 0;
378            } else if (colorfulness > colorfulnessMean && colorfulnessVar > MEDIUM_VARIANCE) {
379                return SMALL_SCORE_INC;
380            } else {
381                return BIG_SCORE_INC; // low variance, colorfulness above the mean
382            }
383        }
384    }
385
386    private float brightnessScore(float brightness) {
387        if (brightnessMean == 0) {
388            brightnessMean = brightness;
389            brightnessVar = 0;
390            return 0;
391        } else {
392            brightnessMean = brightnessMean * (1 - DECAY) + brightness * DECAY;
393            brightnessVar = brightnessVar * (1 - DECAY) + (brightness - brightnessMean) *
394                    (brightness - brightnessMean) * DECAY;
395            if (brightnessVar < LOW_VARIANCE) {
396                return BIG_SCORE_INC;
397            } else if (brightness < brightnessMean && brightnessVar > MEDIUM_VARIANCE) {
398                return -BIG_SCORE_INC;
399            } else if (brightness < brightnessMean) {
400                return -SMALL_SCORE_INC;
401            } else if (brightness > brightnessMean && brightnessVar > 100) {
402                return 0;
403            } else if (brightness > brightnessMean && brightnessVar > MEDIUM_VARIANCE) {
404                return SMALL_SCORE_INC;
405            } else {
406                return BIG_SCORE_INC; // low variance, brightness above the mean
407            }
408        }
409    }
410}
411