1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.test.hwuicompare;
18
19import java.io.File;
20import java.io.FileInputStream;
21import java.io.FileOutputStream;
22import java.io.IOException;
23import java.util.ArrayList;
24import java.util.Comparator;
25import java.util.HashMap;
26import java.util.TreeSet;
27
28import org.json.JSONException;
29import org.json.JSONObject;
30
31import android.os.Bundle;
32import android.os.Environment;
33import android.os.Trace;
34import android.util.Log;
35import android.widget.ImageView;
36import android.widget.Toast;
37
38public class AutomaticActivity extends CompareActivity {
39    private static final String LOG_TAG = "AutomaticActivity";
40    private static final float ERROR_DISPLAY_THRESHOLD = 0.01f;
41    protected static final boolean DRAW_BITMAPS = false;
42
43    /**
44     * Threshold of error change required to consider a test regressed/improved
45     */
46    private static final float ERROR_CHANGE_THRESHOLD = 0.001f;
47
48    private static final float[] ERROR_CUTOFFS = {
49            0, 0.005f, 0.01f, 0.02f, 0.05f, 0.1f, 0.25f, 0.5f, 1f, 2f
50    };
51
52    private final float[] mErrorRates = new float[ERROR_CUTOFFS.length];
53    private float mTotalTests = 0;
54    private float mTotalError = 0;
55    private int mTestsRegressed = 0;
56    private int mTestsImproved = 0;
57
58    private ImageView mSoftwareImageView = null;
59    private ImageView mHardwareImageView = null;
60
61
62    public abstract static class FinalCallback {
63        abstract void report(String name, float value);
64        void complete() {};
65    }
66
67    private final ArrayList<FinalCallback> mFinalCallbacks = new ArrayList<FinalCallback>();
68
69    Runnable mRunnable = new Runnable() {
70        @Override
71        public void run() {
72            loadBitmaps();
73            if (mSoftwareBitmap == null || mHardwareBitmap == null) {
74                Log.e(LOG_TAG, "bitmap is null");
75                return;
76            }
77
78            if (DRAW_BITMAPS) {
79                mSoftwareImageView.setImageBitmap(mSoftwareBitmap);
80                mHardwareImageView.setImageBitmap(mHardwareBitmap);
81            }
82
83            Trace.traceBegin(Trace.TRACE_TAG_ALWAYS, "calculateError");
84            float error = mErrorCalculator.calcErrorRS(mSoftwareBitmap, mHardwareBitmap);
85            Trace.traceEnd(Trace.TRACE_TAG_ALWAYS);
86
87            final String[] modifierNames = DisplayModifier.getLastAppliedModifications();
88            handleError(modifierNames, error);
89
90            if (DisplayModifier.step()) {
91                finishTest();
92            } else {
93                mHardwareView.invalidate();
94                if (DRAW_BITMAPS) {
95                    mSoftwareImageView.invalidate();
96                    mHardwareImageView.invalidate();
97                }
98            }
99            mHandler.removeCallbacks(mRunnable);
100        }
101    };
102
103    @Override
104    protected void onPause() {
105        super.onPause();
106        mHandler.removeCallbacks(mRunnable);
107    };
108
109    @Override
110    protected void onCreate(Bundle savedInstanceState) {
111        super.onCreate(savedInstanceState);
112        setContentView(R.layout.automatic_layout);
113
114        mSoftwareImageView = findViewById(R.id.software_image_view);
115        mHardwareImageView = findViewById(R.id.hardware_image_view);
116
117        onCreateCommon(mRunnable);
118        beginTest();
119    }
120
121    private static class TestResult {
122        TestResult(String label, float error) {
123            mLabel = label;
124            mTotalError = error;
125            mCount = 1;
126        }
127        public void addInto(float error) {
128            mTotalError += error;
129            mCount++;
130        }
131        public float getAverage() {
132            return mTotalError / mCount;
133        }
134        final String mLabel;
135        float mTotalError;
136        int mCount;
137    }
138
139    JSONObject mOutputJson = null;
140    JSONObject mInputJson = null;
141    final HashMap<String, TestResult> mModifierResults = new HashMap<String, TestResult>();
142    final HashMap<String, TestResult> mIndividualResults = new HashMap<String, TestResult>();
143    final HashMap<String, TestResult> mModifierDiffResults = new HashMap<String, TestResult>();
144    final HashMap<String, TestResult> mIndividualDiffResults = new HashMap<String, TestResult>();
145    private void beginTest() {
146        mFinalCallbacks.add(new FinalCallback() {
147            @Override
148            void report(String name, float value) {
149                Log.d(LOG_TAG, name + " " + value);
150            };
151        });
152
153        File inputFile = new File(Environment.getExternalStorageDirectory(),
154                "CanvasCompareInput.json");
155        if (inputFile.exists() && inputFile.canRead() && inputFile.length() > 0) {
156            try {
157                FileInputStream inputStream = new FileInputStream(inputFile);
158                Log.d(LOG_TAG, "Parsing input file...");
159                StringBuffer content = new StringBuffer((int)inputFile.length());
160                byte[] buffer = new byte[1024];
161                while (inputStream.read(buffer) != -1) {
162                    content.append(new String(buffer));
163                }
164                mInputJson = new JSONObject(content.toString());
165                inputStream.close();
166                Log.d(LOG_TAG, "Parsed input file with " + mInputJson.length() + " entries");
167            } catch (JSONException e) {
168                Log.e(LOG_TAG, "error parsing input json", e);
169            } catch (IOException e) {
170                Log.e(LOG_TAG, "error reading input json from sd", e);
171            }
172        }
173
174        mOutputJson = new JSONObject();
175    }
176
177    private static void logTestResultHash(String label, HashMap<String, TestResult> map) {
178        Log.d(LOG_TAG, "---------------");
179        Log.d(LOG_TAG, label + ":");
180        Log.d(LOG_TAG, "---------------");
181        TreeSet<TestResult> set = new TreeSet<TestResult>(new Comparator<TestResult>() {
182            @Override
183            public int compare(TestResult lhs, TestResult rhs) {
184                if (lhs == rhs) return 0; // don't need to worry about complex equality
185
186                int cmp = Float.compare(lhs.getAverage(), rhs.getAverage());
187                if (cmp != 0) {
188                    return cmp;
189                }
190                return lhs.mLabel.compareTo(rhs.mLabel);
191            }
192        });
193
194        for (TestResult t : map.values()) {
195            set.add(t);
196        }
197
198        for (TestResult t : set.descendingSet()) {
199            if (Math.abs(t.getAverage()) > ERROR_DISPLAY_THRESHOLD) {
200                Log.d(LOG_TAG, String.format("%2.4f : %s", t.getAverage(), t.mLabel));
201            }
202        }
203        Log.d(LOG_TAG, "");
204    }
205
206    private void finishTest() {
207        for (FinalCallback c : mFinalCallbacks) {
208            c.report("averageError", (mTotalError / mTotalTests));
209            for (int i = 1; i < ERROR_CUTOFFS.length; i++) {
210                c.report(String.format("tests with error over %1.3f", ERROR_CUTOFFS[i]),
211                        mErrorRates[i]);
212            }
213            if (mInputJson != null) {
214                c.report("tests regressed", mTestsRegressed);
215                c.report("tests improved", mTestsImproved);
216            }
217            c.complete();
218        }
219
220        try {
221            if (mOutputJson != null) {
222                String outputString = mOutputJson.toString(4);
223                File outputFile = new File(Environment.getExternalStorageDirectory(),
224                        "CanvasCompareOutput.json");
225                FileOutputStream outputStream = new FileOutputStream(outputFile);
226                outputStream.write(outputString.getBytes());
227                outputStream.close();
228                Log.d(LOG_TAG, "Saved output file with " + mOutputJson.length() + " entries");
229            }
230        } catch (JSONException e) {
231            Log.e(LOG_TAG, "error during JSON stringify", e);
232        } catch (IOException e) {
233            Log.e(LOG_TAG, "error storing JSON output on sd", e);
234        }
235
236        logTestResultHash("Modifier change vs previous", mModifierDiffResults);
237        logTestResultHash("Invidual test change vs previous", mIndividualDiffResults);
238        logTestResultHash("Modifier average test results", mModifierResults);
239        logTestResultHash("Individual test results", mIndividualResults);
240
241        Toast.makeText(getApplicationContext(), "done!", Toast.LENGTH_SHORT).show();
242        finish();
243    }
244
245    /**
246     * Inserts the error value into all TestResult objects, associated with each of its modifiers
247     */
248    private static void addForAllModifiers(String fullName, float error, String[] modifierNames,
249            HashMap<String, TestResult> modifierResults) {
250        for (String modifierName : modifierNames) {
251            TestResult r = modifierResults.get(fullName);
252            if (r == null) {
253                modifierResults.put(modifierName, new TestResult(modifierName, error));
254            } else {
255                r.addInto(error);
256            }
257        }
258    }
259
260    private void handleError(final String[] modifierNames, final float error) {
261        String fullName = "";
262        for (String s : modifierNames) {
263            fullName = fullName.concat("." + s);
264        }
265        fullName = fullName.substring(1);
266
267        float deltaError = 0;
268        if (mInputJson != null) {
269            try {
270                deltaError = error - (float)mInputJson.getDouble(fullName);
271            } catch (JSONException e) {
272                Log.w(LOG_TAG, "Warning: unable to read from input json", e);
273            }
274            if (deltaError > ERROR_CHANGE_THRESHOLD) mTestsRegressed++;
275            if (deltaError < -ERROR_CHANGE_THRESHOLD) mTestsImproved++;
276            mIndividualDiffResults.put(fullName, new TestResult(fullName, deltaError));
277            addForAllModifiers(fullName, deltaError, modifierNames, mModifierDiffResults);
278        }
279
280        mIndividualResults.put(fullName, new TestResult(fullName, error));
281        addForAllModifiers(fullName, error, modifierNames, mModifierResults);
282
283        try {
284            if (mOutputJson != null) {
285                mOutputJson.put(fullName, error);
286            }
287        } catch (JSONException e) {
288            Log.e(LOG_TAG, "exception during JSON recording", e);
289            mOutputJson = null;
290        }
291
292        for (int i = 0; i < ERROR_CUTOFFS.length; i++) {
293            if (error <= ERROR_CUTOFFS[i]) break;
294            mErrorRates[i]++;
295        }
296        mTotalError += error;
297        mTotalTests++;
298    }
299
300    @Override
301    protected boolean forceRecreateBitmaps() {
302        // disable, unless needed for drawing into imageviews
303        return DRAW_BITMAPS;
304    }
305
306    // FOR TESTING
307    public void setFinalCallback(FinalCallback c) {
308        mFinalCallbacks.add(c);
309    }
310}
311