1c3683b552f592d8039a466c663f7de8c8286e975Chris Craik/* 2c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * Copyright (C) 2012 The Android Open Source Project 3c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * 4c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * Licensed under the Apache License, Version 2.0 (the "License"); 5c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * you may not use this file except in compliance with the License. 6c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * You may obtain a copy of the License at 7c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * 8c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * http://www.apache.org/licenses/LICENSE-2.0 9c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * 10c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * Unless required by applicable law or agreed to in writing, software 11c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * distributed under the License is distributed on an "AS IS" BASIS, 12c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * See the License for the specific language governing permissions and 14c3683b552f592d8039a466c663f7de8c8286e975Chris Craik * limitations under the License. 15c3683b552f592d8039a466c663f7de8c8286e975Chris Craik */ 16c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 17c3683b552f592d8039a466c663f7de8c8286e975Chris Craikpackage com.android.test.hwuicompare; 18c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 195f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craikimport java.io.File; 205f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craikimport java.io.FileInputStream; 215f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craikimport java.io.FileOutputStream; 225f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craikimport java.io.IOException; 23c3683b552f592d8039a466c663f7de8c8286e975Chris Craikimport java.util.ArrayList; 245f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craikimport java.util.Comparator; 255f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craikimport java.util.HashMap; 265f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craikimport java.util.TreeSet; 27c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 285f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craikimport org.json.JSONException; 295f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craikimport org.json.JSONObject; 30c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 31c3683b552f592d8039a466c663f7de8c8286e975Chris Craikimport android.os.Bundle; 325f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craikimport android.os.Environment; 33c3683b552f592d8039a466c663f7de8c8286e975Chris Craikimport android.os.Trace; 34c3683b552f592d8039a466c663f7de8c8286e975Chris Craikimport android.util.Log; 35c3683b552f592d8039a466c663f7de8c8286e975Chris Craikimport android.widget.ImageView; 36c3683b552f592d8039a466c663f7de8c8286e975Chris Craikimport android.widget.Toast; 37c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 38c3683b552f592d8039a466c663f7de8c8286e975Chris Craikpublic class AutomaticActivity extends CompareActivity { 39c3683b552f592d8039a466c663f7de8c8286e975Chris Craik private static final String LOG_TAG = "AutomaticActivity"; 40c3683b552f592d8039a466c663f7de8c8286e975Chris Craik private static final float ERROR_DISPLAY_THRESHOLD = 0.01f; 41c3683b552f592d8039a466c663f7de8c8286e975Chris Craik protected static final boolean DRAW_BITMAPS = false; 42c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 435f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik /** 445f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik * Threshold of error change required to consider a test regressed/improved 455f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik */ 465f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private static final float ERROR_CHANGE_THRESHOLD = 0.001f; 47c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 485f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private static final float[] ERROR_CUTOFFS = { 495f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 0, 0.005f, 0.01f, 0.02f, 0.05f, 0.1f, 0.25f, 0.5f, 1f, 2f 505f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik }; 515f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 525f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private final float[] mErrorRates = new float[ERROR_CUTOFFS.length]; 53c3683b552f592d8039a466c663f7de8c8286e975Chris Craik private float mTotalTests = 0; 54c3683b552f592d8039a466c663f7de8c8286e975Chris Craik private float mTotalError = 0; 555f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private int mTestsRegressed = 0; 565f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private int mTestsImproved = 0; 57c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 585f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private ImageView mSoftwareImageView = null; 595f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private ImageView mHardwareImageView = null; 605f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 615f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 625f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik public abstract static class FinalCallback { 63c3683b552f592d8039a466c663f7de8c8286e975Chris Craik abstract void report(String name, float value); 645f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik void complete() {}; 65c3683b552f592d8039a466c663f7de8c8286e975Chris Craik } 66c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 675f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private final ArrayList<FinalCallback> mFinalCallbacks = new ArrayList<FinalCallback>(); 68c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 69c3683b552f592d8039a466c663f7de8c8286e975Chris Craik Runnable mRunnable = new Runnable() { 70c3683b552f592d8039a466c663f7de8c8286e975Chris Craik @Override 71c3683b552f592d8039a466c663f7de8c8286e975Chris Craik public void run() { 72c3683b552f592d8039a466c663f7de8c8286e975Chris Craik loadBitmaps(); 73c3683b552f592d8039a466c663f7de8c8286e975Chris Craik if (mSoftwareBitmap == null || mHardwareBitmap == null) { 74c3683b552f592d8039a466c663f7de8c8286e975Chris Craik Log.e(LOG_TAG, "bitmap is null"); 75c3683b552f592d8039a466c663f7de8c8286e975Chris Craik return; 76c3683b552f592d8039a466c663f7de8c8286e975Chris Craik } 77c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 78c3683b552f592d8039a466c663f7de8c8286e975Chris Craik if (DRAW_BITMAPS) { 79c3683b552f592d8039a466c663f7de8c8286e975Chris Craik mSoftwareImageView.setImageBitmap(mSoftwareBitmap); 80c3683b552f592d8039a466c663f7de8c8286e975Chris Craik mHardwareImageView.setImageBitmap(mHardwareBitmap); 81c3683b552f592d8039a466c663f7de8c8286e975Chris Craik } 82c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 83c3683b552f592d8039a466c663f7de8c8286e975Chris Craik Trace.traceBegin(Trace.TRACE_TAG_ALWAYS, "calculateError"); 84c3683b552f592d8039a466c663f7de8c8286e975Chris Craik float error = mErrorCalculator.calcErrorRS(mSoftwareBitmap, mHardwareBitmap); 85c3683b552f592d8039a466c663f7de8c8286e975Chris Craik Trace.traceEnd(Trace.TRACE_TAG_ALWAYS); 86c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 875f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik final String[] modifierNames = DisplayModifier.getLastAppliedModifications(); 885f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik handleError(modifierNames, error); 89c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 90c3683b552f592d8039a466c663f7de8c8286e975Chris Craik if (DisplayModifier.step()) { 915f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik finishTest(); 92c3683b552f592d8039a466c663f7de8c8286e975Chris Craik } else { 93c3683b552f592d8039a466c663f7de8c8286e975Chris Craik mHardwareView.invalidate(); 94c3683b552f592d8039a466c663f7de8c8286e975Chris Craik if (DRAW_BITMAPS) { 95c3683b552f592d8039a466c663f7de8c8286e975Chris Craik mSoftwareImageView.invalidate(); 96c3683b552f592d8039a466c663f7de8c8286e975Chris Craik mHardwareImageView.invalidate(); 97c3683b552f592d8039a466c663f7de8c8286e975Chris Craik } 98c3683b552f592d8039a466c663f7de8c8286e975Chris Craik } 99c3683b552f592d8039a466c663f7de8c8286e975Chris Craik mHandler.removeCallbacks(mRunnable); 100c3683b552f592d8039a466c663f7de8c8286e975Chris Craik } 101c3683b552f592d8039a466c663f7de8c8286e975Chris Craik }; 102c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 103c3683b552f592d8039a466c663f7de8c8286e975Chris Craik @Override 104c3683b552f592d8039a466c663f7de8c8286e975Chris Craik protected void onPause() { 105c3683b552f592d8039a466c663f7de8c8286e975Chris Craik super.onPause(); 106c3683b552f592d8039a466c663f7de8c8286e975Chris Craik mHandler.removeCallbacks(mRunnable); 107c3683b552f592d8039a466c663f7de8c8286e975Chris Craik }; 108c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 109c3683b552f592d8039a466c663f7de8c8286e975Chris Craik @Override 110c3683b552f592d8039a466c663f7de8c8286e975Chris Craik protected void onCreate(Bundle savedInstanceState) { 111c3683b552f592d8039a466c663f7de8c8286e975Chris Craik super.onCreate(savedInstanceState); 112c3683b552f592d8039a466c663f7de8c8286e975Chris Craik setContentView(R.layout.automatic_layout); 113c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 114c3683b552f592d8039a466c663f7de8c8286e975Chris Craik mSoftwareImageView = (ImageView) findViewById(R.id.software_image_view); 115c3683b552f592d8039a466c663f7de8c8286e975Chris Craik mHardwareImageView = (ImageView) findViewById(R.id.hardware_image_view); 116c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 117c3683b552f592d8039a466c663f7de8c8286e975Chris Craik onCreateCommon(mRunnable); 1185f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik beginTest(); 1195f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 1205f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 1215f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private static class TestResult { 1225f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik TestResult(String label, float error) { 1235f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mLabel = label; 1245f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mTotalError = error; 1255f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mCount = 1; 1265f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 1275f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik public void addInto(float error) { 1285f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mTotalError += error; 1295f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mCount++; 1305f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 1315f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik public float getAverage() { 1325f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik return mTotalError / mCount; 1335f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 1345f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik final String mLabel; 1355f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik float mTotalError; 1365f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik int mCount; 1375f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 1385f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 1395f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik JSONObject mOutputJson = null; 1405f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik JSONObject mInputJson = null; 1415f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik final HashMap<String, TestResult> mModifierResults = new HashMap<String, TestResult>(); 1425f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik final HashMap<String, TestResult> mIndividualResults = new HashMap<String, TestResult>(); 1435f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik final HashMap<String, TestResult> mModifierDiffResults = new HashMap<String, TestResult>(); 1445f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik final HashMap<String, TestResult> mIndividualDiffResults = new HashMap<String, TestResult>(); 1455f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private void beginTest() { 1465f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mFinalCallbacks.add(new FinalCallback() { 1475f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik @Override 148c3683b552f592d8039a466c663f7de8c8286e975Chris Craik void report(String name, float value) { 149c3683b552f592d8039a466c663f7de8c8286e975Chris Craik Log.d(LOG_TAG, name + " " + value); 150c3683b552f592d8039a466c663f7de8c8286e975Chris Craik }; 151c3683b552f592d8039a466c663f7de8c8286e975Chris Craik }); 1525f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 1535f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik File inputFile = new File(Environment.getExternalStorageDirectory(), 1545f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik "CanvasCompareInput.json"); 1555f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik if (inputFile.exists() && inputFile.canRead() && inputFile.length() > 0) { 1565f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik try { 1575f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik FileInputStream inputStream = new FileInputStream(inputFile); 1585f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.d(LOG_TAG, "Parsing input file..."); 1595f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik StringBuffer content = new StringBuffer((int)inputFile.length()); 1605f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik byte[] buffer = new byte[1024]; 1615f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik while (inputStream.read(buffer) != -1) { 1625f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik content.append(new String(buffer)); 1635f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 1645f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mInputJson = new JSONObject(content.toString()); 1655f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik inputStream.close(); 1665f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.d(LOG_TAG, "Parsed input file with " + mInputJson.length() + " entries"); 1675f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } catch (JSONException e) { 1685f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.e(LOG_TAG, "error parsing input json", e); 1695f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } catch (IOException e) { 1705f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.e(LOG_TAG, "error reading input json from sd", e); 1715f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 1725f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 1735f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 1745f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mOutputJson = new JSONObject(); 1755f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 1765f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 1775f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private static void logTestResultHash(String label, HashMap<String, TestResult> map) { 1785f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.d(LOG_TAG, "---------------"); 1795f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.d(LOG_TAG, label + ":"); 1805f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.d(LOG_TAG, "---------------"); 1815f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik TreeSet<TestResult> set = new TreeSet<TestResult>(new Comparator<TestResult>() { 1825f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik @Override 1835f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik public int compare(TestResult lhs, TestResult rhs) { 1845f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik if (lhs == rhs) return 0; // don't need to worry about complex equality 1855f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 1865f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik int cmp = Float.compare(lhs.getAverage(), rhs.getAverage()); 1875f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik if (cmp != 0) { 1885f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik return cmp; 1895f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 1905f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik return lhs.mLabel.compareTo(rhs.mLabel); 1915f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 1925f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik }); 1935f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 1945f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik for (TestResult t : map.values()) { 1955f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik set.add(t); 1965f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 1975f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 1985f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik for (TestResult t : set.descendingSet()) { 1995f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik if (Math.abs(t.getAverage()) > ERROR_DISPLAY_THRESHOLD) { 2005f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.d(LOG_TAG, String.format("%2.4f : %s", t.getAverage(), t.mLabel)); 2015f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2025f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2035f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.d(LOG_TAG, ""); 2045f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2055f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 2065f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private void finishTest() { 2075f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik for (FinalCallback c : mFinalCallbacks) { 2085f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik c.report("averageError", (mTotalError / mTotalTests)); 2095f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik for (int i = 1; i < ERROR_CUTOFFS.length; i++) { 2105f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik c.report(String.format("tests with error over %1.3f", ERROR_CUTOFFS[i]), 2115f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mErrorRates[i]); 2125f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2135f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik if (mInputJson != null) { 2145f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik c.report("tests regressed", mTestsRegressed); 2155f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik c.report("tests improved", mTestsImproved); 2165f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2175f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik c.complete(); 2185f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2195f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 2205f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik try { 2215f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik if (mOutputJson != null) { 2225f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik String outputString = mOutputJson.toString(4); 2235f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik File outputFile = new File(Environment.getExternalStorageDirectory(), 2245f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik "CanvasCompareOutput.json"); 2255f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik FileOutputStream outputStream = new FileOutputStream(outputFile); 2265f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik outputStream.write(outputString.getBytes()); 2275f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik outputStream.close(); 2285f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.d(LOG_TAG, "Saved output file with " + mOutputJson.length() + " entries"); 2295f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2305f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } catch (JSONException e) { 2315f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.e(LOG_TAG, "error during JSON stringify", e); 2325f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } catch (IOException e) { 2335f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.e(LOG_TAG, "error storing JSON output on sd", e); 2345f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2355f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 2365f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik logTestResultHash("Modifier change vs previous", mModifierDiffResults); 2375f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik logTestResultHash("Invidual test change vs previous", mIndividualDiffResults); 2385f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik logTestResultHash("Modifier average test results", mModifierResults); 2395f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik logTestResultHash("Individual test results", mIndividualResults); 2405f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 2415f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Toast.makeText(getApplicationContext(), "done!", Toast.LENGTH_SHORT).show(); 2425f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik finish(); 2435f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2445f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 2455f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik /** 2465f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik * Inserts the error value into all TestResult objects, associated with each of its modifiers 2475f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik */ 2485f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private static void addForAllModifiers(String fullName, float error, String[] modifierNames, 2495f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik HashMap<String, TestResult> modifierResults) { 2505f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik for (String modifierName : modifierNames) { 2515f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik TestResult r = modifierResults.get(fullName); 2525f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik if (r == null) { 2535f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik modifierResults.put(modifierName, new TestResult(modifierName, error)); 2545f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } else { 2555f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik r.addInto(error); 2565f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2575f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2585f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2595f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 2605f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik private void handleError(final String[] modifierNames, final float error) { 2615f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik String fullName = ""; 2625f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik for (String s : modifierNames) { 2635f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik fullName = fullName.concat("." + s); 2645f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2655f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik fullName = fullName.substring(1); 2665f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 2675f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik float deltaError = 0; 2685f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik if (mInputJson != null) { 2695f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik try { 2705f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik deltaError = error - (float)mInputJson.getDouble(fullName); 2715f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } catch (JSONException e) { 2725f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.w(LOG_TAG, "Warning: unable to read from input json", e); 2735f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2745f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik if (deltaError > ERROR_CHANGE_THRESHOLD) mTestsRegressed++; 2755f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik if (deltaError < -ERROR_CHANGE_THRESHOLD) mTestsImproved++; 2765f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mIndividualDiffResults.put(fullName, new TestResult(fullName, deltaError)); 2775f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik addForAllModifiers(fullName, deltaError, modifierNames, mModifierDiffResults); 2785f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2795f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 2805f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mIndividualResults.put(fullName, new TestResult(fullName, error)); 2815f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik addForAllModifiers(fullName, error, modifierNames, mModifierResults); 2825f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 2835f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik try { 2845f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik if (mOutputJson != null) { 2855f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mOutputJson.put(fullName, error); 2865f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2875f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } catch (JSONException e) { 2885f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik Log.e(LOG_TAG, "exception during JSON recording", e); 2895f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mOutputJson = null; 2905f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2915f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik 2925f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik for (int i = 0; i < ERROR_CUTOFFS.length; i++) { 2935f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik if (error <= ERROR_CUTOFFS[i]) break; 2945f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mErrorRates[i]++; 2955f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik } 2965f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mTotalError += error; 2975f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mTotalTests++; 298c3683b552f592d8039a466c663f7de8c8286e975Chris Craik } 299c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 300c3683b552f592d8039a466c663f7de8c8286e975Chris Craik @Override 301c3683b552f592d8039a466c663f7de8c8286e975Chris Craik protected boolean forceRecreateBitmaps() { 302c3683b552f592d8039a466c663f7de8c8286e975Chris Craik // disable, unless needed for drawing into imageviews 303c3683b552f592d8039a466c663f7de8c8286e975Chris Craik return DRAW_BITMAPS; 304c3683b552f592d8039a466c663f7de8c8286e975Chris Craik } 305c3683b552f592d8039a466c663f7de8c8286e975Chris Craik 306c3683b552f592d8039a466c663f7de8c8286e975Chris Craik // FOR TESTING 3075f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik public void setFinalCallback(FinalCallback c) { 3085f7831158439f92f33c987d5d29dc9546bfe7c79Chris Craik mFinalCallbacks.add(c); 309c3683b552f592d8039a466c663f7de8c8286e975Chris Craik } 310c3683b552f592d8039a466c663f7de8c8286e975Chris Craik} 311