1263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks/* 2263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * Copyright (C) 2018 The Android Open Source Project 3263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * 4263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * Licensed under the Apache License, Version 2.0 (the "License"); 5263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * you may not use this file except in compliance with the License. 6263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * You may obtain a copy of the License at 7263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * 8263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * http://www.apache.org/licenses/LICENSE-2.0 9263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * 10263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * Unless required by applicable law or agreed to in writing, software 11263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * distributed under the License is distributed on an "AS IS" BASIS, 12263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * See the License for the specific language governing permissions and 14263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * limitations under the License. 15263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks */ 16263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 17263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubankspackage android.perftests.utils; 18263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 19263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanksimport android.app.Activity; 20263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanksimport android.app.Instrumentation; 21263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanksimport android.os.Bundle; 22263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanksimport android.util.Log; 23263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 24263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanksimport java.util.ArrayList; 25263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanksimport java.util.concurrent.TimeUnit; 26263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 27263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks/** 28263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * Provides a benchmark framework. 29263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * 30263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * This differs from BenchmarkState in that rather than the class measuring the the elapsed time, 31263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * the test passes in the elapsed time. 32263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * 33263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * Example usage: 34263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * 35263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * public void sampleMethod() { 36263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * ManualBenchmarkState state = new ManualBenchmarkState(); 37263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * 38263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * int[] src = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; 39263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * long elapsedTime = 0; 40263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * while (state.keepRunning(elapsedTime)) { 41263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * long startTime = System.nanoTime(); 42263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * int[] dest = new int[src.length]; 43263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * System.arraycopy(src, 0, dest, 0, src.length); 44263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * elapsedTime = System.nanoTime() - startTime; 45263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * } 46263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * System.out.println(state.summaryLine()); 47263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * } 48263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * 49263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * Or use the PerfManualStatusReporter TestRule. 50263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * 51263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * Make sure that the overhead of checking the clock does not noticeably affect the results. 52263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks */ 53263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubankspublic final class ManualBenchmarkState { 54263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private static final String TAG = ManualBenchmarkState.class.getSimpleName(); 55263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 56263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks // TODO: Tune these values. 57263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks // warm-up for duration 58263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private static final long WARMUP_DURATION_NS = TimeUnit.SECONDS.toNanos(5); 59263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks // minimum iterations to warm-up for 60263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private static final int WARMUP_MIN_ITERATIONS = 8; 61263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 62263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks // target testing for duration 63263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private static final long TARGET_TEST_DURATION_NS = TimeUnit.SECONDS.toNanos(16); 64263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private static final int MAX_TEST_ITERATIONS = 1000000; 65263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private static final int MIN_TEST_ITERATIONS = 10; 66263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 67263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private static final int NOT_STARTED = 0; // The benchmark has not started yet. 68263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private static final int WARMUP = 1; // The benchmark is warming up. 69263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private static final int RUNNING = 2; // The benchmark is running. 70263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private static final int FINISHED = 3; // The benchmark has stopped. 71263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 72263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private int mState = NOT_STARTED; // Current benchmark state. 73263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 74263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private long mWarmupStartTime = 0; 75263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private int mWarmupIterations = 0; 76263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 77263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private int mMaxIterations = 0; 78263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 79263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks // Individual duration in nano seconds. 80263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private ArrayList<Long> mResults = new ArrayList<>(); 81263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 82263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks // Statistics. These values will be filled when the benchmark has finished. 83263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks // The computation needs double precision, but long int is fine for final reporting. 84263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private Stats mStats; 85263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 86263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private void beginBenchmark(long warmupDuration, int iterations) { 87263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks mMaxIterations = (int) (TARGET_TEST_DURATION_NS / (warmupDuration / iterations)); 88263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks mMaxIterations = Math.min(MAX_TEST_ITERATIONS, 89263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks Math.max(mMaxIterations, MIN_TEST_ITERATIONS)); 90263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks mState = RUNNING; 91263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks } 92263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 93263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks /** 94263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * Judges whether the benchmark needs more samples. 95263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * 96263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks * For the usage, see class comment. 97263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks */ 98263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks public boolean keepRunning(long duration) { 99263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks if (duration < 0) { 100263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks throw new RuntimeException("duration is negative: " + duration); 101263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks } 102263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks switch (mState) { 103263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks case NOT_STARTED: 104263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks mState = WARMUP; 105263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks mWarmupStartTime = System.nanoTime(); 106263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks return true; 107263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks case WARMUP: { 108263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks final long timeSinceStartingWarmup = System.nanoTime() - mWarmupStartTime; 109263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks ++mWarmupIterations; 110263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks if (mWarmupIterations >= WARMUP_MIN_ITERATIONS 111263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks && timeSinceStartingWarmup >= WARMUP_DURATION_NS) { 112263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks beginBenchmark(timeSinceStartingWarmup, mWarmupIterations); 113263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks } 114263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks return true; 115263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks } 116263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks case RUNNING: { 117263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks mResults.add(duration); 118263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks final boolean keepRunning = mResults.size() < mMaxIterations; 119263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks if (!keepRunning) { 120263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks mStats = new Stats(mResults); 121263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks mState = FINISHED; 122263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks } 123263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks return keepRunning; 124263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks } 125263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks case FINISHED: 126263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks throw new IllegalStateException("The benchmark has finished."); 127263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks default: 128263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks throw new IllegalStateException("The benchmark is in an unknown state."); 129263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks } 130263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks } 131263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 132263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks private String summaryLine() { 133263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks final StringBuilder sb = new StringBuilder(); 134263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks sb.append("Summary: "); 135263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks sb.append("median=").append(mStats.getMedian()).append("ns, "); 136263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks sb.append("mean=").append(mStats.getMean()).append("ns, "); 137263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks sb.append("min=").append(mStats.getMin()).append("ns, "); 138263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks sb.append("max=").append(mStats.getMax()).append("ns, "); 139263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks sb.append("sigma=").append(mStats.getStandardDeviation()).append(", "); 140263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks sb.append("iteration=").append(mResults.size()).append(", "); 141263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks sb.append("values=").append(mResults.toString()); 142263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks return sb.toString(); 143263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks } 144263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 145263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks public void sendFullStatusReport(Instrumentation instrumentation, String key) { 146263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks if (mState != FINISHED) { 147263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks throw new IllegalStateException("The benchmark hasn't finished"); 148263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks } 149263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks Log.i(TAG, key + summaryLine()); 150263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks final Bundle status = new Bundle(); 151263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks status.putLong(key + "_median", mStats.getMedian()); 152263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks status.putLong(key + "_mean", (long) mStats.getMean()); 1533ffc6e7e0f3f473e9d9bf2d0e32d4948c809b8b8Arthur Eubanks status.putLong(key + "_percentile90", mStats.getPercentile90()); 1543ffc6e7e0f3f473e9d9bf2d0e32d4948c809b8b8Arthur Eubanks status.putLong(key + "_percentile95", mStats.getPercentile95()); 155263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks status.putLong(key + "_stddev", (long) mStats.getStandardDeviation()); 156263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks instrumentation.sendStatus(Activity.RESULT_OK, status); 157263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks } 158263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks} 159263d674d598f77a1f91bdfc73be808efd3446133Arthur Eubanks 160