1/*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package android.perftests.utils;
18
19import android.app.Activity;
20import android.app.Instrumentation;
21import android.os.Bundle;
22import android.os.Debug;
23import android.support.test.InstrumentationRegistry;
24import android.util.Log;
25
26import java.io.File;
27import java.util.ArrayList;
28import java.util.concurrent.TimeUnit;
29
30/**
31 * Provides a benchmark framework.
32 *
33 * Example usage:
34 * // Executes the code while keepRunning returning true.
35 *
36 * public void sampleMethod() {
37 *     BenchmarkState state = new BenchmarkState();
38 *
39 *     int[] src = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
40 *     while (state.keepRunning()) {
41 *         int[] dest = new int[src.length];
42 *         System.arraycopy(src, 0, dest, 0, src.length);
43 *     }
44 *     System.out.println(state.summaryLine());
45 * }
46 */
47public final class BenchmarkState {
48
49    private static final String TAG = "BenchmarkState";
50    private static final boolean ENABLE_PROFILING = false;
51
52    private static final int NOT_STARTED = 0;  // The benchmark has not started yet.
53    private static final int WARMUP = 1; // The benchmark is warming up.
54    private static final int RUNNING = 2;  // The benchmark is running.
55    private static final int FINISHED = 3;  // The benchmark has stopped.
56
57    private int mState = NOT_STARTED;  // Current benchmark state.
58
59    private static final long WARMUP_DURATION_NS = ms2ns(250); // warm-up for at least 250ms
60    private static final int WARMUP_MIN_ITERATIONS = 16; // minimum iterations to warm-up for
61
62    // TODO: Tune these values.
63    private static final long TARGET_TEST_DURATION_NS = ms2ns(500); // target testing for 500 ms
64    private static final int MAX_TEST_ITERATIONS = 1000000;
65    private static final int MIN_TEST_ITERATIONS = 10;
66    private static final int REPEAT_COUNT = 5;
67
68    private long mStartTimeNs = 0;  // Previously captured System.nanoTime().
69    private boolean mPaused;
70    private long mPausedTimeNs = 0; // The System.nanoTime() when the pauseTiming() is called.
71    private long mPausedDurationNs = 0;  // The duration of paused state in nano sec.
72
73    private int mIteration = 0;
74    private int mMaxIterations = 0;
75
76    private int mRepeatCount = 0;
77
78    // Statistics. These values will be filled when the benchmark has finished.
79    // The computation needs double precision, but long int is fine for final reporting.
80    private Stats mStats;
81
82    // Individual duration in nano seconds.
83    private ArrayList<Long> mResults = new ArrayList<>();
84
85    private static final long ms2ns(long ms) {
86        return TimeUnit.MILLISECONDS.toNanos(ms);
87    }
88
89    // Stops the benchmark timer.
90    // This method can be called only when the timer is running.
91    public void pauseTiming() {
92        if (mPaused) {
93            throw new IllegalStateException(
94                    "Unable to pause the benchmark. The benchmark has already paused.");
95        }
96        mPausedTimeNs = System.nanoTime();
97        mPaused = true;
98    }
99
100    // Starts the benchmark timer.
101    // This method can be called only when the timer is stopped.
102    public void resumeTiming() {
103        if (!mPaused) {
104            throw new IllegalStateException(
105                    "Unable to resume the benchmark. The benchmark is already running.");
106        }
107        mPausedDurationNs += System.nanoTime() - mPausedTimeNs;
108        mPausedTimeNs = 0;
109        mPaused = false;
110    }
111
112    private void beginWarmup() {
113        mStartTimeNs = System.nanoTime();
114        mIteration = 0;
115        mState = WARMUP;
116    }
117
118    private void beginBenchmark(long warmupDuration, int iterations) {
119        if (ENABLE_PROFILING) {
120            File f = new File(InstrumentationRegistry.getContext().getDataDir(), "benchprof");
121            Log.d(TAG, "Tracing to: " + f.getAbsolutePath());
122            Debug.startMethodTracingSampling(f.getAbsolutePath(), 16 * 1024 * 1024, 100);
123        }
124        mMaxIterations = (int) (TARGET_TEST_DURATION_NS / (warmupDuration / iterations));
125        mMaxIterations = Math.min(MAX_TEST_ITERATIONS,
126                Math.max(mMaxIterations, MIN_TEST_ITERATIONS));
127        mPausedDurationNs = 0;
128        mIteration = 0;
129        mRepeatCount = 0;
130        mState = RUNNING;
131        mStartTimeNs = System.nanoTime();
132    }
133
134    private boolean startNextTestRun() {
135        final long currentTime = System.nanoTime();
136        mResults.add((currentTime - mStartTimeNs - mPausedDurationNs) / mMaxIterations);
137        mRepeatCount++;
138        if (mRepeatCount >= REPEAT_COUNT) {
139            if (ENABLE_PROFILING) {
140                Debug.stopMethodTracing();
141            }
142            mStats = new Stats(mResults);
143            mState = FINISHED;
144            return false;
145        }
146        mPausedDurationNs = 0;
147        mIteration = 0;
148        mStartTimeNs = System.nanoTime();
149        return true;
150    }
151
152    /**
153     * Judges whether the benchmark needs more samples.
154     *
155     * For the usage, see class comment.
156     */
157    public boolean keepRunning() {
158        switch (mState) {
159            case NOT_STARTED:
160                beginWarmup();
161                return true;
162            case WARMUP:
163                mIteration++;
164                // Only check nanoTime on every iteration in WARMUP since we
165                // don't yet have a target iteration count.
166                final long duration = System.nanoTime() - mStartTimeNs;
167                if (mIteration >= WARMUP_MIN_ITERATIONS && duration >= WARMUP_DURATION_NS) {
168                    beginBenchmark(duration, mIteration);
169                }
170                return true;
171            case RUNNING:
172                mIteration++;
173                if (mIteration >= mMaxIterations) {
174                    return startNextTestRun();
175                }
176                if (mPaused) {
177                    throw new IllegalStateException(
178                            "Benchmark step finished with paused state. " +
179                            "Resume the benchmark before finishing each step.");
180                }
181                return true;
182            case FINISHED:
183                throw new IllegalStateException("The benchmark has finished.");
184            default:
185                throw new IllegalStateException("The benchmark is in unknown state.");
186        }
187    }
188
189    private long mean() {
190        if (mState != FINISHED) {
191            throw new IllegalStateException("The benchmark hasn't finished");
192        }
193        return (long) mStats.getMean();
194    }
195
196    private long median() {
197        if (mState != FINISHED) {
198            throw new IllegalStateException("The benchmark hasn't finished");
199        }
200        return mStats.getMedian();
201    }
202
203    private long min() {
204        if (mState != FINISHED) {
205            throw new IllegalStateException("The benchmark hasn't finished");
206        }
207        return mStats.getMin();
208    }
209
210    private long standardDeviation() {
211        if (mState != FINISHED) {
212            throw new IllegalStateException("The benchmark hasn't finished");
213        }
214        return (long) mStats.getStandardDeviation();
215    }
216
217    private String summaryLine() {
218        StringBuilder sb = new StringBuilder();
219        sb.append("Summary: ");
220        sb.append("median=").append(median()).append("ns, ");
221        sb.append("mean=").append(mean()).append("ns, ");
222        sb.append("min=").append(min()).append("ns, ");
223        sb.append("sigma=").append(standardDeviation()).append(", ");
224        sb.append("iteration=").append(mResults.size()).append(", ");
225        // print out the first few iterations' number for double checking.
226        int sampleNumber = Math.min(mResults.size(), 16);
227        for (int i = 0; i < sampleNumber; i++) {
228            sb.append("No ").append(i).append(" result is ").append(mResults.get(i)).append(", ");
229        }
230        return sb.toString();
231    }
232
233    public void sendFullStatusReport(Instrumentation instrumentation, String key) {
234        Log.i(TAG, key + summaryLine());
235        Bundle status = new Bundle();
236        status.putLong(key + "_median", median());
237        status.putLong(key + "_mean", mean());
238        status.putLong(key + "_min", min());
239        status.putLong(key + "_standardDeviation", standardDeviation());
240        instrumentation.sendStatus(Activity.RESULT_OK, status);
241    }
242}
243