1/*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package android.perftests.utils;
18
19import android.app.Activity;
20import android.app.Instrumentation;
21import android.os.Bundle;
22import android.util.Log;
23
24import java.util.ArrayList;
25import java.util.concurrent.TimeUnit;
26
27/**
28 * Provides a benchmark framework.
29 *
30 * This differs from BenchmarkState in that rather than the class measuring the the elapsed time,
31 * the test passes in the elapsed time.
32 *
33 * Example usage:
34 *
35 * public void sampleMethod() {
36 *     ManualBenchmarkState state = new ManualBenchmarkState();
37 *
38 *     int[] src = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
39 *     long elapsedTime = 0;
40 *     while (state.keepRunning(elapsedTime)) {
41 *         long startTime = System.nanoTime();
42 *         int[] dest = new int[src.length];
43 *         System.arraycopy(src, 0, dest, 0, src.length);
44 *         elapsedTime = System.nanoTime() - startTime;
45 *     }
46 *     System.out.println(state.summaryLine());
47 * }
48 *
49 * Or use the PerfManualStatusReporter TestRule.
50 *
51 * Make sure that the overhead of checking the clock does not noticeably affect the results.
52 */
53public final class ManualBenchmarkState {
54    private static final String TAG = ManualBenchmarkState.class.getSimpleName();
55
56    // TODO: Tune these values.
57    // warm-up for duration
58    private static final long WARMUP_DURATION_NS = TimeUnit.SECONDS.toNanos(5);
59    // minimum iterations to warm-up for
60    private static final int WARMUP_MIN_ITERATIONS = 8;
61
62    // target testing for duration
63    private static final long TARGET_TEST_DURATION_NS = TimeUnit.SECONDS.toNanos(16);
64    private static final int MAX_TEST_ITERATIONS = 1000000;
65    private static final int MIN_TEST_ITERATIONS = 10;
66
67    private static final int NOT_STARTED = 0;  // The benchmark has not started yet.
68    private static final int WARMUP = 1; // The benchmark is warming up.
69    private static final int RUNNING = 2;  // The benchmark is running.
70    private static final int FINISHED = 3;  // The benchmark has stopped.
71
72    private int mState = NOT_STARTED;  // Current benchmark state.
73
74    private long mWarmupStartTime = 0;
75    private int mWarmupIterations = 0;
76
77    private int mMaxIterations = 0;
78
79    // Individual duration in nano seconds.
80    private ArrayList<Long> mResults = new ArrayList<>();
81
82    // Statistics. These values will be filled when the benchmark has finished.
83    // The computation needs double precision, but long int is fine for final reporting.
84    private Stats mStats;
85
86    private void beginBenchmark(long warmupDuration, int iterations) {
87        mMaxIterations = (int) (TARGET_TEST_DURATION_NS / (warmupDuration / iterations));
88        mMaxIterations = Math.min(MAX_TEST_ITERATIONS,
89                Math.max(mMaxIterations, MIN_TEST_ITERATIONS));
90        mState = RUNNING;
91    }
92
93    /**
94     * Judges whether the benchmark needs more samples.
95     *
96     * For the usage, see class comment.
97     */
98    public boolean keepRunning(long duration) {
99        if (duration < 0) {
100            throw new RuntimeException("duration is negative: " + duration);
101        }
102        switch (mState) {
103            case NOT_STARTED:
104                mState = WARMUP;
105                mWarmupStartTime = System.nanoTime();
106                return true;
107            case WARMUP: {
108                final long timeSinceStartingWarmup = System.nanoTime() - mWarmupStartTime;
109                ++mWarmupIterations;
110                if (mWarmupIterations >= WARMUP_MIN_ITERATIONS
111                        && timeSinceStartingWarmup >= WARMUP_DURATION_NS) {
112                    beginBenchmark(timeSinceStartingWarmup, mWarmupIterations);
113                }
114                return true;
115            }
116            case RUNNING: {
117                mResults.add(duration);
118                final boolean keepRunning = mResults.size() < mMaxIterations;
119                if (!keepRunning) {
120                    mStats = new Stats(mResults);
121                    mState = FINISHED;
122                }
123                return keepRunning;
124            }
125            case FINISHED:
126                throw new IllegalStateException("The benchmark has finished.");
127            default:
128                throw new IllegalStateException("The benchmark is in an unknown state.");
129        }
130    }
131
132    private String summaryLine() {
133        final StringBuilder sb = new StringBuilder();
134        sb.append("Summary: ");
135        sb.append("median=").append(mStats.getMedian()).append("ns, ");
136        sb.append("mean=").append(mStats.getMean()).append("ns, ");
137        sb.append("min=").append(mStats.getMin()).append("ns, ");
138        sb.append("max=").append(mStats.getMax()).append("ns, ");
139        sb.append("sigma=").append(mStats.getStandardDeviation()).append(", ");
140        sb.append("iteration=").append(mResults.size()).append(", ");
141        sb.append("values=").append(mResults.toString());
142        return sb.toString();
143    }
144
145    public void sendFullStatusReport(Instrumentation instrumentation, String key) {
146        if (mState != FINISHED) {
147            throw new IllegalStateException("The benchmark hasn't finished");
148        }
149        Log.i(TAG, key + summaryLine());
150        final Bundle status = new Bundle();
151        status.putLong(key + "_median", mStats.getMedian());
152        status.putLong(key + "_mean", (long) mStats.getMean());
153        status.putLong(key + "_percentile90", mStats.getPercentile90());
154        status.putLong(key + "_percentile95", mStats.getPercentile95());
155        status.putLong(key + "_stddev", (long) mStats.getStandardDeviation());
156        instrumentation.sendStatus(Activity.RESULT_OK, status);
157    }
158}
159
160