1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the License);
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an AS IS BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.benchmark.results;
18
19import android.content.ContentValues;
20import android.content.Context;
21import android.database.Cursor;
22import android.database.sqlite.SQLiteDatabase;
23import android.database.sqlite.SQLiteOpenHelper;
24import android.view.FrameMetrics;
25import android.widget.Toast;
26
27import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
28
29import java.io.FileWriter;
30import java.io.IOException;
31import java.text.DateFormat;
32import java.util.ArrayList;
33import java.util.Date;
34import java.util.HashMap;
35
36public class GlobalResultsStore extends SQLiteOpenHelper {
37    private static final int VERSION = 2;
38
39    private static GlobalResultsStore sInstance;
40    private static final String UI_RESULTS_TABLE = "ui_results";
41
42    private final Context mContext;
43
44    private GlobalResultsStore(Context context) {
45        super(context, "BenchmarkResults", null, VERSION);
46        mContext = context;
47    }
48
49    public static GlobalResultsStore getInstance(Context context) {
50        if (sInstance == null) {
51            sInstance = new GlobalResultsStore(context.getApplicationContext());
52        }
53
54        return sInstance;
55    }
56
57    @Override
58    public void onCreate(SQLiteDatabase sqLiteDatabase) {
59        sqLiteDatabase.execSQL("CREATE TABLE " + UI_RESULTS_TABLE + " (" +
60                " _id INTEGER PRIMARY KEY AUTOINCREMENT," +
61                " name TEXT," +
62                " run_id INTEGER," +
63                " iteration INTEGER," +
64                " timestamp TEXT,"  +
65                " unknown_delay REAL," +
66                " input REAL," +
67                " animation REAL," +
68                " layout REAL," +
69                " draw REAL," +
70                " sync REAL," +
71                " command_issue REAL," +
72                " swap_buffers REAL," +
73                " total_duration REAL," +
74                " jank_frame BOOLEAN, " +
75                " device_charging INTEGER);");
76    }
77
78    public void storeRunResults(String testName, int runId, int iteration,
79                                UiBenchmarkResult result) {
80        SQLiteDatabase db = getWritableDatabase();
81        db.beginTransaction();
82
83        try {
84            String date = DateFormat.getDateTimeInstance().format(new Date());
85            int jankIndexIndex = 0;
86            int[] sortedJankIndices = result.getSortedJankFrameIndices();
87            int totalFrameCount = result.getTotalFrameCount();
88            for (int frameIdx = 0; frameIdx < totalFrameCount; frameIdx++) {
89                ContentValues cv = new ContentValues();
90                cv.put("name", testName);
91                cv.put("run_id", runId);
92                cv.put("iteration", iteration);
93                cv.put("timestamp", date);
94                cv.put("unknown_delay",
95                        result.getMetricAtIndex(frameIdx, FrameMetrics.UNKNOWN_DELAY_DURATION));
96                cv.put("input",
97                        result.getMetricAtIndex(frameIdx, FrameMetrics.INPUT_HANDLING_DURATION));
98                cv.put("animation",
99                        result.getMetricAtIndex(frameIdx, FrameMetrics.ANIMATION_DURATION));
100                cv.put("layout",
101                        result.getMetricAtIndex(frameIdx, FrameMetrics.LAYOUT_MEASURE_DURATION));
102                cv.put("draw",
103                        result.getMetricAtIndex(frameIdx, FrameMetrics.DRAW_DURATION));
104                cv.put("sync",
105                        result.getMetricAtIndex(frameIdx, FrameMetrics.SYNC_DURATION));
106                cv.put("command_issue",
107                        result.getMetricAtIndex(frameIdx, FrameMetrics.COMMAND_ISSUE_DURATION));
108                cv.put("swap_buffers",
109                        result.getMetricAtIndex(frameIdx, FrameMetrics.SWAP_BUFFERS_DURATION));
110                cv.put("total_duration",
111                        result.getMetricAtIndex(frameIdx, FrameMetrics.TOTAL_DURATION));
112                if (jankIndexIndex < sortedJankIndices.length &&
113                        sortedJankIndices[jankIndexIndex] == frameIdx) {
114                    jankIndexIndex++;
115                    cv.put("jank_frame", true);
116                } else {
117                    cv.put("jank_frame", false);
118                }
119                db.insert(UI_RESULTS_TABLE, null, cv);
120            }
121            db.setTransactionSuccessful();
122            Toast.makeText(mContext, "Score: " + result.getScore()
123                    + " Jank: " + (100 * sortedJankIndices.length) / (float) totalFrameCount + "%",
124                    Toast.LENGTH_LONG).show();
125        } finally {
126            db.endTransaction();
127        }
128
129    }
130
131    public ArrayList<UiBenchmarkResult> loadTestResults(String testName, int runId) {
132        SQLiteDatabase db = getReadableDatabase();
133        ArrayList<UiBenchmarkResult> resultList = new ArrayList<>();
134        try {
135            String[] columnsToQuery = new String[] {
136                    "name",
137                    "run_id",
138                    "iteration",
139                    "unknown_delay",
140                    "input",
141                    "animation",
142                    "layout",
143                    "draw",
144                    "sync",
145                    "command_issue",
146                    "swap_buffers",
147                    "total_duration",
148            };
149
150            Cursor cursor = db.query(
151                    UI_RESULTS_TABLE, columnsToQuery, "run_id=? AND name=?",
152                    new String[] { Integer.toString(runId), testName }, null, null, "iteration");
153
154            double[] values = new double[columnsToQuery.length - 3];
155
156            while (cursor.moveToNext()) {
157                int iteration = cursor.getInt(cursor.getColumnIndexOrThrow("iteration"));
158
159                values[0] = cursor.getDouble(
160                        cursor.getColumnIndexOrThrow("unknown_delay"));
161                values[1] = cursor.getDouble(
162                        cursor.getColumnIndexOrThrow("input"));
163                values[2] = cursor.getDouble(
164                        cursor.getColumnIndexOrThrow("animation"));
165                values[3] = cursor.getDouble(
166                        cursor.getColumnIndexOrThrow("layout"));
167                values[4] = cursor.getDouble(
168                        cursor.getColumnIndexOrThrow("draw"));
169                values[5] = cursor.getDouble(
170                        cursor.getColumnIndexOrThrow("sync"));
171                values[6] = cursor.getDouble(
172                        cursor.getColumnIndexOrThrow("command_issue"));
173                values[7] = cursor.getDouble(
174                        cursor.getColumnIndexOrThrow("swap_buffers"));
175                values[8] = cursor.getDouble(
176                        cursor.getColumnIndexOrThrow("total_duration"));
177
178                UiBenchmarkResult iterationResult;
179                if (resultList.size() == iteration) {
180                    iterationResult = new UiBenchmarkResult(values);
181                    resultList.add(iteration, iterationResult);
182                } else {
183                    iterationResult = resultList.get(iteration);
184                    iterationResult.update(values);
185                }
186            }
187
188            cursor.close();
189        } finally {
190            db.close();
191        }
192
193        int total = resultList.get(0).getTotalFrameCount();
194        for (int i = 0; i < total; i++) {
195            System.out.println(""+ resultList.get(0).getMetricAtIndex(0, FrameMetrics.TOTAL_DURATION));
196        }
197
198        return resultList;
199    }
200
201    public HashMap<String, ArrayList<UiBenchmarkResult>> loadDetailedResults(int runId) {
202        SQLiteDatabase db = getReadableDatabase();
203        HashMap<String, ArrayList<UiBenchmarkResult>> results = new HashMap<>();
204        try {
205            String[] columnsToQuery = new String[] {
206                    "name",
207                    "run_id",
208                    "iteration",
209                    "unknown_delay",
210                    "input",
211                    "animation",
212                    "layout",
213                    "draw",
214                    "sync",
215                    "command_issue",
216                    "swap_buffers",
217                    "total_duration",
218            };
219
220            Cursor cursor = db.query(
221                    UI_RESULTS_TABLE, columnsToQuery, "run_id=?",
222                    new String[] { Integer.toString(runId) }, null, null, "name, iteration");
223
224            double[] values = new double[columnsToQuery.length - 3];
225            while (cursor.moveToNext()) {
226                int iteration = cursor.getInt(cursor.getColumnIndexOrThrow("iteration"));
227                String name = cursor.getString(cursor.getColumnIndexOrThrow("name"));
228                ArrayList<UiBenchmarkResult> resultList = results.get(name);
229                if (resultList == null) {
230                    resultList = new ArrayList<>();
231                    results.put(name, resultList);
232                }
233
234                values[0] = cursor.getDouble(
235                        cursor.getColumnIndexOrThrow("unknown_delay"));
236                values[1] = cursor.getDouble(
237                        cursor.getColumnIndexOrThrow("input"));
238                values[2] = cursor.getDouble(
239                        cursor.getColumnIndexOrThrow("animation"));
240                values[3] = cursor.getDouble(
241                        cursor.getColumnIndexOrThrow("layout"));
242                values[4] = cursor.getDouble(
243                        cursor.getColumnIndexOrThrow("draw"));
244                values[5] = cursor.getDouble(
245                        cursor.getColumnIndexOrThrow("sync"));
246                values[6] = cursor.getDouble(
247                        cursor.getColumnIndexOrThrow("command_issue"));
248                values[7] = cursor.getDouble(
249                        cursor.getColumnIndexOrThrow("swap_buffers"));
250                values[8] = cursor.getDouble(
251                        cursor.getColumnIndexOrThrow("total_duration"));
252                values[8] = cursor.getDouble(
253                        cursor.getColumnIndexOrThrow("total_duration"));
254
255                UiBenchmarkResult iterationResult;
256                if (resultList.size() == iteration) {
257                    iterationResult = new UiBenchmarkResult(values);
258                    resultList.add(iterationResult);
259                } else {
260                    iterationResult = resultList.get(iteration);
261                    iterationResult.update(values);
262                }
263            }
264
265            cursor.close();
266        } finally {
267            db.close();
268        }
269
270        return results;
271    }
272
273    public void exportToCsv() throws IOException {
274        String path = mContext.getFilesDir() + "/results-" + System.currentTimeMillis() + ".csv";
275        SQLiteDatabase db = getReadableDatabase();
276
277        // stats across metrics for each run and each test
278        HashMap<String, DescriptiveStatistics> stats = new HashMap<>();
279
280        Cursor runIdCursor = db.query(
281                UI_RESULTS_TABLE, new String[] { "run_id" }, null, null, "run_id", null, null);
282
283        while (runIdCursor.moveToNext()) {
284
285            int runId = runIdCursor.getInt(runIdCursor.getColumnIndexOrThrow("run_id"));
286            HashMap<String, ArrayList<UiBenchmarkResult>> detailedResults =
287                    loadDetailedResults(runId);
288
289            writeRawResults(runId, detailedResults);
290
291            DescriptiveStatistics overall = new DescriptiveStatistics();
292            try (FileWriter writer = new FileWriter(path, true)) {
293                writer.write("Run ID, " + runId + "\n");
294                writer.write("Test, Iteration, Score, Jank Penalty, Consistency Bonus, 95th, " +
295                        "90th\n");
296                for (String testName : detailedResults.keySet()) {
297                    ArrayList<UiBenchmarkResult> results = detailedResults.get(testName);
298                    DescriptiveStatistics scoreStats = new DescriptiveStatistics();
299                    DescriptiveStatistics jankPenalty = new DescriptiveStatistics();
300                    DescriptiveStatistics consistencyBonus = new DescriptiveStatistics();
301                    for (int i = 0; i < results.size(); i++) {
302                        UiBenchmarkResult result = results.get(i);
303                        int score = result.getScore();
304                        scoreStats.addValue(score);
305                        overall.addValue(score);
306                        jankPenalty.addValue(result.getJankPenalty());
307                        consistencyBonus.addValue(result.getConsistencyBonus());
308
309                        writer.write(testName);
310                        writer.write(",");
311                        writer.write("" + i);
312                        writer.write(",");
313                        writer.write("" + score);
314                        writer.write(",");
315                        writer.write("" + result.getJankPenalty());
316                        writer.write(",");
317                        writer.write("" + result.getConsistencyBonus());
318                        writer.write(",");
319                        writer.write(Double.toString(
320                                result.getPercentile(FrameMetrics.TOTAL_DURATION, 95)));
321                        writer.write(",");
322                        writer.write(Double.toString(
323                                result.getPercentile(FrameMetrics.TOTAL_DURATION, 90)));
324                        writer.write("\n");
325                    }
326
327                    writer.write("Score CV," +
328                            (100 * scoreStats.getStandardDeviation()
329                                    / scoreStats.getMean()) + "%\n");
330                    writer.write("Jank Penalty CV, " +
331                            (100 * jankPenalty.getStandardDeviation()
332                                    / jankPenalty.getMean()) + "%\n");
333                    writer.write("Consistency Bonus CV, " +
334                            (100 * consistencyBonus.getStandardDeviation()
335                                    / consistencyBonus.getMean()) + "%\n");
336                    writer.write("\n");
337                }
338
339                writer.write("Overall Score CV,"  +
340                        (100 * overall.getStandardDeviation() / overall.getMean()) + "%\n");
341                writer.flush();
342            }
343        }
344
345        runIdCursor.close();
346    }
347
348    private void writeRawResults(int runId,
349                                 HashMap<String, ArrayList<UiBenchmarkResult>> detailedResults) {
350        StringBuilder path = new StringBuilder();
351        path.append(mContext.getFilesDir());
352        path.append("/");
353        path.append(Integer.toString(runId));
354        path.append(".csv");
355        try (FileWriter writer = new FileWriter(path.toString())) {
356            for (String test : detailedResults.keySet()) {
357                writer.write("Test, " + test + "\n");
358                writer.write("iteration, unknown delay, input, animation, layout, draw, sync, " +
359                        "command issue, swap buffers\n");
360                ArrayList<UiBenchmarkResult> runs = detailedResults.get(test);
361                for (int i = 0; i < runs.size(); i++) {
362                    UiBenchmarkResult run = runs.get(i);
363                    for (int j = 0; j < run.getTotalFrameCount(); j++) {
364                        writer.write(Integer.toString(i) + "," +
365                                run.getMetricAtIndex(j, FrameMetrics.UNKNOWN_DELAY_DURATION) + "," +
366                                run.getMetricAtIndex(j, FrameMetrics.INPUT_HANDLING_DURATION) + "," +
367                                run.getMetricAtIndex(j, FrameMetrics.ANIMATION_DURATION) + "," +
368                                run.getMetricAtIndex(j, FrameMetrics.LAYOUT_MEASURE_DURATION) + "," +
369                                run.getMetricAtIndex(j, FrameMetrics.DRAW_DURATION) + "," +
370                                run.getMetricAtIndex(j, FrameMetrics.SYNC_DURATION) + "," +
371                                run.getMetricAtIndex(j, FrameMetrics.COMMAND_ISSUE_DURATION) + "," +
372                                run.getMetricAtIndex(j, FrameMetrics.SWAP_BUFFERS_DURATION) + "," +
373                                run.getMetricAtIndex(j, FrameMetrics.TOTAL_DURATION) + "\n");
374                    }
375                }
376            }
377        } catch (IOException e) {
378            e.printStackTrace();
379        }
380    }
381
382    @Override
383    public void onUpgrade(SQLiteDatabase sqLiteDatabase, int oldVersion, int currentVersion) {
384        if (oldVersion < VERSION) {
385            sqLiteDatabase.execSQL("ALTER TABLE "
386                    + UI_RESULTS_TABLE + " ADD COLUMN timestamp TEXT;");
387        }
388    }
389}
390