1#!/usr/bin/env python
2# SPDX-License-Identifier: Apache-2.0
3#
4# Copyright (C) 2017, ARM Limited, Google, and contributors.
5#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17#
18
19import os
20import pandas as pd
21import sqlite3
22import argparse
23import shutil
24
25# Run a comparison experiment between 2 Jankbench result directories
26# containing multiple subtests. This makes it easy to compare results
27# between 2 jankbench runs.
28#
29# Sample run:
30# ./compare_jankbench.py --baseline='./results/Jankbench_baseline'
31# --compare-with='./results/Jankbench_kernelchange'
32#
33# The output will be something like (only showing 25% and 50%):
34#                       25% compare  25%_diff     50%_compare 50%_diff
35#  test_name
36#  image_list_view       2.11249      0.0178108    5.7952  0.0242445
37#  list_view             2.02227      -3.65839     5.74957  -0.095421
38#  shadow_grid           6.00877      -0.000898    6.23746 -0.0057695
39#  high_hitrate_text     5.81625      0.0264913    6.03504  0.0017795
40#
41# (Note that baseline_df is only used for calculations.
42
43JANKBENCH_DB_NAME = 'BenchmarkResults'
44
45def get_results(out_dir):
46    """
47    Extract data from results db and return as a pandas dataframe
48
49    :param out_dir: Output directory for a run of the Jankbench workload
50    :type out_dir: str
51    """
52    path = os.path.join(out_dir, JANKBENCH_DB_NAME)
53    columns = ['_id', 'name', 'run_id', 'iteration', 'total_duration', 'jank_frame']
54    data = []
55    conn = sqlite3.connect(path)
56    for row in conn.execute('SELECT {} FROM ui_results'.format(','.join(columns))):
57        data.append(row)
58    return pd.DataFrame(data, columns=columns)
59
60def build_stats_df(test_outdir):
61    """
62    Build a .describe() df with statistics
63    """
64    stats_dfs = []
65    for t in tests:
66        test_dir = os.path.join(test_outdir, t)
67        res_df = get_results(test_dir).describe(percentiles=[0.25,0.5,0.75,0.9,0.95,0.99])
68        stats_df = res_df['total_duration']
69        stats_df['test_name'] = t
70        stats_dfs.append(stats_df)
71    fdf = pd.concat(stats_dfs, axis = 1).T
72    fdf.set_index('test_name', inplace=True)
73    return fdf
74
75
76parser = argparse.ArgumentParser(description='Jankbench comparisons')
77
78parser.add_argument('--baseline', dest='baseline', action='store', default='default',
79                    required=True, help='baseline out directory')
80
81parser.add_argument('--compare-with', dest='compare_with', action='store', default='default',
82                    required=True, help='out directory to compare with baseline')
83
84args = parser.parse_args()
85
86# Get list of Jankbench tests available
87tests = os.listdir(args.baseline)
88tests = [t for t in tests if os.path.isdir(os.path.join(args.baseline, t))]
89
90# Build a baseline df (experiment baseline - say without kernel change)
91#  compare df (experiment with change)
92#  diff (difference in stats between compare and baseline)
93
94baseline_df = build_stats_df(args.baseline)
95compare_df = build_stats_df(args.compare_with)
96diff = compare_df - baseline_df
97
98diff.columns = [str(col) + '_diff' for col in diff.columns]
99baseline_df.columns = [str(col) + '_baseline' for col in baseline_df.columns]
100compare_df.columns  = [str(col) + '_compare' for col in compare_df.columns]
101
102final_df = pd.concat([compare_df, diff], axis=1)
103final_df = final_df.reindex_axis(sorted(final_df.columns), axis=1)
104
105# Print the results
106print final_df
107