gen_bench_expectations.py revision b6f431612fb6bcf86217f9136846b00ab57efa5c 
1#!/usr/bin/env python
2# Copyright (c) 2014 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" Generate bench_expectations file from a given set of bench data files. """
7
8import argparse
9import bench_util
10import os
11import re
12import sys
13
14# Parameters for calculating bench ranges.
15RANGE_RATIO_UPPER = 1.5  # Ratio of range for upper bounds.
16RANGE_RATIO_LOWER = 2.0  # Ratio of range for lower bounds.
17ERR_RATIO = 0.08  # Further widens the range by the ratio of average value.
18ERR_UB = 1.0  # Adds an absolute upper error to cope with small benches.
19ERR_LB = 1.5
20
21# List of bench configs to monitor. Ignore all other configs.
22CONFIGS_TO_INCLUDE = ['simple_viewport_1000x1000',
23                      'simple_viewport_1000x1000_angle',
24                      'simple_viewport_1000x1000_gpu',
25                      'simple_viewport_1000x1000_scalar_1.100000',
26                      'simple_viewport_1000x1000_scalar_1.100000_gpu',
27                     ]
28
29# List of flaky entries that should be excluded. Each entry is defined by a list
30# of 3 strings, corresponding to the substrings of [bench, config, builder] to
31# search for. A bench expectations line is excluded when each of the 3 strings
32# in the list is a substring of the corresponding element of the given line. For
33# instance, ['desk_yahooanswers', 'gpu', 'Ubuntu'] will skip expectation entries
34# of SKP benchs whose name contains 'desk_yahooanswers' on all gpu-related
35# configs of all Ubuntu builders.
36ENTRIES_TO_EXCLUDE = [
37                     ]
38
39
40def compute_ranges(benches):
41  """Given a list of bench numbers, calculate the alert range.
42
43  Args:
44    benches: a list of float bench values.
45
46  Returns:
47    a list of float [lower_bound, upper_bound].
48  """
49  avg = sum(benches) / len(benches)
50  squared_avg = avg ** 2
51  avg_squared = sum([bench**2 for bench in benches])/len(benches)
52  std_dev = (avg_squared - squared_avg) ** 0.5
53
54  # If the results are normally distributed, 2 standard deviations
55  # captures something like ~95% of the possible range of results I think
56  return [avg - 2*std_dev, avg + 2*std_dev]
57
58
59def create_expectations_dict(revision_data_points, builder):
60  """Convert list of bench data points into a dictionary of expectations data.
61
62  Args:
63    revision_data_points: a list of BenchDataPoint objects.
64    builder: string of the corresponding buildbot builder name.
65
66  Returns:
67    a dictionary of this form:
68        keys = tuple of (config, bench) strings.
69        values = list of float [expected, lower_bound, upper_bound] for the key.
70  """
71  bench_dict = {}
72  for point in revision_data_points:
73    if (point.time_type or  # Not walltime which has time_type ''
74        not point.config in CONFIGS_TO_INCLUDE):
75      continue
76    to_skip = False
77    for bench_substr, config_substr, builder_substr in ENTRIES_TO_EXCLUDE:
78      if (bench_substr in point.bench and config_substr in point.config and
79          builder_substr in builder):
80        to_skip = True
81        break
82    if to_skip:
83      continue
84    key = (point.config, point.bench)
85    if key in bench_dict:
86      raise Exception('Duplicate bench entry: ' + str(key))
87    bench_dict[key] = [point.time] + compute_ranges(point.per_iter_time)
88
89  return bench_dict
90
91
92def main():
93    """Reads bench data points, then calculate and export expectations.
94    """
95    parser = argparse.ArgumentParser()
96    parser.add_argument(
97        '-a', '--representation_alg', default='25th',
98        help='bench representation algorithm to use, see bench_util.py.')
99    parser.add_argument(
100        '-b', '--builder', required=True,
101        help='name of the builder whose bench ranges we are computing.')
102    parser.add_argument(
103        '-d', '--input_dir', required=True,
104        help='a directory containing bench data files.')
105    parser.add_argument(
106        '-o', '--output_file', required=True,
107        help='file path and name for storing the output bench expectations.')
108    parser.add_argument(
109        '-r', '--git_revision', required=True,
110        help='the git hash to indicate the revision of input data to use.')
111    args = parser.parse_args()
112
113    builder = args.builder
114
115    data_points = bench_util.parse_skp_bench_data(
116        args.input_dir, args.git_revision, args.representation_alg)
117
118    expectations_dict = create_expectations_dict(data_points, builder)
119
120    out_lines = []
121    keys = expectations_dict.keys()
122    keys.sort()
123    for (config, bench) in keys:
124      (expected, lower_bound, upper_bound) = expectations_dict[(config, bench)]
125      out_lines.append('%(bench)s_%(config)s_,%(builder)s-%(representation)s,'
126          '%(expected)s,%(lower_bound)s,%(upper_bound)s' % {
127              'bench': bench,
128              'config': config,
129              'builder': builder,
130              'representation': args.representation_alg,
131              'expected': expected,
132              'lower_bound': lower_bound,
133              'upper_bound': upper_bound})
134
135    with open(args.output_file, 'w') as file_handle:
136      file_handle.write('\n'.join(out_lines))
137
138
139if __name__ == "__main__":
140    main()
141