gen_bench_expectations.py revision 093ed317cb99f4e2c3283b81cb0da16cc36b980c 
1#!/usr/bin/env python
2# Copyright (c) 2014 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" Generate bench_expectations file from a given set of bench data files. """
7
8import argparse
9import bench_util
10import os
11import re
12import sys
13
14# Parameters for calculating bench ranges.
15RANGE_RATIO = 1.0  # Ratio of range for upper and lower bounds.
16ERR_RATIO = 0.05  # Further widens the range by the ratio of average value.
17
18# List of bench configs to monitor. Ignore all other configs.
19CONFIGS_TO_INCLUDE = ['simple_viewport_1000x1000',
20                      'simple_viewport_1000x1000_gpu',
21                      'simple_viewport_1000x1000_scalar_1.100000',
22                      'simple_viewport_1000x1000_scalar_1.100000_gpu',
23                     ]
24
25
26def compute_ranges(benches):
27  """Given a list of bench numbers, calculate the alert range.
28
29  Args:
30    benches: a list of float bench values.
31
32  Returns:
33    a list of float [lower_bound, upper_bound].
34  """
35  minimum = min(benches)
36  maximum = max(benches)
37  diff = maximum - minimum
38  avg = sum(benches) / len(benches)
39
40  return [minimum - diff * RANGE_RATIO - avg * ERR_RATIO,
41          maximum + diff * RANGE_RATIO + avg * ERR_RATIO]
42
43
44def create_expectations_dict(revision_data_points):
45  """Convert list of bench data points into a dictionary of expectations data.
46
47  Args:
48    revision_data_points: a list of BenchDataPoint objects.
49
50  Returns:
51    a dictionary of this form:
52        keys = tuple of (config, bench) strings.
53        values = list of float [expected, lower_bound, upper_bound] for the key.
54  """
55  bench_dict = {}
56  for point in revision_data_points:
57    if (point.time_type or  # Not walltime which has time_type ''
58        not point.config in CONFIGS_TO_INCLUDE):
59      continue
60    key = (point.config, point.bench)
61    if key in bench_dict:
62      raise Exception('Duplicate bench entry: ' + str(key))
63    bench_dict[key] = [point.time] + compute_ranges(point.per_iter_time)
64
65  return bench_dict
66
67
68def main():
69    """Reads bench data points, then calculate and export expectations.
70    """
71    parser = argparse.ArgumentParser()
72    parser.add_argument(
73        '-a', '--representation_alg', default='25th',
74        help='bench representation algorithm to use, see bench_util.py.')
75    parser.add_argument(
76        '-b', '--builder', required=True,
77        help='name of the builder whose bench ranges we are computing.')
78    parser.add_argument(
79        '-d', '--input_dir', required=True,
80        help='a directory containing bench data files.')
81    parser.add_argument(
82        '-o', '--output_file', required=True,
83        help='file path and name for storing the output bench expectations.')
84    parser.add_argument(
85        '-r', '--git_revision', required=True,
86        help='the git hash to indicate the revision of input data to use.')
87    args = parser.parse_args()
88
89    builder = args.builder
90
91    data_points = bench_util.parse_skp_bench_data(
92        args.input_dir, args.git_revision, args.representation_alg)
93
94    expectations_dict = create_expectations_dict(data_points)
95
96    out_lines = []
97    keys = expectations_dict.keys()
98    keys.sort()
99    for (config, bench) in keys:
100      (expected, lower_bound, upper_bound) = expectations_dict[(config, bench)]
101      out_lines.append('%(bench)s_%(config)s_,%(builder)s-%(representation)s,'
102          '%(expected)s,%(lower_bound)s,%(upper_bound)s' % {
103              'bench': bench,
104              'config': config,
105              'builder': builder,
106              'representation': args.representation_alg,
107              'expected': expected,
108              'lower_bound': lower_bound,
109              'upper_bound': upper_bound})
110
111    with open(args.output_file, 'w') as file_handle:
112      file_handle.write('\n'.join(out_lines))
113
114
115if __name__ == "__main__":
116    main()
117