gen_bench_expectations.py revision b1bcb21631275fd5d0f5313b1ea37d0b975c066a 
1#!/usr/bin/env python
2# Copyright (c) 2014 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" Generate bench_expectations file from a given set of bench data files. """
7
8import argparse
9import bench_util
10import os
11import re
12import sys
13
14# Parameters for calculating bench ranges.
15RANGE_RATIO = 1.0  # Ratio of range for upper and lower bounds.
16ABS_ERR = 1.0  # Additional allowed error in milliseconds.
17
18# List of bench configs to monitor. Ignore all other configs.
19CONFIGS_TO_INCLUDE = ['simple_viewport_1000x1000',
20                      'simple_viewport_1000x1000_gpu',
21                      'simple_viewport_1000x1000_scalar_1.100000',
22                      'simple_viewport_1000x1000_scalar_1.100000_gpu',
23                     ]
24
25
26def compute_ranges(benches):
27  """Given a list of bench numbers, calculate the alert range.
28
29  Args:
30    benches: a list of float bench values.
31
32  Returns:
33    a list of float [lower_bound, upper_bound].
34  """
35  minimum = min(benches)
36  maximum = max(benches)
37  diff = maximum - minimum
38
39  return [minimum - diff * RANGE_RATIO - ABS_ERR,
40          maximum + diff * RANGE_RATIO + ABS_ERR]
41
42
43def create_expectations_dict(revision_data_points):
44  """Convert list of bench data points into a dictionary of expectations data.
45
46  Args:
47    revision_data_points: a list of BenchDataPoint objects.
48
49  Returns:
50    a dictionary of this form:
51        keys = tuple of (config, bench) strings.
52        values = list of float [expected, lower_bound, upper_bound] for the key.
53  """
54  bench_dict = {}
55  for point in revision_data_points:
56    if (point.time_type or  # Not walltime which has time_type ''
57        not point.config in CONFIGS_TO_INCLUDE):
58      continue
59    key = (point.config, point.bench)
60    if key in bench_dict:
61      raise Exception('Duplicate bench entry: ' + str(key))
62    bench_dict[key] = [point.time] + compute_ranges(point.per_iter_time)
63
64  return bench_dict
65
66
67def main():
68    """Reads bench data points, then calculate and export expectations.
69    """
70    parser = argparse.ArgumentParser()
71    parser.add_argument(
72        '-a', '--representation_alg', default='25th',
73        help='bench representation algorithm to use, see bench_util.py.')
74    parser.add_argument(
75        '-b', '--builder', required=True,
76        help='name of the builder whose bench ranges we are computing.')
77    parser.add_argument(
78        '-d', '--input_dir', required=True,
79        help='a directory containing bench data files.')
80    parser.add_argument(
81        '-o', '--output_file', required=True,
82        help='file path and name for storing the output bench expectations.')
83    parser.add_argument(
84        '-r', '--git_revision', required=True,
85        help='the git hash to indicate the revision of input data to use.')
86    args = parser.parse_args()
87
88    builder = args.builder
89
90    data_points = bench_util.parse_skp_bench_data(
91        args.input_dir, args.git_revision, args.representation_alg)
92
93    expectations_dict = create_expectations_dict(data_points)
94
95    out_lines = []
96    keys = expectations_dict.keys()
97    keys.sort()
98    for (config, bench) in keys:
99      (expected, lower_bound, upper_bound) = expectations_dict[(config, bench)]
100      out_lines.append('%(bench)s_%(config)s_,%(builder)s-%(representation)s,'
101          '%(expected)s,%(lower_bound)s,%(upper_bound)s' % {
102              'bench': bench,
103              'config': config,
104              'builder': builder,
105              'representation': args.representation_alg,
106              'expected': expected,
107              'lower_bound': lower_bound,
108              'upper_bound': upper_bound})
109
110    with open(args.output_file, 'w') as file_handle:
111      file_handle.write('\n'.join(out_lines))
112
113
114if __name__ == "__main__":
115    main()
116