gen_bench_expectations.py revision 5a220a1f479f02676bc1a92fea0b2cc2556ec4f9
1#!/usr/bin/env python 2# Copyright (c) 2014 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6""" Generate bench_expectations file from a given set of bench data files. """ 7 8import argparse 9import bench_util 10import os 11import re 12import sys 13 14# Parameters for calculating bench ranges. 15RANGE_RATIO_UPPER = 1.5 # Ratio of range for upper bounds. 16RANGE_RATIO_LOWER = 2.0 # Ratio of range for lower bounds. 17ERR_RATIO = 0.08 # Further widens the range by the ratio of average value. 18ERR_UB = 1.0 # Adds an absolute upper error to cope with small benches. 19ERR_LB = 1.5 20 21# List of bench configs to monitor. Ignore all other configs. 22CONFIGS_TO_INCLUDE = ['simple_viewport_1000x1000', 23 'simple_viewport_1000x1000_angle', 24 'simple_viewport_1000x1000_gpu', 25 'simple_viewport_1000x1000_scalar_1.100000', 26 'simple_viewport_1000x1000_scalar_1.100000_gpu', 27 ] 28 29# List of flaky entries that should be excluded. Each entry is defined by a list 30# of 3 strings, corresponding to the substrings of [bench, config, builder] to 31# search for. A bench expectations line is excluded when each of the 3 strings 32# in the list is a substring of the corresponding element of the given line. For 33# instance, ['desk_yahooanswers', 'gpu', 'Ubuntu'] will skip expectation entries 34# of SKP benchs whose name contains 'desk_yahooanswers' on all gpu-related 35# configs of all Ubuntu builders. 36ENTRIES_TO_EXCLUDE = [ 37 ] 38 39 40def compute_ranges(benches): 41 """Given a list of bench numbers, calculate the alert range. 42 43 Args: 44 benches: a list of float bench values. 45 46 Returns: 47 a list of float [lower_bound, upper_bound]. 48 """ 49 avg = sum(benches) / len(benches) 50 squared_avg = avg ** 2 51 avg_sum_squared = sum([bench**2 for bench in benches])/len(benches) 52 std_dev = (abs(avg_sum_squared - squared_avg) + 0.05*abs(avg)) ** 0.5 53 54 # If the results are normally distributed, 2 standard deviations 55 # captures something like ~95% of the possible range of results I think 56 return [avg - 2*std_dev, avg + 2*std_dev] 57 58 59def create_expectations_dict(revision_data_points, builder): 60 """Convert list of bench data points into a dictionary of expectations data. 61 62 Args: 63 revision_data_points: a list of BenchDataPoint objects. 64 builder: string of the corresponding buildbot builder name. 65 66 Returns: 67 a dictionary of this form: 68 keys = tuple of (config, bench) strings. 69 values = list of float [expected, lower_bound, upper_bound] for the key. 70 """ 71 bench_dict = {} 72 for point in revision_data_points: 73 if (point.time_type or # Not walltime which has time_type '' 74 not point.config in CONFIGS_TO_INCLUDE): 75 continue 76 to_skip = False 77 for bench_substr, config_substr, builder_substr in ENTRIES_TO_EXCLUDE: 78 if (bench_substr in point.bench and config_substr in point.config and 79 builder_substr in builder): 80 to_skip = True 81 break 82 if to_skip: 83 continue 84 key = (point.config, point.bench) 85 if key in bench_dict: 86 raise Exception('Duplicate bench entry: ' + str(key)) 87 bench_dict[key] = [point.time] + compute_ranges(point.per_iter_time) 88 89 return bench_dict 90 91 92def main(): 93 """Reads bench data points, then calculate and export expectations. 94 """ 95 parser = argparse.ArgumentParser() 96 parser.add_argument( 97 '-a', '--representation_alg', default='25th', 98 help='bench representation algorithm to use, see bench_util.py.') 99 parser.add_argument( 100 '-b', '--builder', required=True, 101 help='name of the builder whose bench ranges we are computing.') 102 parser.add_argument( 103 '-d', '--input_dir', required=True, 104 help='a directory containing bench data files.') 105 parser.add_argument( 106 '-o', '--output_file', required=True, 107 help='file path and name for storing the output bench expectations.') 108 parser.add_argument( 109 '-r', '--git_revision', required=True, 110 help='the git hash to indicate the revision of input data to use.') 111 args = parser.parse_args() 112 113 builder = args.builder 114 115 data_points = bench_util.parse_skp_bench_data( 116 args.input_dir, args.git_revision, args.representation_alg) 117 118 expectations_dict = create_expectations_dict(data_points, builder) 119 120 out_lines = [] 121 keys = expectations_dict.keys() 122 keys.sort() 123 for (config, bench) in keys: 124 (expected, lower_bound, upper_bound) = expectations_dict[(config, bench)] 125 out_lines.append('%(bench)s_%(config)s_,%(builder)s-%(representation)s,' 126 '%(expected)s,%(lower_bound)s,%(upper_bound)s' % { 127 'bench': bench, 128 'config': config, 129 'builder': builder, 130 'representation': args.representation_alg, 131 'expected': expected, 132 'lower_bound': lower_bound, 133 'upper_bound': upper_bound}) 134 135 with open(args.output_file, 'w') as file_handle: 136 file_handle.write('\n'.join(out_lines)) 137 138 139if __name__ == "__main__": 140 main() 141