1363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger#!/usr/bin/env python 2363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger# Use of this source code is governed by a BSD-style license that can be found 4363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger# in the LICENSE file. 5363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger""" Analyze recent SkPicture or Microbench data, and output suggested ranges. 7363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 8363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerThe outputs can be edited and pasted to bench_expectations.txt to trigger 9363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerbuildbot alerts if the actual benches are out of range. Details are documented 10363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerin the .txt file. 11363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 1258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek SollenbergerCurrently the easiest way to batch update bench_expectations.txt is to delete 1358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerall bench lines, run this script, and redirect outputs (">>") to be added to the 14363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger.txt file. 1558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek SollenbergerYou can also just manually change a few lines of interest, of course. 16363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 17363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerNote: since input data are stored in Google Storage, you will need to set up 18363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerthe corresponding library. 19363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerSee http://developers.google.com/storage/docs/gspythonlibrary for details. 20363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger""" 21363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 22363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger__author__ = 'bensong@google.com (Ben Chen)' 23363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 24363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerimport bench_util 25363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerimport boto 26363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerimport cStringIO 27363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerimport optparse 28363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerimport re 29363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerimport shutil 30363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 31363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerfrom oauth2_plugin import oauth2_plugin 32363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 33363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 34363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger# Ratios for calculating suggested picture bench upper and lower bounds. 35363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerBENCH_UB = 1.1 # Allow for 10% room for normal variance on the up side. 36096defe64d408e54474fe19f418c95bf1a554fc7Derek SollenbergerBENCH_LB = 0.9 37363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 38363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger# Further allow for a fixed amount of noise. This is especially useful for 39363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger# benches of smaller absolute value. Keeping this value small will not affect 40363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger# performance tunings. 41363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerBENCH_ALLOWED_NOISE = 10 42363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 437839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger# Name prefix for benchmark builders. 447839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek SollenbergerBENCH_BUILDER_PREFIX = 'Perf-' 457839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 46096defe64d408e54474fe19f418c95bf1a554fc7Derek Sollenberger# List of platforms to track. Feel free to change it to meet your needs. 477839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek SollenbergerPLATFORMS = ['Perf-Mac10.8-MacMini4.1-GeForce320M-x86-Release', 487839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 'Perf-Android-Nexus7-Tegra3-Arm7-Release', 497839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 'Perf-Ubuntu12-ShuttleA-ATI5770-x86-Release', 507839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 'Perf-Win7-ShuttleA-HD2000-x86-Release', 51363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger ] 52363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 53363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger# Filter for configs of no interest. They are old config names replaced by more 54363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger# specific ones. 55363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerCONFIGS_TO_FILTER = ['gpu', 'raster'] 56363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 57363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger# Template for gsutil uri. 58363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerGOOGLE_STORAGE_URI_SCHEME = 'gs' 59363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerURI_BUCKET = 'chromium-skia-gm' 60363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 61363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger# Constants for optparse. 62363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerUSAGE_STRING = 'USAGE: %s [options]' 63363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerHOWTO_STRING = """ 64363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerFeel free to revise PLATFORMS for your own needs. The default is the most common 65363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergercombination that we care most about. Platforms that did not run bench_pictures 6658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergeror benchmain in the given revision range will not have corresponding outputs. 67363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerPlease check http://go/skpbench to choose a range that fits your needs. 68363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerBENCH_UB, BENCH_LB and BENCH_ALLOWED_NOISE can be changed to expand or narrow 69363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerthe permitted bench ranges without triggering buidbot alerts. 70363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger""" 71363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerHELP_STRING = """ 72363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerOutputs expectation picture bench ranges for the latest revisions for the given 73363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerrevision range. For instance, --rev_range=6000:6000 will return only bench 7458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerranges for the bots that ran benches at rev 6000; --rev-range=6000:7000 75363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergermay have multiple bench data points for each bench configuration, and the code 76363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerreturns bench data for the latest revision of all available (closer to 7000). 77363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger""" + HOWTO_STRING 78363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 79363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerOPTION_REVISION_RANGE = '--rev-range' 80363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerOPTION_REVISION_RANGE_SHORT = '-r' 8158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# Bench representation algorithm flag. 82363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerOPTION_REPRESENTATION_ALG = '--algorithm' 83363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerOPTION_REPRESENTATION_ALG_SHORT = '-a' 8458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# Bench type to examine. Either 'micro' or 'skp'. 8558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek SollenbergerOPTION_BENCH_TYPE = '--bench-type' 8658190644c30e1c4aa8e527f3503c58f841e0fcf3Derek SollenbergerOPTION_BENCH_TYPE_SHORT = '-b' 87363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 8858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# List of valid bench types. 8958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek SollenbergerBENCH_TYPES = ['micro', 'skp'] 90363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger# List of valid representation algorithms. 91363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek SollenbergerREPRESENTATION_ALGS = ['avg', 'min', 'med', '25th'] 92363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 9358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerdef OutputBenchExpectations(bench_type, rev_min, rev_max, representation_alg): 9458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger """Reads bench data from google storage, and outputs expectations. 95363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 96363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger Ignores data with revisions outside [rev_min, rev_max] integer range. For 97363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger bench data with multiple revisions, we use higher revisions to calculate 98363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger expected bench values. 9958190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger bench_type is either 'micro' or 'skp', according to the flag '-b'. 100363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger Uses the provided representation_alg for calculating bench representations. 101363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger """ 10258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger if bench_type not in BENCH_TYPES: 10358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger raise Exception('Not valid bench_type! (%s)' % BENCH_TYPES) 104363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger expectation_dic = {} 105363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger uri = boto.storage_uri(URI_BUCKET, GOOGLE_STORAGE_URI_SCHEME) 106363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger for obj in uri.get_bucket(): 10758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger # Filters out non-bench files. 1087839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger if ((not obj.name.startswith('perfdata/%s' % BENCH_BUILDER_PREFIX) and 1097839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger not obj.name.startswith( 1107839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 'playback/perfdata/%s' % BENCH_BUILDER_PREFIX)) or 11158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger obj.name.find('_data') < 0): 11258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger continue 11358190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger if ((bench_type == 'micro' and obj.name.find('_data_skp_') > 0) or 11458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger (bench_type == 'skp' and obj.name.find('_skp_') < 0)): 11558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger # Skips wrong bench type. 116363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger continue 117363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger # Ignores uninterested platforms. 118096defe64d408e54474fe19f418c95bf1a554fc7Derek Sollenberger platform = obj.name.split('/')[1] 1197839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger if not platform.startswith(BENCH_BUILDER_PREFIX): 120096defe64d408e54474fe19f418c95bf1a554fc7Derek Sollenberger platform = obj.name.split('/')[2] 1217839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger if not platform.startswith(BENCH_BUILDER_PREFIX): 122096defe64d408e54474fe19f418c95bf1a554fc7Derek Sollenberger continue # Ignores non-platform object 123363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger if platform not in PLATFORMS: 124363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger continue 125363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger # Filters by revision. 126096defe64d408e54474fe19f418c95bf1a554fc7Derek Sollenberger to_filter = True 127363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger for rev in range(rev_min, rev_max + 1): 128096defe64d408e54474fe19f418c95bf1a554fc7Derek Sollenberger if '_r%s_' % rev in obj.name: 129096defe64d408e54474fe19f418c95bf1a554fc7Derek Sollenberger to_filter = False 130096defe64d408e54474fe19f418c95bf1a554fc7Derek Sollenberger break 131096defe64d408e54474fe19f418c95bf1a554fc7Derek Sollenberger if to_filter: 132096defe64d408e54474fe19f418c95bf1a554fc7Derek Sollenberger continue 133363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger contents = cStringIO.StringIO() 134363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger obj.get_file(contents) 135363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger for point in bench_util.parse('', contents.getvalue().split('\n'), 136363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger representation_alg): 137363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger if point.config in CONFIGS_TO_FILTER: 138363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger continue 139363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 140363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger key = '%s_%s_%s,%s-%s' % (point.bench, point.config, point.time_type, 141363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger platform, representation_alg) 142363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger # It is fine to have later revisions overwrite earlier benches, since we 143363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger # only use the latest bench within revision range to set expectations. 144363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger expectation_dic[key] = point.time 145363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger keys = expectation_dic.keys() 146363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger keys.sort() 147363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger for key in keys: 148363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger bench_val = expectation_dic[key] 149363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger # Prints out expectation lines. 150363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger print '%s,%.3f,%.3f,%.3f' % (key, bench_val, 151363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger bench_val * BENCH_LB - BENCH_ALLOWED_NOISE, 152363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger bench_val * BENCH_UB + BENCH_ALLOWED_NOISE) 153363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 154363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerdef main(): 15558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger """Parses flags and outputs expected Skia bench results.""" 156363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING) 157363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger parser.add_option(OPTION_REVISION_RANGE_SHORT, OPTION_REVISION_RANGE, 158363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger dest='rev_range', 159363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger help='(Mandatory) revision range separated by ":", e.g., 6000:6005') 16058190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger parser.add_option(OPTION_BENCH_TYPE_SHORT, OPTION_BENCH_TYPE, 16158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger dest='bench_type', default='skp', 16258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger help=('Bench type, either "skp" or "micro". Default to "skp".')) 163363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger parser.add_option(OPTION_REPRESENTATION_ALG_SHORT, OPTION_REPRESENTATION_ALG, 164363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger dest='alg', default='25th', 165363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger help=('Bench representation algorithm. One of ' 166363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger '%s. Default to "25th".' % str(REPRESENTATION_ALGS))) 167363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger (options, args) = parser.parse_args() 168363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger if options.rev_range: 169363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger range_match = re.search('(\d+)\:(\d+)', options.rev_range) 170363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger if not range_match: 171363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger parser.error('Wrong format for rev-range [%s]' % options.rev_range) 172363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger else: 173363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger rev_min = int(range_match.group(1)) 174363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger rev_max = int(range_match.group(2)) 17558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger OutputBenchExpectations(options.bench_type, rev_min, rev_max, options.alg) 176363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger else: 177363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger parser.error('Please provide mandatory flag %s' % OPTION_REVISION_RANGE) 178363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 179363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger 180363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenbergerif '__main__' == __name__: 181363e546ed626b6dbbc42f5db87b3594bc0b5944bDerek Sollenberger main() 182