1#!/usr/bin/env python 2# Copyright (c) 2013 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Run Performance Test Bisect Tool 7 8This script is used by a trybot to run the src/tools/bisect-perf-regression.py 9script with the parameters specified in run-bisect-perf-regression.cfg. It will 10check out a copy of the depot in a subdirectory 'bisect' of the working 11directory provided, and run the bisect-perf-regression.py script there. 12 13""" 14 15import imp 16import optparse 17import os 18import subprocess 19import sys 20import traceback 21 22import bisect_utils 23bisect = imp.load_source('bisect-perf-regression', 24 os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), 25 'bisect-perf-regression.py')) 26 27 28CROS_BOARD_ENV = 'BISECT_CROS_BOARD' 29CROS_IP_ENV = 'BISECT_CROS_IP' 30 31 32class Goma(object): 33 34 def __init__(self, path_to_goma): 35 self._abs_path_to_goma = None 36 self._abs_path_to_goma_file = None 37 if path_to_goma: 38 self._abs_path_to_goma = os.path.abspath(path_to_goma) 39 self._abs_path_to_goma_file = self._GetExecutablePath( 40 self._abs_path_to_goma) 41 42 def __enter__(self): 43 if self._HasGOMAPath(): 44 self._SetupAndStart() 45 return self 46 47 def __exit__(self, *_): 48 if self._HasGOMAPath(): 49 self._Stop() 50 51 def _HasGOMAPath(self): 52 return bool(self._abs_path_to_goma) 53 54 def _GetExecutablePath(self, path_to_goma): 55 if os.name == 'nt': 56 return os.path.join(path_to_goma, 'goma_ctl.bat') 57 else: 58 return os.path.join(path_to_goma, 'goma_ctl.sh') 59 60 def _SetupEnvVars(self): 61 if os.name == 'nt': 62 os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') + 63 ' cl.exe') 64 os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') + 65 ' cl.exe') 66 else: 67 os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma, 68 os.environ['PATH']]) 69 70 def _SetupAndStart(self): 71 """Sets up GOMA and launches it. 72 73 Args: 74 path_to_goma: Path to goma directory. 75 76 Returns: 77 True if successful.""" 78 self._SetupEnvVars() 79 80 # Sometimes goma is lingering around if something went bad on a previous 81 # run. Stop it before starting a new process. Can ignore the return code 82 # since it will return an error if it wasn't running. 83 self._Stop() 84 85 if subprocess.call([self._abs_path_to_goma_file, 'start']): 86 raise RuntimeError('GOMA failed to start.') 87 88 def _Stop(self): 89 subprocess.call([self._abs_path_to_goma_file, 'stop']) 90 91 92 93def _LoadConfigFile(path_to_file): 94 """Attempts to load the specified config file as a module 95 and grab the global config dict. 96 97 Args: 98 path_to_file: Path to the file. 99 100 Returns: 101 The config dict which should be formatted as follows: 102 {'command': string, 'good_revision': string, 'bad_revision': string 103 'metric': string, etc...}. 104 Returns None on failure. 105 """ 106 try: 107 local_vars = {} 108 execfile(path_to_file, local_vars) 109 110 return local_vars['config'] 111 except: 112 print 113 traceback.print_exc() 114 print 115 return {} 116 117 118def _OutputFailedResults(text_to_print): 119 bisect_utils.OutputAnnotationStepStart('Results - Failed') 120 print 121 print text_to_print 122 print 123 bisect_utils.OutputAnnotationStepClosed() 124 125 126def _CreateBisectOptionsFromConfig(config): 127 opts_dict = {} 128 opts_dict['command'] = config['command'] 129 opts_dict['metric'] = config['metric'] 130 131 if config['repeat_count']: 132 opts_dict['repeat_test_count'] = int(config['repeat_count']) 133 134 if config['truncate_percent']: 135 opts_dict['truncate_percent'] = int(config['truncate_percent']) 136 137 if config['max_time_minutes']: 138 opts_dict['max_time_minutes'] = int(config['max_time_minutes']) 139 140 if config.has_key('use_goma'): 141 opts_dict['use_goma'] = config['use_goma'] 142 143 opts_dict['build_preference'] = 'ninja' 144 opts_dict['output_buildbot_annotations'] = True 145 146 if '--browser=cros' in config['command']: 147 opts_dict['target_platform'] = 'cros' 148 149 if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]: 150 opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV] 151 opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV] 152 else: 153 raise RuntimeError('Cros build selected, but BISECT_CROS_IP or' 154 'BISECT_CROS_BOARD undefined.') 155 elif 'android' in config['command']: 156 if 'android-chrome' in config['command']: 157 opts_dict['target_platform'] = 'android-chrome' 158 else: 159 opts_dict['target_platform'] = 'android' 160 161 return bisect.BisectOptions.FromDict(opts_dict) 162 163 164def _RunPerformanceTest(config, path_to_file): 165 # Bisect script expects to be run from src 166 os.chdir(os.path.join(path_to_file, '..')) 167 168 bisect_utils.OutputAnnotationStepStart('Building With Patch') 169 170 opts = _CreateBisectOptionsFromConfig(config) 171 b = bisect.BisectPerformanceMetrics(None, opts) 172 173 if bisect_utils.RunGClient(['runhooks']): 174 raise RuntimeError('Failed to run gclient runhooks') 175 176 if not b.BuildCurrentRevision('chromium'): 177 raise RuntimeError('Patched version failed to build.') 178 179 bisect_utils.OutputAnnotationStepClosed() 180 bisect_utils.OutputAnnotationStepStart('Running With Patch') 181 182 results_with_patch = b.RunPerformanceTestAndParseResults( 183 opts.command, opts.metric, reset_on_first_run=True, results_label='Patch') 184 185 if results_with_patch[1]: 186 raise RuntimeError('Patched version failed to run performance test.') 187 188 bisect_utils.OutputAnnotationStepClosed() 189 190 bisect_utils.OutputAnnotationStepStart('Reverting Patch') 191 if bisect_utils.RunGClient(['revert']): 192 raise RuntimeError('Failed to run gclient runhooks') 193 bisect_utils.OutputAnnotationStepClosed() 194 195 bisect_utils.OutputAnnotationStepStart('Building Without Patch') 196 197 if bisect_utils.RunGClient(['runhooks']): 198 raise RuntimeError('Failed to run gclient runhooks') 199 200 if not b.BuildCurrentRevision('chromium'): 201 raise RuntimeError('Unpatched version failed to build.') 202 203 bisect_utils.OutputAnnotationStepClosed() 204 bisect_utils.OutputAnnotationStepStart('Running Without Patch') 205 206 results_without_patch = b.RunPerformanceTestAndParseResults( 207 opts.command, opts.metric, upload_on_last_run=True, results_label='ToT') 208 209 if results_without_patch[1]: 210 raise RuntimeError('Unpatched version failed to run performance test.') 211 212 # Find the link to the cloud stored results file. 213 output = results_without_patch[2] 214 cloud_file_link = [t for t in output.splitlines() 215 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t] 216 if cloud_file_link: 217 # What we're getting here is basically "View online at http://..." so parse 218 # out just the url portion. 219 cloud_file_link = cloud_file_link[0] 220 cloud_file_link = [t for t in cloud_file_link.split(' ') 221 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t] 222 assert cloud_file_link, "Couldn't parse url from output." 223 cloud_file_link = cloud_file_link[0] 224 else: 225 cloud_file_link = '' 226 227 # Calculate the % difference in the means of the 2 runs. 228 percent_diff_in_means = (results_with_patch[0]['mean'] / 229 max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0 230 std_err = bisect.CalculatePooledStandardError( 231 [results_with_patch[0]['values'], results_without_patch[0]['values']]) 232 233 bisect_utils.OutputAnnotationStepClosed() 234 bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' % 235 (percent_diff_in_means, std_err)) 236 print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '), 237 'Std. Error'.center(20, ' ')) 238 print ' %s %s %s' % ('Patch'.center(10, ' '), 239 ('%.02f' % results_with_patch[0]['mean']).center(20, ' '), 240 ('%.02f' % results_with_patch[0]['std_err']).center(20, ' ')) 241 print ' %s %s %s' % ('No Patch'.center(10, ' '), 242 ('%.02f' % results_without_patch[0]['mean']).center(20, ' '), 243 ('%.02f' % results_without_patch[0]['std_err']).center(20, ' ')) 244 if cloud_file_link: 245 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) 246 bisect_utils.OutputAnnotationStepClosed() 247 248 249def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma): 250 """Attempts to build and run the current revision with and without the 251 current patch, with the parameters passed in. 252 253 Args: 254 config: The config read from run-perf-test.cfg. 255 path_to_file: Path to the bisect-perf-regression.py script. 256 path_to_goma: Path to goma directory. 257 258 Returns: 259 0 on success, otherwise 1. 260 """ 261 try: 262 with Goma(path_to_goma) as goma: 263 config['use_goma'] = bool(path_to_goma) 264 _RunPerformanceTest(config, path_to_file) 265 return 0 266 except RuntimeError, e: 267 bisect_utils.OutputAnnotationStepClosed() 268 _OutputFailedResults('Error: %s' % e.message) 269 return 1 270 271 272def _RunBisectionScript(config, working_directory, path_to_file, path_to_goma, 273 path_to_extra_src, dry_run): 274 """Attempts to execute src/tools/bisect-perf-regression.py with the parameters 275 passed in. 276 277 Args: 278 config: A dict containing the parameters to pass to the script. 279 working_directory: A working directory to provide to the 280 bisect-perf-regression.py script, where it will store it's own copy of 281 the depot. 282 path_to_file: Path to the bisect-perf-regression.py script. 283 path_to_goma: Path to goma directory. 284 path_to_extra_src: Path to extra source file. 285 dry_run: Do a dry run, skipping sync, build, and performance testing steps. 286 287 Returns: 288 0 on success, otherwise 1. 289 """ 290 bisect_utils.OutputAnnotationStepStart('Config') 291 print 292 for k, v in config.iteritems(): 293 print ' %s : %s' % (k, v) 294 print 295 bisect_utils.OutputAnnotationStepClosed() 296 297 cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'), 298 '-c', config['command'], 299 '-g', config['good_revision'], 300 '-b', config['bad_revision'], 301 '-m', config['metric'], 302 '--working_directory', working_directory, 303 '--output_buildbot_annotations'] 304 305 if config['repeat_count']: 306 cmd.extend(['-r', config['repeat_count']]) 307 308 if config['truncate_percent']: 309 cmd.extend(['-t', config['truncate_percent']]) 310 311 if config['max_time_minutes']: 312 cmd.extend(['--max_time_minutes', config['max_time_minutes']]) 313 314 cmd.extend(['--build_preference', 'ninja']) 315 316 if '--browser=cros' in config['command']: 317 cmd.extend(['--target_platform', 'cros']) 318 319 if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]: 320 cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]]) 321 cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]]) 322 else: 323 print 'Error: Cros build selected, but BISECT_CROS_IP or'\ 324 'BISECT_CROS_BOARD undefined.' 325 print 326 return 1 327 328 if 'android' in config['command']: 329 if 'android-chrome' in config['command']: 330 cmd.extend(['--target_platform', 'android-chrome']) 331 else: 332 cmd.extend(['--target_platform', 'android']) 333 334 if path_to_goma: 335 cmd.append('--use_goma') 336 337 if path_to_extra_src: 338 cmd.extend(['--extra_src', path_to_extra_src]) 339 340 if dry_run: 341 cmd.extend(['--debug_ignore_build', '--debug_ignore_sync', 342 '--debug_ignore_perf_test']) 343 cmd = [str(c) for c in cmd] 344 345 with Goma(path_to_goma) as goma: 346 return_code = subprocess.call(cmd) 347 348 if return_code: 349 print 'Error: bisect-perf-regression.py returned with error %d' %\ 350 return_code 351 print 352 353 return return_code 354 355 356def main(): 357 358 usage = ('%prog [options] [-- chromium-options]\n' 359 'Used by a trybot to run the bisection script using the parameters' 360 ' provided in the run-bisect-perf-regression.cfg file.') 361 362 parser = optparse.OptionParser(usage=usage) 363 parser.add_option('-w', '--working_directory', 364 type='str', 365 help='A working directory to supply to the bisection ' 366 'script, which will use it as the location to checkout ' 367 'a copy of the chromium depot.') 368 parser.add_option('-p', '--path_to_goma', 369 type='str', 370 help='Path to goma directory. If this is supplied, goma ' 371 'builds will be enabled.') 372 parser.add_option('--extra_src', 373 type='str', 374 help='Path to extra source file. If this is supplied, ' 375 'bisect script will use this to override default behavior.') 376 parser.add_option('--dry_run', 377 action="store_true", 378 help='The script will perform the full bisect, but ' 379 'without syncing, building, or running the performance ' 380 'tests.') 381 (opts, args) = parser.parse_args() 382 383 path_to_current_directory = os.path.abspath(os.path.dirname(sys.argv[0])) 384 path_to_bisect_cfg = os.path.join(path_to_current_directory, 385 'run-bisect-perf-regression.cfg') 386 387 config = _LoadConfigFile(path_to_bisect_cfg) 388 389 # Check if the config is empty 390 config_has_values = [v for v in config.values() if v] 391 392 if config and config_has_values: 393 if not opts.working_directory: 394 print 'Error: missing required parameter: --working_directory' 395 print 396 parser.print_help() 397 return 1 398 399 return _RunBisectionScript(config, opts.working_directory, 400 path_to_current_directory, opts.path_to_goma, opts.extra_src, 401 opts.dry_run) 402 else: 403 perf_cfg_files = ['run-perf-test.cfg', os.path.join('..', 'third_party', 404 'WebKit', 'Tools', 'run-perf-test.cfg')] 405 406 for current_perf_cfg_file in perf_cfg_files: 407 path_to_perf_cfg = os.path.join( 408 os.path.abspath(os.path.dirname(sys.argv[0])), current_perf_cfg_file) 409 410 config = _LoadConfigFile(path_to_perf_cfg) 411 config_has_values = [v for v in config.values() if v] 412 413 if config and config_has_values: 414 return _SetupAndRunPerformanceTest(config, path_to_current_directory, 415 opts.path_to_goma) 416 417 print 'Error: Could not load config file. Double check your changes to '\ 418 'run-bisect-perf-regression.cfg/run-perf-test.cfg for syntax errors.' 419 print 420 return 1 421 422 423if __name__ == '__main__': 424 sys.exit(main()) 425