12a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#!/usr/bin/env python 22a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Copyright (c) 2013 The Chromium Authors. All rights reserved. 32a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 42a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# found in the LICENSE file. 52a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 62a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)"""Run Performance Test Bisect Tool 72a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 81320f92c476a1ad9d19dba2a48c72b75566198e9Primiano TucciThis script is used by a try bot to run the bisect script with the parameters 91320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccispecified in the bisect config file. It checks out a copy of the depot in 101320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccia subdirectory 'bisect' of the working directory provided, annd runs the 111320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccibisect scrip there. 122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)""" 132a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 142a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import optparse 152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import os 165c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liuimport platform 172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import subprocess 182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import sys 19868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)import traceback 202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 211320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccifrom auto_bisect import bisect_perf_regression 22116680a4aac90f2aa7413d9095a592090648e557Ben Murdochfrom auto_bisect import bisect_utils 235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)from auto_bisect import math_utils 24116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 25868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)CROS_BOARD_ENV = 'BISECT_CROS_BOARD' 26868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)CROS_IP_ENV = 'BISECT_CROS_IP' 272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 281320f92c476a1ad9d19dba2a48c72b75566198e9Primiano TucciSCRIPT_DIR = os.path.dirname(__file__) 291320f92c476a1ad9d19dba2a48c72b75566198e9Primiano TucciSRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir) 301320f92c476a1ad9d19dba2a48c72b75566198e9Primiano TucciBISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg') 311320f92c476a1ad9d19dba2a48c72b75566198e9Primiano TucciRUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg') 321320f92c476a1ad9d19dba2a48c72b75566198e9Primiano TucciWEBKIT_RUN_TEST_CONFIG_PATH = os.path.join( 331320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg') 341320f92c476a1ad9d19dba2a48c72b75566198e9Primiano TucciBISECT_SCRIPT_DIR = os.path.join(SCRIPT_DIR, 'auto_bisect') 351320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 368bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 378bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)class Goma(object): 388bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 398bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) def __init__(self, path_to_goma): 408bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) self._abs_path_to_goma = None 418bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) self._abs_path_to_goma_file = None 425f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if not path_to_goma: 435f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return 445f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self._abs_path_to_goma = os.path.abspath(path_to_goma) 455f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) filename = 'goma_ctl.bat' if os.name == 'nt' else 'goma_ctl.sh' 465f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) self._abs_path_to_goma_file = os.path.join(self._abs_path_to_goma, filename) 478bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 488bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) def __enter__(self): 495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if self._HasGomaPath(): 508bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) self._SetupAndStart() 518bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) return self 528bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 538bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) def __exit__(self, *_): 545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if self._HasGomaPath(): 558bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) self._Stop() 568bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 575f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) def _HasGomaPath(self): 588bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) return bool(self._abs_path_to_goma) 598bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 608bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) def _SetupEnvVars(self): 618bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if os.name == 'nt': 628bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') + 638bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) ' cl.exe') 648bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') + 658bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) ' cl.exe') 668bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) else: 678bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma, 688bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) os.environ['PATH']]) 698bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 708bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) def _SetupAndStart(self): 715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Sets up goma and launches it. 728bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 738bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) Args: 748bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) path_to_goma: Path to goma directory. 758bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 768bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) Returns: 778bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) True if successful.""" 788bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) self._SetupEnvVars() 798bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 808bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) # Sometimes goma is lingering around if something went bad on a previous 818bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) # run. Stop it before starting a new process. Can ignore the return code 828bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) # since it will return an error if it wasn't running. 838bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) self._Stop() 848bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 858bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if subprocess.call([self._abs_path_to_goma_file, 'start']): 865f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) raise RuntimeError('Goma failed to start.') 878bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 888bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) def _Stop(self): 898bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) subprocess.call([self._abs_path_to_goma_file, 'stop']) 908bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 918bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 925f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def _LoadConfigFile(config_file_path): 938bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) """Attempts to load the specified config file as a module 942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) and grab the global config dict. 952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 962a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) Args: 975f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) config_file_path: Path to the config file. 982a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 992a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) Returns: 1005f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) If successful, returns the config dict loaded from the file. If no 1015f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) such dictionary could be loaded, returns the empty dictionary. 1022a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """ 1032a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) try: 1042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) local_vars = {} 1055f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) execfile(config_file_path, local_vars) 1062a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return local_vars['config'] 1075f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) except Exception: 108868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) print 109868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) traceback.print_exc() 110868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) print 1111e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) return {} 1122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1132a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 114a02191e04bc25c4935f804f2c080ae28663d096dBen Murdochdef _ValidateConfigFile(config_contents, valid_parameters): 115a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch """Validates the config file contents, checking whether all values are 116a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch non-empty. 117a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 118a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch Args: 1195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) config_contents: A config dictionary. 120a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch valid_parameters: A list of parameters to check for. 121a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 122a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch Returns: 123a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch True if valid. 124a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch """ 1255f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for parameter in valid_parameters: 1265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if parameter not in config_contents: 1275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return False 1285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) value = config_contents[parameter] 1295f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if not value or type(value) is not str: 1305f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return False 1315f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return True 132a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 133a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 134a02191e04bc25c4935f804f2c080ae28663d096dBen Murdochdef _ValidatePerfConfigFile(config_contents): 1355f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Validates the perf config file contents. 1365f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1375f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) This is used when we're doing a perf try job, rather than a bisect. 1385f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) The config file is called run-perf-test.cfg by default. 139a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 140a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch The parameters checked are the required parameters; any additional optional 141a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch parameters won't be checked and validation will still pass. 142a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 143a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch Args: 1445f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) config_contents: A config dictionary. 145a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 146a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch Returns: 147a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch True if valid. 148a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch """ 1495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) valid_parameters = [ 1505f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'command', 1515f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'repeat_count', 1525f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'truncate_percent', 1535f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'max_time_minutes', 1545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ] 155a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch return _ValidateConfigFile(config_contents, valid_parameters) 156a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 157a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 158a02191e04bc25c4935f804f2c080ae28663d096dBen Murdochdef _ValidateBisectConfigFile(config_contents): 1595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Validates the bisect config file contents. 1605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1615f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) The parameters checked are the required parameters; any additional optional 1625f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) parameters won't be checked and validation will still pass. 163a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 164a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch Args: 1655f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) config_contents: A config dictionary. 166a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 167a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch Returns: 168a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch True if valid. 169a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch """ 1705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) valid_params = [ 1715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'command', 1725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'good_revision', 1735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'bad_revision', 1745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'metric', 1755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'repeat_count', 1765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'truncate_percent', 1775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'max_time_minutes', 1785f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ] 179a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch return _ValidateConfigFile(config_contents, valid_params) 180a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 181a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 1828bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)def _OutputFailedResults(text_to_print): 1838bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepStart('Results - Failed') 1848bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) print 1858bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) print text_to_print 1868bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) print 1878bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepClosed() 1888bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 1898bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 1908bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)def _CreateBisectOptionsFromConfig(config): 191116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch print config['command'] 1928bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts_dict = {} 1938bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts_dict['command'] = config['command'] 1946e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) opts_dict['metric'] = config.get('metric') 1958bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 1968bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if config['repeat_count']: 1978bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts_dict['repeat_test_count'] = int(config['repeat_count']) 1988bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 1998bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if config['truncate_percent']: 2008bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts_dict['truncate_percent'] = int(config['truncate_percent']) 2018bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2028bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if config['max_time_minutes']: 2038bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts_dict['max_time_minutes'] = int(config['max_time_minutes']) 2048bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2058bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if config.has_key('use_goma'): 2068bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts_dict['use_goma'] = config['use_goma'] 207116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if config.has_key('goma_dir'): 208116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch opts_dict['goma_dir'] = config['goma_dir'] 2098bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2108bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts_dict['build_preference'] = 'ninja' 2118bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts_dict['output_buildbot_annotations'] = True 2128bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2138bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if '--browser=cros' in config['command']: 2148bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts_dict['target_platform'] = 'cros' 2158bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2168bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]: 2178bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV] 2188bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV] 2198bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) else: 2201320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or' 2218bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 'BISECT_CROS_BOARD undefined.') 2228bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) elif 'android' in config['command']: 223116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if 'android-chrome-shell' in config['command']: 224116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch opts_dict['target_platform'] = 'android' 225116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch elif 'android-chrome' in config['command']: 226a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) opts_dict['target_platform'] = 'android-chrome' 227a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) else: 228a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) opts_dict['target_platform'] = 'android' 2298bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2301320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci return bisect_perf_regression.BisectOptions.FromDict(opts_dict) 2318bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2328bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2331320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccidef _RunPerformanceTest(config): 2345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Runs a performance test with and without the current patch. 2355f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2365f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Args: 2375f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) config: Contents of the config file, a dictionary. 2385f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2395f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Attempts to build and run the current revision with and without the 2405f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) current patch, with the parameters passed in. 2415f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 2425f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Bisect script expects to be run from the src directory 2431320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci os.chdir(SRC_DIR) 2448bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2458bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepStart('Building With Patch') 2468bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2478bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts = _CreateBisectOptionsFromConfig(config) 2481320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci b = bisect_perf_regression.BisectPerformanceMetrics(None, opts) 2498bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2508bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if bisect_utils.RunGClient(['runhooks']): 2518bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) raise RuntimeError('Failed to run gclient runhooks') 2528bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2538bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if not b.BuildCurrentRevision('chromium'): 2548bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) raise RuntimeError('Patched version failed to build.') 2558bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2568bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepClosed() 2578bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepStart('Running With Patch') 2588bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2598bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) results_with_patch = b.RunPerformanceTestAndParseResults( 2608bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts.command, opts.metric, reset_on_first_run=True, results_label='Patch') 2618bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2628bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if results_with_patch[1]: 2638bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) raise RuntimeError('Patched version failed to run performance test.') 2648bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2658bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepClosed() 2668bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2678bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepStart('Reverting Patch') 2686e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) # TODO: When this is re-written to recipes, this should use bot_update's 2696e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) # revert mechanism to fully revert the client. But for now, since we know that 2701320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # the perf try bot currently only supports src/ and src/third_party/WebKit, we 2716e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) # simply reset those two directories. 2726e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) bisect_utils.CheckRunGit(['reset', '--hard']) 2736e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) bisect_utils.CheckRunGit(['reset', '--hard'], 2746e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) os.path.join('third_party', 'WebKit')) 2758bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepClosed() 2768bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2778bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepStart('Building Without Patch') 2788bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2798bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if bisect_utils.RunGClient(['runhooks']): 2808bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) raise RuntimeError('Failed to run gclient runhooks') 2818bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2828bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if not b.BuildCurrentRevision('chromium'): 2838bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) raise RuntimeError('Unpatched version failed to build.') 2848bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2858bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepClosed() 2868bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepStart('Running Without Patch') 2878bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2888bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) results_without_patch = b.RunPerformanceTestAndParseResults( 2898bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) opts.command, opts.metric, upload_on_last_run=True, results_label='ToT') 2908bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2918bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if results_without_patch[1]: 2928bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) raise RuntimeError('Unpatched version failed to run performance test.') 2938bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2948bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) # Find the link to the cloud stored results file. 2958bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) output = results_without_patch[2] 2968bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) cloud_file_link = [t for t in output.splitlines() 2978bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if 'storage.googleapis.com/chromium-telemetry/html-results/' in t] 2988bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if cloud_file_link: 2991e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) # What we're getting here is basically "View online at http://..." so parse 3001320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # out just the URL portion. 3011e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) cloud_file_link = cloud_file_link[0] 3021e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) cloud_file_link = [t for t in cloud_file_link.split(' ') 3031e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) if 'storage.googleapis.com/chromium-telemetry/html-results/' in t] 3041320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci assert cloud_file_link, 'Couldn\'t parse URL from output.' 3058bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) cloud_file_link = cloud_file_link[0] 3068bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) else: 3078bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) cloud_file_link = '' 3088bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 3098bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) # Calculate the % difference in the means of the 2 runs. 3106e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) percent_diff_in_means = None 3116e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) std_err = None 3126e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) if (results_with_patch[0].has_key('mean') and 3136e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) results_with_patch[0].has_key('values')): 3146e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) percent_diff_in_means = (results_with_patch[0]['mean'] / 3156e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0 3166e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) std_err = math_utils.PooledStandardError( 3176e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) [results_with_patch[0]['values'], results_without_patch[0]['values']]) 3188bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 3198bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepClosed() 3206e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) if percent_diff_in_means is not None and std_err is not None: 3216e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' % 3226e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) (percent_diff_in_means, std_err)) 3236e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '), 3246e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) 'Std. Error'.center(20, ' ')) 3256e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) print ' %s %s %s' % ('Patch'.center(10, ' '), 3266e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) ('%.02f' % results_with_patch[0]['mean']).center(20, ' '), 3276e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) ('%.02f' % results_with_patch[0]['std_err']).center(20, ' ')) 3286e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) print ' %s %s %s' % ('No Patch'.center(10, ' '), 3296e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) ('%.02f' % results_without_patch[0]['mean']).center(20, ' '), 3306e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) ('%.02f' % results_without_patch[0]['std_err']).center(20, ' ')) 3316e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) if cloud_file_link: 3326e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) 3336e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) bisect_utils.OutputAnnotationStepClosed() 3346e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) elif cloud_file_link: 3358bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) 3368bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 3378bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 3381320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccidef _SetupAndRunPerformanceTest(config, path_to_goma): 3398bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) """Attempts to build and run the current revision with and without the 3408bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) current patch, with the parameters passed in. 3418bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 3428bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) Args: 3438bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) config: The config read from run-perf-test.cfg. 3448bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) path_to_goma: Path to goma directory. 3458bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 3468bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) Returns: 3471320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci An exit code: 0 on success, otherwise 1. 3488bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) """ 3491320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci if platform.release() == 'XP': 3501320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci print 'Windows XP is not supported for perf try jobs because it lacks ' 3511320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci print 'goma support. Please refer to crbug.com/330900.' 3521320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci return 1 3538bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) try: 3545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) with Goma(path_to_goma) as _: 3558bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) config['use_goma'] = bool(path_to_goma) 3561320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci if config['use_goma']: 3571320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci config['goma_dir'] = os.path.abspath(path_to_goma) 3581320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci _RunPerformanceTest(config) 3598bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) return 0 3608bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) except RuntimeError, e: 3618bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bisect_utils.OutputAnnotationStepClosed() 3628bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) _OutputFailedResults('Error: %s' % e.message) 3638bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) return 1 3648bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 3658bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 3665f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def _RunBisectionScript( 3671320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci config, working_directory, path_to_goma, path_to_extra_src, dry_run): 3681320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci """Attempts to execute the bisect script with the given parameters. 3692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3702a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) Args: 3712a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) config: A dict containing the parameters to pass to the script. 3721320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci working_directory: A working directory to provide to the bisect script, 3731320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci where it will store it's own copy of the depot. 3742a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) path_to_goma: Path to goma directory. 375a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) path_to_extra_src: Path to extra source file. 3761e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) dry_run: Do a dry run, skipping sync, build, and performance testing steps. 3772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) Returns: 3795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) An exit status code: 0 on success, otherwise 1. 3802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """ 3815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _PrintConfigStep(config) 3822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3831320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci cmd = ['python', os.path.join(BISECT_SCRIPT_DIR, 'bisect_perf_regression.py'), 3842a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '-c', config['command'], 3852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '-g', config['good_revision'], 3862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '-b', config['bad_revision'], 3872a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '-m', config['metric'], 3882a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '--working_directory', working_directory, 3892a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '--output_buildbot_annotations'] 3902a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3916e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) if config.get('metric'): 3926e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) cmd.extend(['-m', config['metric']]) 3936e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) 3942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if config['repeat_count']: 3952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) cmd.extend(['-r', config['repeat_count']]) 3962a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3972a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if config['truncate_percent']: 3982a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) cmd.extend(['-t', config['truncate_percent']]) 3992a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 400c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) if config['max_time_minutes']: 4014e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) cmd.extend(['--max_time_minutes', config['max_time_minutes']]) 402c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 4035c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu if config.has_key('bisect_mode'): 4045c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu cmd.extend(['--bisect_mode', config['bisect_mode']]) 4055c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 406868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) cmd.extend(['--build_preference', 'ninja']) 407868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 408868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if '--browser=cros' in config['command']: 409868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) cmd.extend(['--target_platform', 'cros']) 410868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 411868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]: 412868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]]) 413868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]]) 414868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) else: 4155f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) print ('Error: Cros build selected, but BISECT_CROS_IP or' 4165f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'BISECT_CROS_BOARD undefined.\n') 417868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return 1 418868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 4194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if 'android' in config['command']: 420116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if 'android-chrome-shell' in config['command']: 421116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch cmd.extend(['--target_platform', 'android']) 422116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch elif 'android-chrome' in config['command']: 423a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) cmd.extend(['--target_platform', 'android-chrome']) 424a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) else: 425a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) cmd.extend(['--target_platform', 'android']) 4262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if path_to_goma: 4285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # For Windows XP platforms, goma service is not supported. 4295c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu # Moreover we don't compile chrome when gs_bucket flag is set instead 4305f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # use builds archives, therefore ignore goma service for Windows XP. 4315f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # See http://crbug.com/330900. 4325c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu if config.get('gs_bucket') and platform.release() == 'XP': 4335c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu print ('Goma doesn\'t have a win32 binary, therefore it is not supported ' 4345c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 'on Windows XP platform. Please refer to crbug.com/330900.') 4355c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu path_to_goma = None 4362a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) cmd.append('--use_goma') 4372a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 438a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) if path_to_extra_src: 439a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) cmd.extend(['--extra_src', path_to_extra_src]) 440a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 441a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch # These flags are used to download build archives from cloud storage if 4421320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # available, otherwise will post a try_job_http request to build it on the 4431320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # try server. 444a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch if config.get('gs_bucket'): 445a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch if config.get('builder_host') and config.get('builder_port'): 446a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch cmd.extend(['--gs_bucket', config['gs_bucket'], 447a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch '--builder_host', config['builder_host'], 448a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch '--builder_port', config['builder_port'] 449a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch ]) 450a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch else: 451a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch print ('Error: Specified gs_bucket, but missing builder_host or ' 452a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 'builder_port information in config.') 453a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch return 1 454a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 4551e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) if dry_run: 4561e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) cmd.extend(['--debug_ignore_build', '--debug_ignore_sync', 4571e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) '--debug_ignore_perf_test']) 4582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) cmd = [str(c) for c in cmd] 4592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) with Goma(path_to_goma) as _: 4618bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) return_code = subprocess.call(cmd) 4622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if return_code: 4641320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci print ('Error: bisect_perf_regression.py returned with error %d\n' 4655f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) % return_code) 4662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4672a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return return_code 4682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def _PrintConfigStep(config): 4715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Prints out the given config, along with Buildbot annotations.""" 4725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) bisect_utils.OutputAnnotationStepStart('Config') 4735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) print 4745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for k, v in config.iteritems(): 4755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) print ' %s : %s' % (k, v) 4765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) print 4775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) bisect_utils.OutputAnnotationStepClosed() 4782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4805f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def _OptionParser(): 4815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Returns the options parser for run-bisect-perf-regression.py.""" 4822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) usage = ('%prog [options] [-- chromium-options]\n' 4831320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'Used by a try bot to run the bisection script using the parameters' 4841320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci ' provided in the auto_bisect/bisect.cfg file.') 4852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parser = optparse.OptionParser(usage=usage) 4862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parser.add_option('-w', '--working_directory', 4872a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) type='str', 4882a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) help='A working directory to supply to the bisection ' 4892a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 'script, which will use it as the location to checkout ' 4902a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 'a copy of the chromium depot.') 4912a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parser.add_option('-p', '--path_to_goma', 4922a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) type='str', 4932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) help='Path to goma directory. If this is supplied, goma ' 4942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 'builds will be enabled.') 495a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch parser.add_option('--path_to_config', 496a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch type='str', 497a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch help='Path to the config file to use. If this is supplied, ' 498a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 'the bisect script will use this to override the default ' 499a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 'config file path. The script will attempt to load it ' 500a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 'as a bisect config first, then a perf config.') 501a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) parser.add_option('--extra_src', 502a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) type='str', 503a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) help='Path to extra source file. If this is supplied, ' 504a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 'bisect script will use this to override default behavior.') 5051e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) parser.add_option('--dry_run', 5061e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) action="store_true", 5071e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) help='The script will perform the full bisect, but ' 5081e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 'without syncing, building, or running the performance ' 5091e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 'tests.') 5105f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return parser 5112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 512a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch 5135f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def main(): 5145f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Entry point for run-bisect-perf-regression.py. 5152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 5165f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Reads the config file, and then tries to either bisect a regression or 5175f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) just run a performance test, depending on the particular config parameters 5185f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) specified in the config file. 5195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 5205f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) parser = _OptionParser() 5215f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) opts, _ = parser.parse_args() 5225f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Use the default config file path unless one was specified. 5241320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci config_path = BISECT_CONFIG_PATH 5255f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if opts.path_to_config: 5265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) config_path = opts.path_to_config 5275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) config = _LoadConfigFile(config_path) 5285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5295f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Check if the config is valid for running bisect job. 530a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch config_is_valid = _ValidateBisectConfigFile(config) 5318bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 532a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch if config and config_is_valid: 5338bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if not opts.working_directory: 5345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) print 'Error: missing required parameter: --working_directory\n' 5358bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) parser.print_help() 5368bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) return 1 5378bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 5385f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return _RunBisectionScript( 5391320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci config, opts.working_directory, opts.path_to_goma, opts.extra_src, 5401320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci opts.dry_run) 5415f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5425f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # If it wasn't valid for running a bisect, then maybe the user wanted 5435f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # to run a perf test instead of a bisect job. Try reading any possible 5445f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # perf test config files. 5451320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci perf_cfg_files = [RUN_TEST_CONFIG_PATH, WEBKIT_RUN_TEST_CONFIG_PATH] 5465f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for current_perf_cfg_file in perf_cfg_files: 5475f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if opts.path_to_config: 5485f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) path_to_perf_cfg = opts.path_to_config 5495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 5505f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) path_to_perf_cfg = os.path.join( 5515f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) os.path.abspath(os.path.dirname(sys.argv[0])), 5525f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) current_perf_cfg_file) 5535f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) config = _LoadConfigFile(path_to_perf_cfg) 5555f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) config_is_valid = _ValidatePerfConfigFile(config) 5565f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5575f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if config and config_is_valid: 5581320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci return _SetupAndRunPerformanceTest(config, opts.path_to_goma) 5595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) print ('Error: Could not load config file. Double check your changes to ' 5611320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n') 5625f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return 1 5632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 5642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 5652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)if __name__ == '__main__': 5662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sys.exit(main()) 567