suite_runner.py revision 278fc7437c8a31554903161eadfac2550507a47f
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import os 6import time 7import shlex 8 9from utils import command_executer 10import test_flag 11 12TEST_THAT_PATH = '/usr/bin/test_that' 13CHROME_MOUNT_DIR = '/tmp/chrome_root' 14 15 16def GetProfilerArgs(profiler_args): 17 # Remove "--" from in front of profiler args. 18 args_list = shlex.split(profiler_args) 19 new_list = [] 20 for arg in args_list: 21 if arg[0:2] == '--': 22 arg = arg[2:] 23 new_list.append(arg) 24 args_list = new_list 25 26 # Remove "perf_options=" from middle of profiler args. 27 new_list = [] 28 for arg in args_list: 29 idx = arg.find('perf_options=') 30 if idx != -1: 31 prefix = arg[0:idx] 32 suffix = arg[idx + len('perf_options=') + 1:-1] 33 new_arg = prefix + "'" + suffix + "'" 34 new_list.append(new_arg) 35 else: 36 new_list.append(arg) 37 args_list = new_list 38 39 return ' '.join(args_list) 40 41 42class SuiteRunner(object): 43 """ This defines the interface from crosperf to test script. 44 """ 45 46 def __init__(self, 47 logger_to_use=None, 48 log_level='verbose', 49 cmd_exec=None, 50 cmd_term=None): 51 self._logger = logger_to_use 52 self.log_level = log_level 53 self._ce = cmd_exec or command_executer.GetCommandExecuter( 54 self._logger, 55 log_level=self.log_level) 56 self._ct = cmd_term or command_executer.CommandTerminator() 57 58 def Run(self, machine, label, benchmark, test_args, profiler_args): 59 for i in range(0, benchmark.retries + 1): 60 self.PinGovernorExecutionFrequencies(machine, label.chromeos_root) 61 if benchmark.suite == 'telemetry': 62 ret_tup = self.Telemetry_Run(machine, label, benchmark, profiler_args) 63 elif benchmark.suite == 'telemetry_Crosperf': 64 ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark, 65 test_args, profiler_args) 66 else: 67 ret_tup = self.Test_That_Run(machine, label, benchmark, test_args, 68 profiler_args) 69 if ret_tup[0] != 0: 70 self._logger.LogOutput('benchmark %s failed. Retries left: %s' % 71 (benchmark.name, benchmark.retries - i)) 72 elif i > 0: 73 self._logger.LogOutput('benchmark %s succeded after %s retries' % 74 (benchmark.name, i)) 75 break 76 else: 77 self._logger.LogOutput('benchmark %s succeded on first try' % 78 benchmark.name) 79 break 80 return ret_tup 81 82 def GetHighestStaticFrequency(self, machine_name, chromeos_root): 83 """ Gets the highest static frequency for the specified machine 84 """ 85 get_avail_freqs = ('cd /sys/devices/system/cpu/cpu0/cpufreq/; ' 86 'if [[ -e scaling_available_frequencies ]]; then ' 87 ' cat scaling_available_frequencies; ' 88 'else ' 89 ' cat scaling_max_freq ; ' 90 'fi') 91 ret, freqs_str, _ = self._ce.CrosRunCommandWOutput( 92 get_avail_freqs, 93 machine=machine_name, 94 chromeos_root=chromeos_root) 95 self._logger.LogFatalIf(ret, 'Could not get available frequencies ' 96 'from machine: %s' % machine_name) 97 freqs = freqs_str.split() 98 # We need to make sure that the frequencies are sorted in decreasing 99 # order 100 freqs.sort(key=int, reverse=True) 101 102 ## When there is no scaling_available_frequencies file, 103 ## we have only 1 choice. 104 if len(freqs) == 1: 105 return freqs[0] 106 # The dynamic frequency ends with a "1000". So, ignore it if found. 107 if freqs[0].endswith('1000'): 108 return freqs[1] 109 else: 110 return freqs[0] 111 112 def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root): 113 """ Set min and max frequencies to max static frequency 114 """ 115 highest_freq = self.GetHighestStaticFrequency(machine_name, chromeos_root) 116 BASH_FOR = 'for f in {list}; do {body}; done' 117 CPUFREQ_DIRS = '/sys/devices/system/cpu/cpu*/cpufreq/' 118 change_max_freq = BASH_FOR.format(list=CPUFREQ_DIRS + 'scaling_max_freq', 119 body='echo %s > $f' % highest_freq) 120 change_min_freq = BASH_FOR.format(list=CPUFREQ_DIRS + 'scaling_min_freq', 121 body='echo %s > $f' % highest_freq) 122 change_perf_gov = BASH_FOR.format(list=CPUFREQ_DIRS + 'scaling_governor', 123 body='echo performance > $f') 124 if self.log_level == 'average': 125 self._logger.LogOutput('Pinning governor execution frequencies for %s' % 126 machine_name) 127 ret = self._ce.CrosRunCommand(' && '.join(( 128 'set -e ', change_max_freq, change_min_freq, change_perf_gov)), 129 machine=machine_name, 130 chromeos_root=chromeos_root) 131 self._logger.LogFatalIf(ret, 'Could not pin frequencies on machine: %s' % 132 machine_name) 133 134 def RebootMachine(self, machine_name, chromeos_root): 135 command = 'reboot && exit' 136 self._ce.CrosRunCommand(command, 137 machine=machine_name, 138 chromeos_root=chromeos_root) 139 time.sleep(60) 140 # Whenever we reboot the machine, we need to restore the governor settings. 141 self.PinGovernorExecutionFrequencies(machine_name, chromeos_root) 142 143 def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args): 144 """Run the test_that test..""" 145 options = '' 146 if label.board: 147 options += ' --board=%s' % label.board 148 if test_args: 149 options += ' %s' % test_args 150 if profiler_args: 151 self._logger.LogFatal('test_that does not support profiler.') 152 command = 'rm -rf /usr/local/autotest/results/*' 153 self._ce.CrosRunCommand(command, 154 machine=machine, 155 chromeos_root=label.chromeos_root) 156 157 # We do this because some tests leave the machine in weird states. 158 # Rebooting between iterations has proven to help with this. 159 self.RebootMachine(machine, label.chromeos_root) 160 161 command = ( 162 ('%s --autotest_dir ~/trunk/src/third_party/autotest/files --fast ' 163 '%s %s %s') % (TEST_THAT_PATH, options, machine, benchmark.test_name)) 164 if self.log_level != 'verbose': 165 self._logger.LogOutput('Running test.') 166 self._logger.LogOutput('CMD: %s' % command) 167 # Use --no-ns-pid so that cros_sdk does not create a different 168 # process namespace and we can kill process created easily by 169 # their process group. 170 return self._ce.ChrootRunCommandWOutput(label.chromeos_root, 171 command, 172 command_terminator=self._ct, 173 cros_sdk_options='--no-ns-pid') 174 175 def RemoveTelemetryTempFile(self, machine, chromeos_root): 176 filename = 'telemetry@%s' % machine 177 fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename) 178 if os.path.exists(fullname): 179 os.remove(fullname) 180 181 def Telemetry_Crosperf_Run(self, machine, label, benchmark, test_args, 182 profiler_args): 183 if not os.path.isdir(label.chrome_src): 184 self._logger.LogFatal('Cannot find chrome src dir to' 185 ' run telemetry: %s' % label.chrome_src) 186 187 # Check for and remove temporary file that may have been left by 188 # previous telemetry runs (and which might prevent this run from 189 # working). 190 self.RemoveTelemetryTempFile(machine, label.chromeos_root) 191 192 # For telemetry runs, we can use the autotest copy from the source 193 # location. No need to have one under /build/<board>. 194 autotest_dir_arg = '--autotest_dir ~/trunk/src/third_party/autotest/files' 195 196 profiler_args = GetProfilerArgs(profiler_args) 197 fast_arg = '' 198 if not profiler_args: 199 # --fast works unless we are doing profiling (autotest limitation). 200 # --fast avoids unnecessary copies of syslogs. 201 fast_arg = '--fast' 202 args_string = '' 203 if test_args: 204 # Strip double quotes off args (so we can wrap them in single 205 # quotes, to pass through to Telemetry). 206 if test_args[0] == '"' and test_args[-1] == '"': 207 test_args = test_args[1:-1] 208 args_string = "test_args='%s'" % test_args 209 210 cmd = ('{} {} {} --board={} --args="{} run_local={} test={} ' 211 '{}" {} telemetry_Crosperf'.format( 212 TEST_THAT_PATH, autotest_dir_arg, fast_arg, label.board, 213 args_string, benchmark.run_local, benchmark.test_name, 214 profiler_args, machine)) 215 216 # Use --no-ns-pid so that cros_sdk does not create a different 217 # process namespace and we can kill process created easily by their 218 # process group. 219 chrome_root_options = ('--no-ns-pid ' 220 '--chrome_root={} --chrome_root_mount={} ' 221 "FEATURES=\"-usersandbox\" " 222 'CHROME_ROOT={}'.format(label.chrome_src, 223 CHROME_MOUNT_DIR, 224 CHROME_MOUNT_DIR)) 225 if self.log_level != 'verbose': 226 self._logger.LogOutput('Running test.') 227 self._logger.LogOutput('CMD: %s' % cmd) 228 return self._ce.ChrootRunCommandWOutput( 229 label.chromeos_root, 230 cmd, 231 command_terminator=self._ct, 232 cros_sdk_options=chrome_root_options) 233 234 def Telemetry_Run(self, machine, label, benchmark, profiler_args): 235 telemetry_run_path = '' 236 if not os.path.isdir(label.chrome_src): 237 self._logger.LogFatal('Cannot find chrome src dir to' ' run telemetry.') 238 else: 239 telemetry_run_path = os.path.join(label.chrome_src, 'src/tools/perf') 240 if not os.path.exists(telemetry_run_path): 241 self._logger.LogFatal('Cannot find %s directory.' % telemetry_run_path) 242 243 if profiler_args: 244 self._logger.LogFatal('Telemetry does not support the perf profiler.') 245 246 # Check for and remove temporary file that may have been left by 247 # previous telemetry runs (and which might prevent this run from 248 # working). 249 if not test_flag.GetTestMode(): 250 self.RemoveTelemetryTempFile(machine, label.chromeos_root) 251 252 rsa_key = os.path.join( 253 label.chromeos_root, 254 'src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa') 255 256 cmd = ('cd {0} && ' 257 './run_measurement ' 258 '--browser=cros-chrome ' 259 '--output-format=csv ' 260 '--remote={1} ' 261 '--identity {2} ' 262 '{3} {4}'.format(telemetry_run_path, machine, rsa_key, 263 benchmark.test_name, benchmark.test_args)) 264 if self.log_level != 'verbose': 265 self._logger.LogOutput('Running test.') 266 self._logger.LogOutput('CMD: %s' % cmd) 267 return self._ce.RunCommandWOutput(cmd, print_to_console=False) 268 269 def Terminate(self): 270 self._ct.Terminate() 271 272 273class MockSuiteRunner(object): 274 275 def __init__(self): 276 self._true = True 277 278 def Run(self, *_args): 279 if self._true: 280 return [0, '', ''] 281 else: 282 return [0, '', ''] 283