suite_runner.py revision 98a53692fb946a8eac46e3e82257f540d1350c18
1#!/usr/bin/python 2 3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7import os 8import time 9import shlex 10 11from utils import command_executer 12 13TEST_THAT_PATH = '/usr/bin/test_that' 14CHROME_MOUNT_DIR = '/tmp/chrome_root' 15 16def GetProfilerArgs (benchmark, profiler_args): 17 # Remove "--" from in front of profiler args. 18 args_list = shlex.split(profiler_args) 19 new_list = [] 20 for arg in args_list: 21 if arg[0:2] == '--': 22 arg = arg[2:] 23 new_list.append(arg) 24 args_list = new_list 25 26 # Remove "perf_options=" from middle of profiler args. 27 new_list = [] 28 for arg in args_list: 29 idx = arg.find("perf_options=") 30 if idx != -1: 31 prefix = arg[0:idx] 32 suffix = arg[idx + len("perf_options=") + 1 : -1] 33 new_arg = prefix + "'" + suffix + "'" 34 new_list.append(new_arg) 35 else: 36 new_list.append(arg) 37 args_list = new_list 38 39 return " ".join(args_list) 40 41 42class SuiteRunner(object): 43 """ This defines the interface from crosperf to test script. 44 """ 45 46 def __init__(self, logger_to_use=None, log_level="verbose"): 47 self._logger = logger_to_use 48 self.log_level = log_level 49 self._ce = command_executer.GetCommandExecuter(self._logger, 50 log_level=self.log_level) 51 self._ct = command_executer.CommandTerminator() 52 53 def Run(self, machine, label, benchmark, test_args, profiler_args): 54 self.PinGovernorExecutionFrequencies(machine, label.chromeos_root) 55 if benchmark.suite == "telemetry": 56 return self.Telemetry_Run(machine, label, benchmark) 57 elif benchmark.suite == "telemetry_Crosperf": 58 return self.Telemetry_Crosperf_Run(machine, label, benchmark, 59 test_args, profiler_args) 60 else: 61 return self.Test_That_Run(machine, label, benchmark, test_args, 62 profiler_args) 63 64 def GetHighestStaticFrequency(self, machine_name, chromeos_root): 65 """ Gets the highest static frequency for the specified machine 66 """ 67 get_avail_freqs = ("cd /sys/devices/system/cpu/cpu0/cpufreq/; " 68 "if [[ -e scaling_available_frequencies ]]; then " 69 " cat scaling_available_frequencies; " 70 "else " 71 " cat scaling_max_freq ; " 72 "fi") 73 ret, freqs_str, _ = self._ce.CrosRunCommand( 74 get_avail_freqs, return_output=True, machine=machine_name, 75 chromeos_root=chromeos_root) 76 self._logger.LogFatalIf(ret, "Could not get available frequencies " 77 "from machine: %s" % machine_name) 78 freqs = freqs_str.split() 79 ## When there is no scaling_available_frequencies file, 80 ## we have only 1 choice. 81 if len(freqs) == 1: 82 return freqs[0] 83 # The dynamic frequency ends with a "1000". So, ignore it if found. 84 if freqs[0].endswith("1000"): 85 return freqs[1] 86 else: 87 return freqs[0] 88 89 def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root): 90 """ Set min and max frequencies to max static frequency 91 """ 92 highest_freq = self.GetHighestStaticFrequency(machine_name, chromeos_root) 93 BASH_FOR = "for f in {list}; do {body}; done" 94 CPUFREQ_DIRS = "/sys/devices/system/cpu/cpu*/cpufreq/" 95 change_max_freq = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_max_freq", 96 body="echo %s > $f" % highest_freq) 97 change_min_freq = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_min_freq", 98 body="echo %s > $f" % highest_freq) 99 change_perf_gov = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_governor", 100 body="echo performance > $f") 101 if self.log_level == "average": 102 self._logger.LogOutput("Pinning governor execution frequencies for %s" 103 % machine_name) 104 ret = self._ce.CrosRunCommand(" && ".join(("set -e ", 105 change_max_freq, 106 change_min_freq, 107 change_perf_gov)), 108 machine=machine_name, 109 chromeos_root=chromeos_root) 110 self._logger.LogFatalIf(ret, "Could not pin frequencies on machine: %s" 111 % machine_name) 112 113 def RebootMachine(self, machine_name, chromeos_root): 114 command = "reboot && exit" 115 self._ce.CrosRunCommand(command, machine=machine_name, 116 chromeos_root=chromeos_root) 117 time.sleep(60) 118 # Whenever we reboot the machine, we need to restore the governor settings. 119 self.PinGovernorExecutionFrequencies(machine_name, chromeos_root) 120 121 def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args): 122 """Run the test_that test..""" 123 options = "" 124 if label.board: 125 options += " --board=%s" % label.board 126 if test_args: 127 options += " %s" % test_args 128 if profiler_args: 129 self._logger.LogError("test_that does not support profiler.") 130 command = "rm -rf /usr/local/autotest/results/*" 131 self._ce.CrosRunCommand(command, machine=machine, username="root", 132 chromeos_root=label.chromeos_root) 133 134 # We do this because some tests leave the machine in weird states. 135 # Rebooting between iterations has proven to help with this. 136 self.RebootMachine(machine, label.chromeos_root) 137 138 command = ("%s %s %s %s" % 139 (TEST_THAT_PATH, options, machine, benchmark.test_name)) 140 if self.log_level != "verbose": 141 self._logger.LogOutput("Running test.") 142 self._logger.LogOutput("CMD: %s" % command) 143 return self._ce.ChrootRunCommand(label.chromeos_root, 144 command, 145 True, 146 self._ct) 147 148 149 def Telemetry_Crosperf_Run (self, machine, label, benchmark, test_args, 150 profiler_args): 151 if not os.path.isdir(label.chrome_src): 152 self._logger.LogFatal("Cannot find chrome src dir to" 153 " run telemetry: %s" % label.chrome_src) 154 155 profiler_args = GetProfilerArgs (benchmark, profiler_args) 156 chrome_root_options = "" 157 158 # If chrome_src is outside the chroot, mount it when entering the 159 # chroot. 160 if label.chrome_src.find(label.chromeos_root) == -1: 161 chrome_root_options = (" --chrome_root={0} --chrome_root_mount={1} " 162 " FEATURES=\"-usersandbox\" " 163 "CHROME_ROOT={2}".format(label.chrome_src, 164 CHROME_MOUNT_DIR, 165 CHROME_MOUNT_DIR)) 166 167 args_string = "" 168 if test_args: 169 # Strip double quotes off args (so we can wrap them in single 170 # quotes, to pass through to Telemetry). 171 if test_args[0] == '"' and test_args[-1] == '"': 172 test_args = test_args[1:-1] 173 args_string = "test_args='%s'" % test_args 174 cmd = ('{0} --board={1} --args="{2} test={3} ' 175 '{4}" {5} telemetry_Crosperf'.format(TEST_THAT_PATH, 176 label.board, 177 args_string, 178 benchmark.test_name, 179 profiler_args, 180 machine)) 181 if self.log_level != "verbose": 182 self._logger.LogOutput("Running test.") 183 self._logger.LogOutput("CMD: %s" % cmd) 184 return self._ce.ChrootRunCommand (label.chromeos_root, 185 cmd, 186 return_output=True, 187 command_terminator=self._ct, 188 cros_sdk_options=chrome_root_options) 189 190 191 def Telemetry_Run(self, machine, label, benchmark): 192 telemetry_run_path = "" 193 if not os.path.isdir(label.chrome_src): 194 self._logger.LogFatal("Cannot find chrome src dir to" 195 " run telemetry.") 196 else: 197 telemetry_run_path = os.path.join(label.chrome_src, "src/tools/perf") 198 if not os.path.exists(telemetry_run_path): 199 self._logger.LogFatal("Cannot find %s directory." % telemetry_run_path) 200 201 rsa_key = os.path.join(label.chromeos_root, 202 "src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa") 203 204 cmd = ("cd {0} && " 205 "./run_measurement " 206 "--browser=cros-chrome " 207 "--output-format=csv " 208 "--remote={1} " 209 "--identity {2} " 210 "{3} {4}".format(telemetry_run_path, machine, 211 rsa_key, 212 benchmark.test_name, 213 benchmark.test_args)) 214 if self.log_level != "verbose": 215 self._logger.LogOutput("Running test.") 216 self._logger.LogOutput("CMD: %s" % cmd) 217 return self._ce.RunCommand(cmd, return_output=True, 218 print_to_console=False) 219 220 def Terminate(self): 221 self._ct.Terminate() 222 223 224class MockSuiteRunner(object): 225 def __init__(self): 226 self._true = True 227 228 def Run(self, *_args): 229 if self._true: 230 return ["", "", 0] 231 else: 232 return ["", "", 0] 233