suite_runner.py revision 5d7e21d72b17274a81a64a8040e1a30697e1223b 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import os
8import time
9import shlex
10
11from utils import command_executer
12
13TEST_THAT_PATH = '/usr/bin/test_that'
14CHROME_MOUNT_DIR = '/tmp/chrome_root'
15
16def GetProfilerArgs (profiler_args):
17  # Remove "--" from in front of profiler args.
18  args_list = shlex.split(profiler_args)
19  new_list = []
20  for arg in args_list:
21    if arg[0:2] == '--':
22      arg = arg[2:]
23    new_list.append(arg)
24  args_list = new_list
25
26  # Remove "perf_options=" from middle of profiler args.
27  new_list = []
28  for arg in args_list:
29    idx = arg.find("perf_options=")
30    if idx != -1:
31      prefix = arg[0:idx]
32      suffix = arg[idx + len("perf_options=") + 1 : -1]
33      new_arg = prefix + "'" + suffix + "'"
34      new_list.append(new_arg)
35    else:
36      new_list.append(arg)
37  args_list = new_list
38
39  return " ".join(args_list)
40
41
42class SuiteRunner(object):
43  """ This defines the interface from crosperf to test script.
44  """
45
46  def __init__(self, logger_to_use=None, log_level="verbose", cmd_exec=None,
47               cmd_term=None):
48    self._logger = logger_to_use
49    self.log_level = log_level
50    self._ce = cmd_exec or command_executer.GetCommandExecuter(self._logger,
51                                                   log_level=self.log_level)
52    self._ct = cmd_term or command_executer.CommandTerminator()
53
54  def Run(self, machine, label, benchmark, test_args, profiler_args):
55    self.PinGovernorExecutionFrequencies(machine, label.chromeos_root)
56    if benchmark.suite == "telemetry":
57      return self.Telemetry_Run(machine, label, benchmark, profiler_args)
58    elif benchmark.suite == "telemetry_Crosperf":
59      return self.Telemetry_Crosperf_Run(machine, label, benchmark,
60                                         test_args, profiler_args)
61    else:
62      return self.Test_That_Run(machine, label, benchmark, test_args,
63                                profiler_args)
64
65  def GetHighestStaticFrequency(self, machine_name, chromeos_root):
66    """ Gets the highest static frequency for the specified machine
67    """
68    get_avail_freqs = ("cd /sys/devices/system/cpu/cpu0/cpufreq/; "
69                       "if [[ -e scaling_available_frequencies ]]; then "
70                       "  cat scaling_available_frequencies; "
71                       "else "
72                       "  cat scaling_max_freq ; "
73                       "fi")
74    ret, freqs_str, _ = self._ce.CrosRunCommand(
75        get_avail_freqs, return_output=True, machine=machine_name,
76        chromeos_root=chromeos_root)
77    self._logger.LogFatalIf(ret, "Could not get available frequencies "
78                            "from machine: %s" % machine_name)
79    freqs = freqs_str.split()
80    ## When there is no scaling_available_frequencies file,
81    ## we have only 1 choice.
82    if len(freqs) == 1:
83      return freqs[0]
84    # The dynamic frequency ends with a "1000". So, ignore it if found.
85    if freqs[0].endswith("1000"):
86      return freqs[1]
87    else:
88      return freqs[0]
89
90  def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root):
91    """ Set min and max frequencies to max static frequency
92    """
93    highest_freq = self.GetHighestStaticFrequency(machine_name, chromeos_root)
94    BASH_FOR = "for f in {list}; do {body}; done"
95    CPUFREQ_DIRS = "/sys/devices/system/cpu/cpu*/cpufreq/"
96    change_max_freq = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_max_freq",
97                                      body="echo %s > $f" % highest_freq)
98    change_min_freq = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_min_freq",
99                                      body="echo %s > $f" % highest_freq)
100    change_perf_gov = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_governor",
101                                      body="echo performance > $f")
102    if self.log_level == "average":
103      self._logger.LogOutput("Pinning governor execution frequencies for %s"
104                           % machine_name)
105    ret = self._ce.CrosRunCommand(" && ".join(("set -e ",
106                                               change_max_freq,
107                                               change_min_freq,
108                                               change_perf_gov)),
109                                  machine=machine_name,
110                                  chromeos_root=chromeos_root)
111    self._logger.LogFatalIf(ret, "Could not pin frequencies on machine: %s"
112                            % machine_name)
113
114  def RebootMachine(self, machine_name, chromeos_root):
115    command = "reboot && exit"
116    self._ce.CrosRunCommand(command, machine=machine_name,
117                      chromeos_root=chromeos_root)
118    time.sleep(60)
119    # Whenever we reboot the machine, we need to restore the governor settings.
120    self.PinGovernorExecutionFrequencies(machine_name, chromeos_root)
121
122  def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
123    """Run the test_that test.."""
124    options = ""
125    if label.board:
126      options += " --board=%s" % label.board
127    if test_args:
128      options += " %s" % test_args
129    if profiler_args:
130      self._logger.LogFatal("test_that does not support profiler.")
131    command = "rm -rf /usr/local/autotest/results/*"
132    self._ce.CrosRunCommand(command, machine=machine, username="root",
133                            chromeos_root=label.chromeos_root)
134
135    # We do this because some tests leave the machine in weird states.
136    # Rebooting between iterations has proven to help with this.
137    self.RebootMachine(machine, label.chromeos_root)
138
139    command = ("%s %s %s %s" %
140               (TEST_THAT_PATH, options, machine, benchmark.test_name))
141    if self.log_level != "verbose":
142      self._logger.LogOutput("Running test.")
143      self._logger.LogOutput("CMD: %s" % command)
144    return self._ce.ChrootRunCommand(label.chromeos_root,
145                                     command,
146                                     True,
147                                     self._ct)
148
149
150  def Telemetry_Crosperf_Run (self, machine, label, benchmark, test_args,
151                              profiler_args):
152    if not os.path.isdir(label.chrome_src):
153      self._logger.LogFatal("Cannot find chrome src dir to"
154                            " run telemetry: %s" % label.chrome_src)
155
156    profiler_args = GetProfilerArgs (profiler_args)
157    fast_arg = ""
158    if not profiler_args:
159      # --fast works unless we are doing profiling (autotest limitation).
160      # --fast avoids unnecessary copies of syslogs.
161      fast_arg = "--fast"
162    args_string = ""
163    if test_args:
164      # Strip double quotes off args (so we can wrap them in single
165      # quotes, to pass through to Telemetry).
166      if test_args[0] == '"' and test_args[-1] == '"':
167        test_args = test_args[1:-1]
168      args_string = "test_args='%s'" % test_args
169    cmd = ('{} {} --board={} --args="{} test={} '
170           '{}" {} telemetry_Crosperf'.format(TEST_THAT_PATH,
171                                              fast_arg,
172                                              label.board,
173                                              args_string,
174                                              benchmark.test_name,
175                                              profiler_args,
176                                              machine))
177
178    chrome_root_options = ""
179    chrome_root_options = (" --chrome_root={} --chrome_root_mount={} "
180                           " FEATURES=\"-usersandbox\" "
181                           "CHROME_ROOT={}".format(label.chrome_src,
182                                                    CHROME_MOUNT_DIR,
183                                                    CHROME_MOUNT_DIR))
184    if self.log_level != "verbose":
185      self._logger.LogOutput("Running test.")
186      self._logger.LogOutput("CMD: %s" % cmd)
187    return self._ce.ChrootRunCommand (label.chromeos_root,
188                                      cmd,
189                                      return_output=True,
190                                      command_terminator=self._ct,
191                                      cros_sdk_options=chrome_root_options)
192
193
194  def Telemetry_Run(self, machine, label, benchmark, profiler_args):
195    telemetry_run_path = ""
196    if not os.path.isdir(label.chrome_src):
197      self._logger.LogFatal("Cannot find chrome src dir to"
198                            " run telemetry.")
199    else:
200      telemetry_run_path = os.path.join(label.chrome_src, "src/tools/perf")
201      if not os.path.exists(telemetry_run_path):
202        self._logger.LogFatal("Cannot find %s directory." % telemetry_run_path)
203
204    if profiler_args:
205      self._logger.LogFatal("Telemetry does not support the perf profiler.")
206
207    rsa_key = os.path.join(label.chromeos_root,
208        "src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa")
209
210    cmd = ("cd {0} && "
211           "./run_measurement "
212           "--browser=cros-chrome "
213           "--output-format=csv "
214           "--remote={1} "
215           "--identity {2} "
216           "{3} {4}".format(telemetry_run_path, machine,
217                            rsa_key,
218                            benchmark.test_name,
219                            benchmark.test_args))
220    if self.log_level != "verbose":
221      self._logger.LogOutput("Running test.")
222      self._logger.LogOutput("CMD: %s" % cmd)
223    return self._ce.RunCommand(cmd, return_output=True,
224                               print_to_console=False)
225
226  def Terminate(self):
227    self._ct.Terminate()
228
229
230class MockSuiteRunner(object):
231  def __init__(self):
232    self._true = True
233
234  def Run(self, *_args):
235    if self._true:
236      return ["", "", 0]
237    else:
238      return ["", "", 0]
239