suite_runner.py revision 45b53c5fee4c044cebd74dbbbd6119fbd51554e7 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import os
8import time
9import shlex
10
11from utils import command_executer
12
13TEST_THAT_PATH = '/usr/bin/test_that'
14CHROME_MOUNT_DIR = '/tmp/chrome_root'
15
16def GetProfilerArgs (profiler_args):
17  # Remove "--" from in front of profiler args.
18  args_list = shlex.split(profiler_args)
19  new_list = []
20  for arg in args_list:
21    if arg[0:2] == '--':
22      arg = arg[2:]
23    new_list.append(arg)
24  args_list = new_list
25
26  # Remove "perf_options=" from middle of profiler args.
27  new_list = []
28  for arg in args_list:
29    idx = arg.find("perf_options=")
30    if idx != -1:
31      prefix = arg[0:idx]
32      suffix = arg[idx + len("perf_options=") + 1 : -1]
33      new_arg = prefix + "'" + suffix + "'"
34      new_list.append(new_arg)
35    else:
36      new_list.append(arg)
37  args_list = new_list
38
39  return " ".join(args_list)
40
41
42class SuiteRunner(object):
43  """ This defines the interface from crosperf to test script.
44  """
45
46  def __init__(self, logger_to_use=None, log_level="verbose", cmd_exec=None,
47               cmd_term=None):
48    self._logger = logger_to_use
49    self.log_level = log_level
50    self._ce = cmd_exec or command_executer.GetCommandExecuter(self._logger,
51                                                   log_level=self.log_level)
52    self._ct = cmd_term or command_executer.CommandTerminator()
53
54  def Run(self, machine, label, benchmark, test_args, profiler_args):
55    for i in range(0, benchmark.retries + 1):
56      self.PinGovernorExecutionFrequencies(machine, label.chromeos_root)
57      if benchmark.suite == "telemetry":
58        ret_tup = self.Telemetry_Run(machine, label, benchmark, profiler_args)
59      elif benchmark.suite == "telemetry_Crosperf":
60        ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark,
61                                              test_args, profiler_args)
62      else:
63        ret_tup = self.Test_That_Run(machine, label, benchmark, test_args,
64                                     profiler_args)
65      if ret_tup[0] != 0:
66        self._logger.LogOutput("benchmark %s failed. Retries left: %s"
67                               % (benchmark.name, benchmark.retries - i))
68      elif i > 0:
69        self._logger.LogOutput("benchmark %s succeded after %s retries"
70                                % (benchmark.name, i))
71        break
72      else:
73        self._logger.LogOutput("benchmark %s succeded on first try"
74                                % benchmark.name)
75        break
76    return ret_tup
77
78  def GetHighestStaticFrequency(self, machine_name, chromeos_root):
79    """ Gets the highest static frequency for the specified machine
80    """
81    get_avail_freqs = ("cd /sys/devices/system/cpu/cpu0/cpufreq/; "
82                       "if [[ -e scaling_available_frequencies ]]; then "
83                       "  cat scaling_available_frequencies; "
84                       "else "
85                       "  cat scaling_max_freq ; "
86                       "fi")
87    ret, freqs_str, _ = self._ce.CrosRunCommand(
88        get_avail_freqs, return_output=True, machine=machine_name,
89        chromeos_root=chromeos_root)
90    self._logger.LogFatalIf(ret, "Could not get available frequencies "
91                            "from machine: %s" % machine_name)
92    freqs = freqs_str.split()
93    ## When there is no scaling_available_frequencies file,
94    ## we have only 1 choice.
95    if len(freqs) == 1:
96      return freqs[0]
97    # The dynamic frequency ends with a "1000". So, ignore it if found.
98    if freqs[0].endswith("1000"):
99      return freqs[1]
100    else:
101      return freqs[0]
102
103  def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root):
104    """ Set min and max frequencies to max static frequency
105    """
106    highest_freq = self.GetHighestStaticFrequency(machine_name, chromeos_root)
107    BASH_FOR = "for f in {list}; do {body}; done"
108    CPUFREQ_DIRS = "/sys/devices/system/cpu/cpu*/cpufreq/"
109    change_max_freq = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_max_freq",
110                                      body="echo %s > $f" % highest_freq)
111    change_min_freq = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_min_freq",
112                                      body="echo %s > $f" % highest_freq)
113    change_perf_gov = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_governor",
114                                      body="echo performance > $f")
115    if self.log_level == "average":
116      self._logger.LogOutput("Pinning governor execution frequencies for %s"
117                           % machine_name)
118    ret = self._ce.CrosRunCommand(" && ".join(("set -e ",
119                                               change_max_freq,
120                                               change_min_freq,
121                                               change_perf_gov)),
122                                  machine=machine_name,
123                                  chromeos_root=chromeos_root)
124    self._logger.LogFatalIf(ret, "Could not pin frequencies on machine: %s"
125                            % machine_name)
126
127  def RebootMachine(self, machine_name, chromeos_root):
128    command = "reboot && exit"
129    self._ce.CrosRunCommand(command, machine=machine_name,
130                      chromeos_root=chromeos_root)
131    time.sleep(60)
132    # Whenever we reboot the machine, we need to restore the governor settings.
133    self.PinGovernorExecutionFrequencies(machine_name, chromeos_root)
134
135  def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
136    """Run the test_that test.."""
137    options = ""
138    if label.board:
139      options += " --board=%s" % label.board
140    if test_args:
141      options += " %s" % test_args
142    if profiler_args:
143      self._logger.LogFatal("test_that does not support profiler.")
144    command = "rm -rf /usr/local/autotest/results/*"
145    self._ce.CrosRunCommand(command, machine=machine, username="root",
146                            chromeos_root=label.chromeos_root)
147
148    # We do this because some tests leave the machine in weird states.
149    # Rebooting between iterations has proven to help with this.
150    self.RebootMachine(machine, label.chromeos_root)
151
152    command = ("%s %s %s %s" %
153               (TEST_THAT_PATH, options, machine, benchmark.test_name))
154    if self.log_level != "verbose":
155      self._logger.LogOutput("Running test.")
156      self._logger.LogOutput("CMD: %s" % command)
157    # Use --no-ns-pid so that cros_sdk does not create a different
158    # process namespace and we can kill process created easily by
159    # their process group.
160    return self._ce.ChrootRunCommand(label.chromeos_root,
161                                     command,
162                                     True,
163                                     self._ct,
164                                     cros_sdk_options="--no-ns-pid")
165
166  def RemoveTelemetryTempFile (self, machine, chromeos_root):
167    filename = "telemetry@%s" % machine
168    fullname = os.path.join (chromeos_root,
169                             "chroot",
170                             "tmp",
171                             filename)
172    if os.path.exists(fullname):
173        os.remove(fullname)
174
175  def Telemetry_Crosperf_Run (self, machine, label, benchmark, test_args,
176                              profiler_args):
177    if not os.path.isdir(label.chrome_src):
178      self._logger.LogFatal("Cannot find chrome src dir to"
179                            " run telemetry: %s" % label.chrome_src)
180
181    # Check for and remove temporary file that may have been left by
182    # previous telemetry runs (and which might prevent this run from
183    # working).
184    self.RemoveTelemetryTempFile (machine, label.chromeos_root)
185
186    # For telemetry runs, we can use the autotest copy from the source
187    # location. No need to have one under /build/<board>.
188    autotest_dir_arg = '--autotest_dir ~/trunk/src/third_party/autotest/files'
189
190    profiler_args = GetProfilerArgs (profiler_args)
191    fast_arg = ""
192    if not profiler_args:
193      # --fast works unless we are doing profiling (autotest limitation).
194      # --fast avoids unnecessary copies of syslogs.
195      fast_arg = "--fast"
196    args_string = ""
197    if test_args:
198      # Strip double quotes off args (so we can wrap them in single
199      # quotes, to pass through to Telemetry).
200      if test_args[0] == '"' and test_args[-1] == '"':
201        test_args = test_args[1:-1]
202      args_string = "test_args='%s'" % test_args
203
204    cmd = ('{} {} {} --board={} --args="{} run_local={} test={} '
205           '{}" {} telemetry_Crosperf'.format(TEST_THAT_PATH,
206                                              autotest_dir_arg,
207                                              fast_arg,
208                                              label.board,
209                                              args_string,
210                                              benchmark.run_local,
211                                              benchmark.test_name,
212                                              profiler_args,
213                                              machine))
214
215    # Use --no-ns-pid so that cros_sdk does not create a different
216    # process namespace and we can kill process created easily by their
217    # process group.
218    chrome_root_options = ("--no-ns-pid "
219                           "--chrome_root={} --chrome_root_mount={} "
220                           "FEATURES=\"-usersandbox\" "
221                           "CHROME_ROOT={}".format(label.chrome_src,
222                                                    CHROME_MOUNT_DIR,
223                                                    CHROME_MOUNT_DIR))
224    if self.log_level != "verbose":
225      self._logger.LogOutput("Running test.")
226      self._logger.LogOutput("CMD: %s" % cmd)
227    return self._ce.ChrootRunCommand(label.chromeos_root,
228                                     cmd,
229                                     return_output=True,
230                                     command_terminator=self._ct,
231                                     cros_sdk_options=chrome_root_options)
232
233
234  def Telemetry_Run(self, machine, label, benchmark, profiler_args):
235    telemetry_run_path = ""
236    if not os.path.isdir(label.chrome_src):
237      self._logger.LogFatal("Cannot find chrome src dir to"
238                            " run telemetry.")
239    else:
240      telemetry_run_path = os.path.join(label.chrome_src, "src/tools/perf")
241      if not os.path.exists(telemetry_run_path):
242        self._logger.LogFatal("Cannot find %s directory." % telemetry_run_path)
243
244    if profiler_args:
245      self._logger.LogFatal("Telemetry does not support the perf profiler.")
246
247    # Check for and remove temporary file that may have been left by
248    # previous telemetry runs (and which might prevent this run from
249    # working).
250    self.RemoveTelemetryTempFile (machine, label.chromeos_root)
251
252    rsa_key = os.path.join(label.chromeos_root,
253        "src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa")
254
255    cmd = ("cd {0} && "
256           "./run_measurement "
257           "--browser=cros-chrome "
258           "--output-format=csv "
259           "--remote={1} "
260           "--identity {2} "
261           "{3} {4}".format(telemetry_run_path, machine,
262                            rsa_key,
263                            benchmark.test_name,
264                            benchmark.test_args))
265    if self.log_level != "verbose":
266      self._logger.LogOutput("Running test.")
267      self._logger.LogOutput("CMD: %s" % cmd)
268    return self._ce.RunCommand(cmd, return_output=True,
269                               print_to_console=False)
270
271  def Terminate(self):
272    self._ct.Terminate()
273
274
275class MockSuiteRunner(object):
276  def __init__(self):
277    self._true = True
278
279  def Run(self, *_args):
280    if self._true:
281      return [0, "", ""]
282    else:
283      return [0, "", ""]
284