suite_runner.py revision 036c9233742004aa773a374df381b1cf137484f5 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import os
8import time
9import shlex
10
11from utils import command_executer
12import test_flag
13
14TEST_THAT_PATH = '/usr/bin/test_that'
15CHROME_MOUNT_DIR = '/tmp/chrome_root'
16
17def GetProfilerArgs (profiler_args):
18  # Remove "--" from in front of profiler args.
19  args_list = shlex.split(profiler_args)
20  new_list = []
21  for arg in args_list:
22    if arg[0:2] == '--':
23      arg = arg[2:]
24    new_list.append(arg)
25  args_list = new_list
26
27  # Remove "perf_options=" from middle of profiler args.
28  new_list = []
29  for arg in args_list:
30    idx = arg.find("perf_options=")
31    if idx != -1:
32      prefix = arg[0:idx]
33      suffix = arg[idx + len("perf_options=") + 1 : -1]
34      new_arg = prefix + "'" + suffix + "'"
35      new_list.append(new_arg)
36    else:
37      new_list.append(arg)
38  args_list = new_list
39
40  return " ".join(args_list)
41
42
43class SuiteRunner(object):
44  """ This defines the interface from crosperf to test script.
45  """
46
47  def __init__(self, logger_to_use=None, log_level="verbose", cmd_exec=None,
48               cmd_term=None):
49    self._logger = logger_to_use
50    self.log_level = log_level
51    self._ce = cmd_exec or command_executer.GetCommandExecuter(self._logger,
52                                                   log_level=self.log_level)
53    self._ct = cmd_term or command_executer.CommandTerminator()
54
55  def Run(self, machine, label, benchmark, test_args, profiler_args):
56    for i in range(0, benchmark.retries + 1):
57      self.PinGovernorExecutionFrequencies(machine, label.chromeos_root)
58      if benchmark.suite == "telemetry":
59        ret_tup = self.Telemetry_Run(machine, label, benchmark, profiler_args)
60      elif benchmark.suite == "telemetry_Crosperf":
61        ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark,
62                                              test_args, profiler_args)
63      else:
64        ret_tup = self.Test_That_Run(machine, label, benchmark, test_args,
65                                     profiler_args)
66      if ret_tup[0] != 0:
67        self._logger.LogOutput("benchmark %s failed. Retries left: %s"
68                               % (benchmark.name, benchmark.retries - i))
69      elif i > 0:
70        self._logger.LogOutput("benchmark %s succeded after %s retries"
71                                % (benchmark.name, i))
72        break
73      else:
74        self._logger.LogOutput("benchmark %s succeded on first try"
75                                % benchmark.name)
76        break
77    return ret_tup
78
79  def GetHighestStaticFrequency(self, machine_name, chromeos_root):
80    """ Gets the highest static frequency for the specified machine
81    """
82    get_avail_freqs = ("cd /sys/devices/system/cpu/cpu0/cpufreq/; "
83                       "if [[ -e scaling_available_frequencies ]]; then "
84                       "  cat scaling_available_frequencies; "
85                       "else "
86                       "  cat scaling_max_freq ; "
87                       "fi")
88    ret, freqs_str, _ = self._ce.CrosRunCommandWOutput(
89        get_avail_freqs, machine=machine_name, chromeos_root=chromeos_root)
90    self._logger.LogFatalIf(ret, "Could not get available frequencies "
91                            "from machine: %s" % machine_name)
92    freqs = freqs_str.split()
93    # We need to make sure that the frequencies are sorted in decreasing
94    # order
95    freqs.sort(key=int, reverse=True)
96
97    ## When there is no scaling_available_frequencies file,
98    ## we have only 1 choice.
99    if len(freqs) == 1:
100      return freqs[0]
101    # The dynamic frequency ends with a "1000". So, ignore it if found.
102    if freqs[0].endswith("1000"):
103      return freqs[1]
104    else:
105      return freqs[0]
106
107  def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root):
108    """ Set min and max frequencies to max static frequency
109    """
110    highest_freq = self.GetHighestStaticFrequency(machine_name, chromeos_root)
111    BASH_FOR = "for f in {list}; do {body}; done"
112    CPUFREQ_DIRS = "/sys/devices/system/cpu/cpu*/cpufreq/"
113    change_max_freq = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_max_freq",
114                                      body="echo %s > $f" % highest_freq)
115    change_min_freq = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_min_freq",
116                                      body="echo %s > $f" % highest_freq)
117    change_perf_gov = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_governor",
118                                      body="echo performance > $f")
119    if self.log_level == "average":
120      self._logger.LogOutput("Pinning governor execution frequencies for %s"
121                           % machine_name)
122    ret = self._ce.CrosRunCommand(" && ".join(("set -e ",
123                                               change_max_freq,
124                                               change_min_freq,
125                                               change_perf_gov)),
126                                  machine=machine_name,
127                                  chromeos_root=chromeos_root)
128    self._logger.LogFatalIf(ret, "Could not pin frequencies on machine: %s"
129                            % machine_name)
130
131  def RebootMachine(self, machine_name, chromeos_root):
132    command = "reboot && exit"
133    self._ce.CrosRunCommand(command, machine=machine_name,
134                      chromeos_root=chromeos_root)
135    time.sleep(60)
136    # Whenever we reboot the machine, we need to restore the governor settings.
137    self.PinGovernorExecutionFrequencies(machine_name, chromeos_root)
138
139  def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
140    """Run the test_that test.."""
141    options = ""
142    if label.board:
143      options += " --board=%s" % label.board
144    if test_args:
145      options += " %s" % test_args
146    if profiler_args:
147      self._logger.LogFatal("test_that does not support profiler.")
148    command = "rm -rf /usr/local/autotest/results/*"
149    self._ce.CrosRunCommand(command, machine=machine,
150                            chromeos_root=label.chromeos_root)
151
152    # We do this because some tests leave the machine in weird states.
153    # Rebooting between iterations has proven to help with this.
154    self.RebootMachine(machine, label.chromeos_root)
155
156    command = (("%s --autotest_dir ~/trunk/src/third_party/autotest/files --fast "
157               "%s %s %s") %
158               (TEST_THAT_PATH, options, machine, benchmark.test_name))
159    if self.log_level != "verbose":
160      self._logger.LogOutput("Running test.")
161      self._logger.LogOutput("CMD: %s" % command)
162    # Use --no-ns-pid so that cros_sdk does not create a different
163    # process namespace and we can kill process created easily by
164    # their process group.
165    return self._ce.ChrootRunCommandWOutput(
166        label.chromeos_root, command, command_terminator=self._ct,
167        cros_sdk_options="--no-ns-pid")
168
169  def RemoveTelemetryTempFile (self, machine, chromeos_root):
170    filename = "telemetry@%s" % machine
171    fullname = os.path.join (chromeos_root,
172                             "chroot",
173                             "tmp",
174                             filename)
175    if os.path.exists(fullname):
176        os.remove(fullname)
177
178  def Telemetry_Crosperf_Run (self, machine, label, benchmark, test_args,
179                              profiler_args):
180    if not os.path.isdir(label.chrome_src):
181      self._logger.LogFatal("Cannot find chrome src dir to"
182                            " run telemetry: %s" % label.chrome_src)
183
184    # Check for and remove temporary file that may have been left by
185    # previous telemetry runs (and which might prevent this run from
186    # working).
187    self.RemoveTelemetryTempFile (machine, label.chromeos_root)
188
189    # For telemetry runs, we can use the autotest copy from the source
190    # location. No need to have one under /build/<board>.
191    autotest_dir_arg = '--autotest_dir ~/trunk/src/third_party/autotest/files'
192
193    profiler_args = GetProfilerArgs (profiler_args)
194    fast_arg = ""
195    if not profiler_args:
196      # --fast works unless we are doing profiling (autotest limitation).
197      # --fast avoids unnecessary copies of syslogs.
198      fast_arg = "--fast"
199    args_string = ""
200    if test_args:
201      # Strip double quotes off args (so we can wrap them in single
202      # quotes, to pass through to Telemetry).
203      if test_args[0] == '"' and test_args[-1] == '"':
204        test_args = test_args[1:-1]
205      args_string = "test_args='%s'" % test_args
206
207    cmd = ('{} {} {} --board={} --args="{} run_local={} test={} '
208           '{}" {} telemetry_Crosperf'.format(TEST_THAT_PATH,
209                                              autotest_dir_arg,
210                                              fast_arg,
211                                              label.board,
212                                              args_string,
213                                              benchmark.run_local,
214                                              benchmark.test_name,
215                                              profiler_args,
216                                              machine))
217
218    # Use --no-ns-pid so that cros_sdk does not create a different
219    # process namespace and we can kill process created easily by their
220    # process group.
221    chrome_root_options = ("--no-ns-pid "
222                           "--chrome_root={} --chrome_root_mount={} "
223                           "FEATURES=\"-usersandbox\" "
224                           "CHROME_ROOT={}".format(label.chrome_src,
225                                                    CHROME_MOUNT_DIR,
226                                                    CHROME_MOUNT_DIR))
227    if self.log_level != "verbose":
228      self._logger.LogOutput("Running test.")
229      self._logger.LogOutput("CMD: %s" % cmd)
230    return self._ce.ChrootRunCommandWOutput(
231        label.chromeos_root, cmd, command_terminator=self._ct,
232        cros_sdk_options=chrome_root_options)
233
234
235  def Telemetry_Run(self, machine, label, benchmark, profiler_args):
236    telemetry_run_path = ""
237    if not os.path.isdir(label.chrome_src):
238      self._logger.LogFatal("Cannot find chrome src dir to"
239                            " run telemetry.")
240    else:
241      telemetry_run_path = os.path.join(label.chrome_src, "src/tools/perf")
242      if not os.path.exists(telemetry_run_path):
243        self._logger.LogFatal("Cannot find %s directory." % telemetry_run_path)
244
245    if profiler_args:
246      self._logger.LogFatal("Telemetry does not support the perf profiler.")
247
248    # Check for and remove temporary file that may have been left by
249    # previous telemetry runs (and which might prevent this run from
250    # working).
251    if not test_flag.GetTestMode():
252      self.RemoveTelemetryTempFile (machine, label.chromeos_root)
253
254    rsa_key = os.path.join(label.chromeos_root,
255        "src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa")
256
257    cmd = ("cd {0} && "
258           "./run_measurement "
259           "--browser=cros-chrome "
260           "--output-format=csv "
261           "--remote={1} "
262           "--identity {2} "
263           "{3} {4}".format(telemetry_run_path, machine,
264                            rsa_key,
265                            benchmark.test_name,
266                            benchmark.test_args))
267    if self.log_level != "verbose":
268      self._logger.LogOutput("Running test.")
269      self._logger.LogOutput("CMD: %s" % cmd)
270    return self._ce.RunCommandWOutput(cmd, print_to_console=False)
271
272  def Terminate(self):
273    self._ct.Terminate()
274
275
276class MockSuiteRunner(object):
277  def __init__(self):
278    self._true = True
279
280  def Run(self, *_args):
281    if self._true:
282      return [0, "", ""]
283    else:
284      return [0, "", ""]
285