suite_runner.py revision f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe 
1
2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import os
7import time
8import shlex
9
10from utils import command_executer
11import test_flag
12
13TEST_THAT_PATH = '/usr/bin/test_that'
14CHROME_MOUNT_DIR = '/tmp/chrome_root'
15
16
17def GetProfilerArgs(profiler_args):
18  # Remove "--" from in front of profiler args.
19  args_list = shlex.split(profiler_args)
20  new_list = []
21  for arg in args_list:
22    if arg[0:2] == '--':
23      arg = arg[2:]
24    new_list.append(arg)
25  args_list = new_list
26
27  # Remove "perf_options=" from middle of profiler args.
28  new_list = []
29  for arg in args_list:
30    idx = arg.find('perf_options=')
31    if idx != -1:
32      prefix = arg[0:idx]
33      suffix = arg[idx + len('perf_options=') + 1:-1]
34      new_arg = prefix + "'" + suffix + "'"
35      new_list.append(new_arg)
36    else:
37      new_list.append(arg)
38  args_list = new_list
39
40  return ' '.join(args_list)
41
42
43class SuiteRunner(object):
44  """ This defines the interface from crosperf to test script.
45  """
46
47  def __init__(self,
48               logger_to_use=None,
49               log_level='verbose',
50               cmd_exec=None,
51               cmd_term=None):
52    self._logger = logger_to_use
53    self.log_level = log_level
54    self._ce = cmd_exec or command_executer.GetCommandExecuter(
55        self._logger,
56        log_level=self.log_level)
57    self._ct = cmd_term or command_executer.CommandTerminator()
58
59  def Run(self, machine, label, benchmark, test_args, profiler_args):
60    for i in range(0, benchmark.retries + 1):
61      self.PinGovernorExecutionFrequencies(machine, label.chromeos_root)
62      if benchmark.suite == 'telemetry':
63        ret_tup = self.Telemetry_Run(machine, label, benchmark, profiler_args)
64      elif benchmark.suite == 'telemetry_Crosperf':
65        ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark,
66                                              test_args, profiler_args)
67      else:
68        ret_tup = self.Test_That_Run(machine, label, benchmark, test_args,
69                                     profiler_args)
70      if ret_tup[0] != 0:
71        self._logger.LogOutput('benchmark %s failed. Retries left: %s' %
72                               (benchmark.name, benchmark.retries - i))
73      elif i > 0:
74        self._logger.LogOutput('benchmark %s succeded after %s retries' %
75                               (benchmark.name, i))
76        break
77      else:
78        self._logger.LogOutput('benchmark %s succeded on first try' %
79                               benchmark.name)
80        break
81    return ret_tup
82
83  def GetHighestStaticFrequency(self, machine_name, chromeos_root):
84    """ Gets the highest static frequency for the specified machine
85    """
86    get_avail_freqs = ('cd /sys/devices/system/cpu/cpu0/cpufreq/; '
87                       'if [[ -e scaling_available_frequencies ]]; then '
88                       '  cat scaling_available_frequencies; '
89                       'else '
90                       '  cat scaling_max_freq ; '
91                       'fi')
92    ret, freqs_str, _ = self._ce.CrosRunCommandWOutput(
93        get_avail_freqs,
94        machine=machine_name,
95        chromeos_root=chromeos_root)
96    self._logger.LogFatalIf(ret, 'Could not get available frequencies '
97                            'from machine: %s' % machine_name)
98    freqs = freqs_str.split()
99    # We need to make sure that the frequencies are sorted in decreasing
100    # order
101    freqs.sort(key=int, reverse=True)
102
103    ## When there is no scaling_available_frequencies file,
104    ## we have only 1 choice.
105    if len(freqs) == 1:
106      return freqs[0]
107    # The dynamic frequency ends with a "1000". So, ignore it if found.
108    if freqs[0].endswith('1000'):
109      return freqs[1]
110    else:
111      return freqs[0]
112
113  def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root):
114    """ Set min and max frequencies to max static frequency
115    """
116    highest_freq = self.GetHighestStaticFrequency(machine_name, chromeos_root)
117    BASH_FOR = 'for f in {list}; do {body}; done'
118    CPUFREQ_DIRS = '/sys/devices/system/cpu/cpu*/cpufreq/'
119    change_max_freq = BASH_FOR.format(list=CPUFREQ_DIRS + 'scaling_max_freq',
120                                      body='echo %s > $f' % highest_freq)
121    change_min_freq = BASH_FOR.format(list=CPUFREQ_DIRS + 'scaling_min_freq',
122                                      body='echo %s > $f' % highest_freq)
123    change_perf_gov = BASH_FOR.format(list=CPUFREQ_DIRS + 'scaling_governor',
124                                      body='echo performance > $f')
125    if self.log_level == 'average':
126      self._logger.LogOutput('Pinning governor execution frequencies for %s' %
127                             machine_name)
128    ret = self._ce.CrosRunCommand(' && '.join((
129        'set -e ', change_max_freq, change_min_freq, change_perf_gov)),
130                                  machine=machine_name,
131                                  chromeos_root=chromeos_root)
132    self._logger.LogFatalIf(ret, 'Could not pin frequencies on machine: %s' %
133                            machine_name)
134
135  def RebootMachine(self, machine_name, chromeos_root):
136    command = 'reboot && exit'
137    self._ce.CrosRunCommand(command,
138                            machine=machine_name,
139                            chromeos_root=chromeos_root)
140    time.sleep(60)
141    # Whenever we reboot the machine, we need to restore the governor settings.
142    self.PinGovernorExecutionFrequencies(machine_name, chromeos_root)
143
144  def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
145    """Run the test_that test.."""
146    options = ''
147    if label.board:
148      options += ' --board=%s' % label.board
149    if test_args:
150      options += ' %s' % test_args
151    if profiler_args:
152      self._logger.LogFatal('test_that does not support profiler.')
153    command = 'rm -rf /usr/local/autotest/results/*'
154    self._ce.CrosRunCommand(command,
155                            machine=machine,
156                            chromeos_root=label.chromeos_root)
157
158    # We do this because some tests leave the machine in weird states.
159    # Rebooting between iterations has proven to help with this.
160    self.RebootMachine(machine, label.chromeos_root)
161
162    command = (
163        ('%s --autotest_dir ~/trunk/src/third_party/autotest/files --fast '
164         '%s %s %s') % (TEST_THAT_PATH, options, machine, benchmark.test_name))
165    if self.log_level != 'verbose':
166      self._logger.LogOutput('Running test.')
167      self._logger.LogOutput('CMD: %s' % command)
168    # Use --no-ns-pid so that cros_sdk does not create a different
169    # process namespace and we can kill process created easily by
170    # their process group.
171    return self._ce.ChrootRunCommandWOutput(label.chromeos_root,
172                                            command,
173                                            command_terminator=self._ct,
174                                            cros_sdk_options='--no-ns-pid')
175
176  def RemoveTelemetryTempFile(self, machine, chromeos_root):
177    filename = 'telemetry@%s' % machine
178    fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename)
179    if os.path.exists(fullname):
180      os.remove(fullname)
181
182  def Telemetry_Crosperf_Run(self, machine, label, benchmark, test_args,
183                             profiler_args):
184    if not os.path.isdir(label.chrome_src):
185      self._logger.LogFatal('Cannot find chrome src dir to'
186                            ' run telemetry: %s' % label.chrome_src)
187
188    # Check for and remove temporary file that may have been left by
189    # previous telemetry runs (and which might prevent this run from
190    # working).
191    self.RemoveTelemetryTempFile(machine, label.chromeos_root)
192
193    # For telemetry runs, we can use the autotest copy from the source
194    # location. No need to have one under /build/<board>.
195    autotest_dir_arg = '--autotest_dir ~/trunk/src/third_party/autotest/files'
196
197    profiler_args = GetProfilerArgs(profiler_args)
198    fast_arg = ''
199    if not profiler_args:
200      # --fast works unless we are doing profiling (autotest limitation).
201      # --fast avoids unnecessary copies of syslogs.
202      fast_arg = '--fast'
203    args_string = ''
204    if test_args:
205      # Strip double quotes off args (so we can wrap them in single
206      # quotes, to pass through to Telemetry).
207      if test_args[0] == '"' and test_args[-1] == '"':
208        test_args = test_args[1:-1]
209      args_string = "test_args='%s'" % test_args
210
211    cmd = ('{} {} {} --board={} --args="{} run_local={} test={} '
212           '{}" {} telemetry_Crosperf'.format(
213               TEST_THAT_PATH, autotest_dir_arg, fast_arg, label.board,
214               args_string, benchmark.run_local, benchmark.test_name,
215               profiler_args, machine))
216
217    # Use --no-ns-pid so that cros_sdk does not create a different
218    # process namespace and we can kill process created easily by their
219    # process group.
220    chrome_root_options = ('--no-ns-pid '
221                           '--chrome_root={} --chrome_root_mount={} '
222                           "FEATURES=\"-usersandbox\" "
223                           'CHROME_ROOT={}'.format(label.chrome_src,
224                                                   CHROME_MOUNT_DIR,
225                                                   CHROME_MOUNT_DIR))
226    if self.log_level != 'verbose':
227      self._logger.LogOutput('Running test.')
228      self._logger.LogOutput('CMD: %s' % cmd)
229    return self._ce.ChrootRunCommandWOutput(
230        label.chromeos_root,
231        cmd,
232        command_terminator=self._ct,
233        cros_sdk_options=chrome_root_options)
234
235  def Telemetry_Run(self, machine, label, benchmark, profiler_args):
236    telemetry_run_path = ''
237    if not os.path.isdir(label.chrome_src):
238      self._logger.LogFatal('Cannot find chrome src dir to' ' run telemetry.')
239    else:
240      telemetry_run_path = os.path.join(label.chrome_src, 'src/tools/perf')
241      if not os.path.exists(telemetry_run_path):
242        self._logger.LogFatal('Cannot find %s directory.' % telemetry_run_path)
243
244    if profiler_args:
245      self._logger.LogFatal('Telemetry does not support the perf profiler.')
246
247    # Check for and remove temporary file that may have been left by
248    # previous telemetry runs (and which might prevent this run from
249    # working).
250    if not test_flag.GetTestMode():
251      self.RemoveTelemetryTempFile(machine, label.chromeos_root)
252
253    rsa_key = os.path.join(
254        label.chromeos_root,
255        'src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa')
256
257    cmd = ('cd {0} && '
258           './run_measurement '
259           '--browser=cros-chrome '
260           '--output-format=csv '
261           '--remote={1} '
262           '--identity {2} '
263           '{3} {4}'.format(telemetry_run_path, machine, rsa_key,
264                            benchmark.test_name, benchmark.test_args))
265    if self.log_level != 'verbose':
266      self._logger.LogOutput('Running test.')
267      self._logger.LogOutput('CMD: %s' % cmd)
268    return self._ce.RunCommandWOutput(cmd, print_to_console=False)
269
270  def Terminate(self):
271    self._ct.Terminate()
272
273
274class MockSuiteRunner(object):
275
276  def __init__(self):
277    self._true = True
278
279  def Run(self, *_args):
280    if self._true:
281      return [0, '', '']
282    else:
283      return [0, '', '']
284