suite_runner.py revision cbc5a2617f8d63e3319e159eab2c407952df1df9 
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import print_function
6
7import os
8import time
9import shlex
10
11from utils import command_executer
12import test_flag
13
14TEST_THAT_PATH = '/usr/bin/test_that'
15CHROME_MOUNT_DIR = '/tmp/chrome_root'
16
17
18def GetProfilerArgs(profiler_args):
19  # Remove "--" from in front of profiler args.
20  args_list = shlex.split(profiler_args)
21  new_list = []
22  for arg in args_list:
23    if arg[0:2] == '--':
24      arg = arg[2:]
25    new_list.append(arg)
26  args_list = new_list
27
28  # Remove "perf_options=" from middle of profiler args.
29  new_list = []
30  for arg in args_list:
31    idx = arg.find('perf_options=')
32    if idx != -1:
33      prefix = arg[0:idx]
34      suffix = arg[idx + len('perf_options=') + 1:-1]
35      new_arg = prefix + "'" + suffix + "'"
36      new_list.append(new_arg)
37    else:
38      new_list.append(arg)
39  args_list = new_list
40
41  return ' '.join(args_list)
42
43
44class SuiteRunner(object):
45  """ This defines the interface from crosperf to test script.
46  """
47
48  def __init__(self,
49               logger_to_use=None,
50               log_level='verbose',
51               cmd_exec=None,
52               cmd_term=None):
53    self._logger = logger_to_use
54    self.log_level = log_level
55    self._ce = cmd_exec or command_executer.GetCommandExecuter(
56        self._logger,
57        log_level=self.log_level)
58    self._ct = cmd_term or command_executer.CommandTerminator()
59
60  def Run(self, machine, label, benchmark, test_args, profiler_args):
61    for i in range(0, benchmark.retries + 1):
62      self.PinGovernorExecutionFrequencies(machine, label.chromeos_root)
63      if benchmark.suite == 'telemetry':
64        ret_tup = self.Telemetry_Run(machine, label, benchmark, profiler_args)
65      elif benchmark.suite == 'telemetry_Crosperf':
66        ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark,
67                                              test_args, profiler_args)
68      else:
69        ret_tup = self.Test_That_Run(machine, label, benchmark, test_args,
70                                     profiler_args)
71      if ret_tup[0] != 0:
72        self._logger.LogOutput('benchmark %s failed. Retries left: %s' %
73                               (benchmark.name, benchmark.retries - i))
74      elif i > 0:
75        self._logger.LogOutput('benchmark %s succeded after %s retries' %
76                               (benchmark.name, i))
77        break
78      else:
79        self._logger.LogOutput('benchmark %s succeded on first try' %
80                               benchmark.name)
81        break
82    return ret_tup
83
84  def GetHighestStaticFrequency(self, machine_name, chromeos_root):
85    """ Gets the highest static frequency for the specified machine
86    """
87    get_avail_freqs = ('cd /sys/devices/system/cpu/cpu0/cpufreq/; '
88                       'if [[ -e scaling_available_frequencies ]]; then '
89                       '  cat scaling_available_frequencies; '
90                       'else '
91                       '  cat scaling_max_freq ; '
92                       'fi')
93    ret, freqs_str, _ = self._ce.CrosRunCommandWOutput(
94        get_avail_freqs,
95        machine=machine_name,
96        chromeos_root=chromeos_root)
97    self._logger.LogFatalIf(ret, 'Could not get available frequencies '
98                            'from machine: %s' % machine_name)
99    freqs = freqs_str.split()
100    # We need to make sure that the frequencies are sorted in decreasing
101    # order
102    freqs.sort(key=int, reverse=True)
103
104    ## When there is no scaling_available_frequencies file,
105    ## we have only 1 choice.
106    if len(freqs) == 1:
107      return freqs[0]
108    # The dynamic frequency ends with a "1000". So, ignore it if found.
109    if freqs[0].endswith('1000'):
110      return freqs[1]
111    else:
112      return freqs[0]
113
114  def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root):
115    """ Set min and max frequencies to max static frequency
116    """
117    highest_freq = self.GetHighestStaticFrequency(machine_name, chromeos_root)
118    BASH_FOR = 'for f in {list}; do {body}; done'
119    CPUFREQ_DIRS = '/sys/devices/system/cpu/cpu*/cpufreq/'
120    change_max_freq = BASH_FOR.format(list=CPUFREQ_DIRS + 'scaling_max_freq',
121                                      body='echo %s > $f' % highest_freq)
122    change_min_freq = BASH_FOR.format(list=CPUFREQ_DIRS + 'scaling_min_freq',
123                                      body='echo %s > $f' % highest_freq)
124    change_perf_gov = BASH_FOR.format(list=CPUFREQ_DIRS + 'scaling_governor',
125                                      body='echo performance > $f')
126    if self.log_level == 'average':
127      self._logger.LogOutput('Pinning governor execution frequencies for %s' %
128                             machine_name)
129    ret = self._ce.CrosRunCommand(' && '.join((
130        'set -e ', change_max_freq, change_min_freq, change_perf_gov)),
131                                  machine=machine_name,
132                                  chromeos_root=chromeos_root)
133    self._logger.LogFatalIf(ret, 'Could not pin frequencies on machine: %s' %
134                            machine_name)
135
136  def RebootMachine(self, machine_name, chromeos_root):
137    command = 'reboot && exit'
138    self._ce.CrosRunCommand(command,
139                            machine=machine_name,
140                            chromeos_root=chromeos_root)
141    time.sleep(60)
142    # Whenever we reboot the machine, we need to restore the governor settings.
143    self.PinGovernorExecutionFrequencies(machine_name, chromeos_root)
144
145  def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
146    """Run the test_that test.."""
147    options = ''
148    if label.board:
149      options += ' --board=%s' % label.board
150    if test_args:
151      options += ' %s' % test_args
152    if profiler_args:
153      self._logger.LogFatal('test_that does not support profiler.')
154    command = 'rm -rf /usr/local/autotest/results/*'
155    self._ce.CrosRunCommand(command,
156                            machine=machine,
157                            chromeos_root=label.chromeos_root)
158
159    # We do this because some tests leave the machine in weird states.
160    # Rebooting between iterations has proven to help with this.
161    self.RebootMachine(machine, label.chromeos_root)
162
163    command = (
164        ('%s --autotest_dir ~/trunk/src/third_party/autotest/files --fast '
165         '%s %s %s') % (TEST_THAT_PATH, options, machine, benchmark.test_name))
166    if self.log_level != 'verbose':
167      self._logger.LogOutput('Running test.')
168      self._logger.LogOutput('CMD: %s' % command)
169    # Use --no-ns-pid so that cros_sdk does not create a different
170    # process namespace and we can kill process created easily by
171    # their process group.
172    return self._ce.ChrootRunCommandWOutput(label.chromeos_root,
173                                            command,
174                                            command_terminator=self._ct,
175                                            cros_sdk_options='--no-ns-pid')
176
177  def RemoveTelemetryTempFile(self, machine, chromeos_root):
178    filename = 'telemetry@%s' % machine
179    fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename)
180    if os.path.exists(fullname):
181      os.remove(fullname)
182
183  def Telemetry_Crosperf_Run(self, machine, label, benchmark, test_args,
184                             profiler_args):
185    if not os.path.isdir(label.chrome_src):
186      self._logger.LogFatal('Cannot find chrome src dir to'
187                            ' run telemetry: %s' % label.chrome_src)
188
189    # Check for and remove temporary file that may have been left by
190    # previous telemetry runs (and which might prevent this run from
191    # working).
192    self.RemoveTelemetryTempFile(machine, label.chromeos_root)
193
194    # For telemetry runs, we can use the autotest copy from the source
195    # location. No need to have one under /build/<board>.
196    autotest_dir_arg = '--autotest_dir ~/trunk/src/third_party/autotest/files'
197
198    profiler_args = GetProfilerArgs(profiler_args)
199    fast_arg = ''
200    if not profiler_args:
201      # --fast works unless we are doing profiling (autotest limitation).
202      # --fast avoids unnecessary copies of syslogs.
203      fast_arg = '--fast'
204    args_string = ''
205    if test_args:
206      # Strip double quotes off args (so we can wrap them in single
207      # quotes, to pass through to Telemetry).
208      if test_args[0] == '"' and test_args[-1] == '"':
209        test_args = test_args[1:-1]
210      args_string = "test_args='%s'" % test_args
211
212    cmd = ('{} {} {} --board={} --args="{} run_local={} test={} '
213           '{}" {} telemetry_Crosperf'.format(
214               TEST_THAT_PATH, autotest_dir_arg, fast_arg, label.board,
215               args_string, benchmark.run_local, benchmark.test_name,
216               profiler_args, machine))
217
218    # Use --no-ns-pid so that cros_sdk does not create a different
219    # process namespace and we can kill process created easily by their
220    # process group.
221    chrome_root_options = ('--no-ns-pid '
222                           '--chrome_root={} --chrome_root_mount={} '
223                           "FEATURES=\"-usersandbox\" "
224                           'CHROME_ROOT={}'.format(label.chrome_src,
225                                                   CHROME_MOUNT_DIR,
226                                                   CHROME_MOUNT_DIR))
227    if self.log_level != 'verbose':
228      self._logger.LogOutput('Running test.')
229      self._logger.LogOutput('CMD: %s' % cmd)
230    return self._ce.ChrootRunCommandWOutput(
231        label.chromeos_root,
232        cmd,
233        command_terminator=self._ct,
234        cros_sdk_options=chrome_root_options)
235
236  def Telemetry_Run(self, machine, label, benchmark, profiler_args):
237    telemetry_run_path = ''
238    if not os.path.isdir(label.chrome_src):
239      self._logger.LogFatal('Cannot find chrome src dir to' ' run telemetry.')
240    else:
241      telemetry_run_path = os.path.join(label.chrome_src, 'src/tools/perf')
242      if not os.path.exists(telemetry_run_path):
243        self._logger.LogFatal('Cannot find %s directory.' % telemetry_run_path)
244
245    if profiler_args:
246      self._logger.LogFatal('Telemetry does not support the perf profiler.')
247
248    # Check for and remove temporary file that may have been left by
249    # previous telemetry runs (and which might prevent this run from
250    # working).
251    if not test_flag.GetTestMode():
252      self.RemoveTelemetryTempFile(machine, label.chromeos_root)
253
254    rsa_key = os.path.join(
255        label.chromeos_root,
256        'src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa')
257
258    cmd = ('cd {0} && '
259           './run_measurement '
260           '--browser=cros-chrome '
261           '--output-format=csv '
262           '--remote={1} '
263           '--identity {2} '
264           '{3} {4}'.format(telemetry_run_path, machine, rsa_key,
265                            benchmark.test_name, benchmark.test_args))
266    if self.log_level != 'verbose':
267      self._logger.LogOutput('Running test.')
268      self._logger.LogOutput('CMD: %s' % cmd)
269    return self._ce.RunCommandWOutput(cmd, print_to_console=False)
270
271  def Terminate(self):
272    self._ct.Terminate()
273
274
275class MockSuiteRunner(object):
276
277  def __init__(self):
278    self._true = True
279
280  def Run(self, *_args):
281    if self._true:
282      return [0, '', '']
283    else:
284      return [0, '', '']
285