suite_runner.py revision 70204447c3c103508f1b8e58d59a9b8f165e1c21 
1# Copyright (c) 2013~2015 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""SuiteRunner defines the interface from crosperf to test script."""
5
6from __future__ import print_function
7
8import os
9import time
10import shlex
11
12from cros_utils import command_executer
13import test_flag
14
15TEST_THAT_PATH = '/usr/bin/test_that'
16CHROME_MOUNT_DIR = '/tmp/chrome_root'
17
18
19def GetProfilerArgs(profiler_args):
20  # Remove "--" from in front of profiler args.
21  args_list = shlex.split(profiler_args)
22  new_list = []
23  for arg in args_list:
24    if arg[0:2] == '--':
25      arg = arg[2:]
26    new_list.append(arg)
27  args_list = new_list
28
29  # Remove "perf_options=" from middle of profiler args.
30  new_list = []
31  for arg in args_list:
32    idx = arg.find('perf_options=')
33    if idx != -1:
34      prefix = arg[0:idx]
35      suffix = arg[idx + len('perf_options=') + 1:-1]
36      new_arg = prefix + "'" + suffix + "'"
37      new_list.append(new_arg)
38    else:
39      new_list.append(arg)
40  args_list = new_list
41
42  return ' '.join(args_list)
43
44
45class SuiteRunner(object):
46  """This defines the interface from crosperf to test script."""
47
48  def __init__(self,
49               logger_to_use=None,
50               log_level='verbose',
51               cmd_exec=None,
52               cmd_term=None):
53    self.logger = logger_to_use
54    self.log_level = log_level
55    self._ce = cmd_exec or command_executer.GetCommandExecuter(
56        self.logger, log_level=self.log_level)
57    self._ct = cmd_term or command_executer.CommandTerminator()
58
59  def Run(self, machine, label, benchmark, test_args, profiler_args):
60    for i in range(0, benchmark.retries + 1):
61      self.PinGovernorExecutionFrequencies(machine, label.chromeos_root)
62      if benchmark.suite == 'telemetry':
63        self.DecreaseWaitTime(machine, label.chromeos_root)
64        ret_tup = self.Telemetry_Run(machine, label, benchmark, profiler_args)
65      elif benchmark.suite == 'telemetry_Crosperf':
66        self.DecreaseWaitTime(machine, label.chromeos_root)
67        ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark,
68                                              test_args, profiler_args)
69      else:
70        ret_tup = self.Test_That_Run(machine, label, benchmark, test_args,
71                                     profiler_args)
72      if ret_tup[0] != 0:
73        self.logger.LogOutput('benchmark %s failed. Retries left: %s' %
74                              (benchmark.name, benchmark.retries - i))
75      elif i > 0:
76        self.logger.LogOutput('benchmark %s succeded after %s retries' %
77                              (benchmark.name, i))
78        break
79      else:
80        self.logger.LogOutput('benchmark %s succeded on first try' %
81                              benchmark.name)
82        break
83    return ret_tup
84
85  def GetHighestStaticFrequency(self, machine_name, chromeos_root):
86    """Gets the highest static frequency for the specified machine."""
87    get_avail_freqs = ('cd /sys/devices/system/cpu/cpu0/cpufreq/; '
88                       'if [[ -e scaling_available_frequencies ]]; then '
89                       '  cat scaling_available_frequencies; '
90                       'else '
91                       '  cat scaling_max_freq ; '
92                       'fi')
93    ret, freqs_str, _ = self._ce.CrosRunCommandWOutput(
94        get_avail_freqs, machine=machine_name, chromeos_root=chromeos_root)
95    self.logger.LogFatalIf(ret, 'Could not get available frequencies '
96                           'from machine: %s' % machine_name)
97    freqs = freqs_str.split()
98    # We need to make sure that the frequencies are sorted in decreasing
99    # order
100    freqs.sort(key=int, reverse=True)
101
102    ## When there is no scaling_available_frequencies file,
103    ## we have only 1 choice.
104    if len(freqs) == 1:
105      return freqs[0]
106    # The dynamic frequency ends with a "1000". So, ignore it if found.
107    if freqs[0].endswith('1000'):
108      return freqs[1]
109    else:
110      return freqs[0]
111
112  def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root):
113    """Set min and max frequencies to max static frequency."""
114    highest_freq = self.GetHighestStaticFrequency(machine_name, chromeos_root)
115    BASH_FOR = 'for f in {list}; do {body}; done'
116    CPUFREQ_DIRS = '/sys/devices/system/cpu/cpu*/cpufreq/'
117    change_max_freq = BASH_FOR.format(
118        list=CPUFREQ_DIRS + 'scaling_max_freq',
119        body='echo %s > $f' % highest_freq)
120    change_min_freq = BASH_FOR.format(
121        list=CPUFREQ_DIRS + 'scaling_min_freq',
122        body='echo %s > $f' % highest_freq)
123    change_perf_gov = BASH_FOR.format(
124        list=CPUFREQ_DIRS + 'scaling_governor', body='echo performance > $f')
125    if self.log_level == 'average':
126      self.logger.LogOutput('Pinning governor execution frequencies for %s' %
127                            machine_name)
128    ret = self._ce.CrosRunCommand(
129        ' && '.join(('set -e ', change_max_freq, change_min_freq,
130                     change_perf_gov)),
131        machine=machine_name,
132        chromeos_root=chromeos_root)
133    self.logger.LogFatalIf(ret, 'Could not pin frequencies on machine: %s' %
134                           machine_name)
135
136  def DecreaseWaitTime(self, machine_name, chromeos_root):
137    """Change the ten seconds wait time for pagecycler to two seconds."""
138    FILE = '/usr/local/telemetry/src/tools/perf/page_sets/page_cycler_story.py'
139    ret = self._ce.CrosRunCommand(
140        'ls ' + FILE, machine=machine_name, chromeos_root=chromeos_root)
141    self.logger.LogFatalIf(ret, 'Could not find {} on machine: {}'.format(
142        FILE, machine_name))
143
144    if not ret:
145      sed_command = 'sed -i "s/_TTI_WAIT_TIME = 10/_TTI_WAIT_TIME = 2/g" '
146      ret = self._ce.CrosRunCommand(
147          sed_command + FILE, machine=machine_name, chromeos_root=chromeos_root)
148      self.logger.LogFatalIf(ret, 'Could not modify {} on machine: {}'.format(
149          FILE, machine_name))
150
151  def RebootMachine(self, machine_name, chromeos_root):
152    command = 'reboot && exit'
153    self._ce.CrosRunCommand(
154        command, machine=machine_name, chromeos_root=chromeos_root)
155    time.sleep(60)
156    # Whenever we reboot the machine, we need to restore the governor settings.
157    self.PinGovernorExecutionFrequencies(machine_name, chromeos_root)
158
159  def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
160    """Run the test_that test.."""
161    options = ''
162    if label.board:
163      options += ' --board=%s' % label.board
164    if test_args:
165      options += ' %s' % test_args
166    if profiler_args:
167      self.logger.LogFatal('test_that does not support profiler.')
168    command = 'rm -rf /usr/local/autotest/results/*'
169    self._ce.CrosRunCommand(
170        command, machine=machine, chromeos_root=label.chromeos_root)
171
172    # We do this because some tests leave the machine in weird states.
173    # Rebooting between iterations has proven to help with this.
174    self.RebootMachine(machine, label.chromeos_root)
175
176    autotest_dir = '~/trunk/src/third_party/autotest/files'
177    if label.autotest_path != '':
178      autotest_dir = label.autotest_path
179    command = (
180        ('%s --autotest_dir %s --fast '
181         '%s %s %s') %
182        (TEST_THAT_PATH, autotest_dir, options, machine, benchmark.test_name))
183    if self.log_level != 'verbose':
184      self.logger.LogOutput('Running test.')
185      self.logger.LogOutput('CMD: %s' % command)
186    # Use --no-ns-pid so that cros_sdk does not create a different
187    # process namespace and we can kill process created easily by
188    # their process group.
189    return self._ce.ChrootRunCommandWOutput(
190        label.chromeos_root,
191        command,
192        command_terminator=self._ct,
193        cros_sdk_options='--no-ns-pid')
194
195  def RemoveTelemetryTempFile(self, machine, chromeos_root):
196    filename = 'telemetry@%s' % machine
197    fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename)
198    if os.path.exists(fullname):
199      os.remove(fullname)
200
201  def Telemetry_Crosperf_Run(self, machine, label, benchmark, test_args,
202                             profiler_args):
203    if not os.path.isdir(label.chrome_src):
204      self.logger.LogFatal('Cannot find chrome src dir to'
205                           ' run telemetry: %s' % label.chrome_src)
206
207    # Check for and remove temporary file that may have been left by
208    # previous telemetry runs (and which might prevent this run from
209    # working).
210    self.RemoveTelemetryTempFile(machine, label.chromeos_root)
211
212    # For telemetry runs, we can use the autotest copy from the source
213    # location. No need to have one under /build/<board>.
214    autotest_dir_arg = '--autotest_dir ~/trunk/src/third_party/autotest/files'
215    if label.autotest_path != '':
216      autotest_dir_arg = '--autotest_dir %s' % label.autotest_path
217
218    profiler_args = GetProfilerArgs(profiler_args)
219    fast_arg = ''
220    if not profiler_args:
221      # --fast works unless we are doing profiling (autotest limitation).
222      # --fast avoids unnecessary copies of syslogs.
223      fast_arg = '--fast'
224    args_string = ''
225    if test_args:
226      # Strip double quotes off args (so we can wrap them in single
227      # quotes, to pass through to Telemetry).
228      if test_args[0] == '"' and test_args[-1] == '"':
229        test_args = test_args[1:-1]
230      args_string = "test_args='%s'" % test_args
231
232    cmd = ('{} {} {} --board={} --args="{} run_local={} test={} '
233           '{}" {} telemetry_Crosperf'.format(TEST_THAT_PATH, autotest_dir_arg,
234                                              fast_arg, label.board,
235                                              args_string, benchmark.run_local,
236                                              benchmark.test_name,
237                                              profiler_args, machine))
238
239    # Use --no-ns-pid so that cros_sdk does not create a different
240    # process namespace and we can kill process created easily by their
241    # process group.
242    chrome_root_options = ('--no-ns-pid '
243                           '--chrome_root={} --chrome_root_mount={} '
244                           "FEATURES=\"-usersandbox\" "
245                           'CHROME_ROOT={}'.format(label.chrome_src,
246                                                   CHROME_MOUNT_DIR,
247                                                   CHROME_MOUNT_DIR))
248    if self.log_level != 'verbose':
249      self.logger.LogOutput('Running test.')
250      self.logger.LogOutput('CMD: %s' % cmd)
251    return self._ce.ChrootRunCommandWOutput(
252        label.chromeos_root,
253        cmd,
254        command_terminator=self._ct,
255        cros_sdk_options=chrome_root_options)
256
257  def Telemetry_Run(self, machine, label, benchmark, profiler_args):
258    telemetry_run_path = ''
259    if not os.path.isdir(label.chrome_src):
260      self.logger.LogFatal('Cannot find chrome src dir to' ' run telemetry.')
261    else:
262      telemetry_run_path = os.path.join(label.chrome_src, 'src/tools/perf')
263      if not os.path.exists(telemetry_run_path):
264        self.logger.LogFatal('Cannot find %s directory.' % telemetry_run_path)
265
266    if profiler_args:
267      self.logger.LogFatal('Telemetry does not support the perf profiler.')
268
269    # Check for and remove temporary file that may have been left by
270    # previous telemetry runs (and which might prevent this run from
271    # working).
272    if not test_flag.GetTestMode():
273      self.RemoveTelemetryTempFile(machine, label.chromeos_root)
274
275    rsa_key = os.path.join(
276        label.chromeos_root,
277        'src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa')
278
279    cmd = ('cd {0} && '
280           './run_measurement '
281           '--browser=cros-chrome '
282           '--output-format=csv '
283           '--remote={1} '
284           '--identity {2} '
285           '{3} {4}'.format(telemetry_run_path, machine, rsa_key,
286                            benchmark.test_name, benchmark.test_args))
287    if self.log_level != 'verbose':
288      self.logger.LogOutput('Running test.')
289      self.logger.LogOutput('CMD: %s' % cmd)
290    return self._ce.RunCommandWOutput(cmd, print_to_console=False)
291
292  def CommandTerminator(self):
293    return self._ct
294
295  def Terminate(self):
296    self._ct.Terminate()
297
298
299class MockSuiteRunner(object):
300  """Mock suite runner for test."""
301
302  def __init__(self):
303    self._true = True
304
305  def Run(self, *_args):
306    if self._true:
307      return [0, '', '']
308    else:
309      return [0, '', '']
310