suite_runner_unittest.py revision 4b133961b76c2cb8bc58f0ea2cded9e3438ffb6f 
1#!/usr/bin/env python2
2#
3# Copyright 2014 Google Inc. All Rights Reserved.
4"""Unittest for suite_runner."""
5
6from __future__ import print_function
7
8import os.path
9import time
10
11import mock
12import unittest
13
14import suite_runner
15import label
16import test_flag
17
18from benchmark import Benchmark
19
20from cros_utils import command_executer
21from cros_utils import logger
22
23
24class SuiteRunnerTest(unittest.TestCase):
25  """Class of SuiteRunner test."""
26  real_logger = logger.GetLogger()
27
28  mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
29  mock_cmd_term = mock.Mock(spec=command_executer.CommandTerminator)
30  mock_logger = mock.Mock(spec=logger.Logger)
31  mock_label = label.MockLabel('lumpy', 'lumpy_chromeos_image', '',
32                               '/tmp/chromeos', 'lumpy',
33                               ['lumpy1.cros', 'lumpy.cros2'], '', '', False,
34                               'average', 'gcc', '')
35  telemetry_crosperf_bench = Benchmark(
36      'b1_test',  # name
37      'octane',  # test_name
38      '',  # test_args
39      3,  # iterations
40      False,  # rm_chroot_tmp
41      'record -e cycles',  # perf_args
42      'telemetry_Crosperf',  # suite
43      True)  # show_all_results
44
45  test_that_bench = Benchmark(
46      'b2_test',  # name
47      'octane',  # test_name
48      '',  # test_args
49      3,  # iterations
50      False,  # rm_chroot_tmp
51      'record -e cycles')  # perf_args
52
53  telemetry_bench = Benchmark(
54      'b3_test',  # name
55      'octane',  # test_name
56      '',  # test_args
57      3,  # iterations
58      False,  # rm_chroot_tmp
59      'record -e cycles',  # perf_args
60      'telemetry',  # suite
61      False)  # show_all_results
62
63  def __init__(self, *args, **kwargs):
64    super(SuiteRunnerTest, self).__init__(*args, **kwargs)
65    self.call_test_that_run = False
66    self.pin_governor_args = []
67    self.test_that_args = []
68    self.telemetry_run_args = []
69    self.telemetry_crosperf_args = []
70    self.call_telemetry_crosperf_run = False
71    self.call_pin_governor = False
72    self.call_telemetry_run = False
73
74  def setUp(self):
75    self.runner = suite_runner.SuiteRunner(self.mock_logger, 'verbose',
76                                           self.mock_cmd_exec,
77                                           self.mock_cmd_term)
78
79  def test_get_profiler_args(self):
80    input_str = ('--profiler=custom_perf --profiler_args=\'perf_options'
81                 '="record -a -e cycles,instructions"\'')
82    output_str = ("profiler=custom_perf profiler_args='record -a -e "
83                  "cycles,instructions'")
84    res = suite_runner.GetProfilerArgs(input_str)
85    self.assertEqual(res, output_str)
86
87  def test_run(self):
88
89    def reset():
90      self.call_pin_governor = False
91      self.call_test_that_run = False
92      self.call_telemetry_run = False
93      self.call_telemetry_crosperf_run = False
94      self.pin_governor_args = []
95      self.test_that_args = []
96      self.telemetry_run_args = []
97      self.telemetry_crosperf_args = []
98
99    def FakePinGovernor(machine, chroot):
100      self.call_pin_governor = True
101      self.pin_governor_args = [machine, chroot]
102
103    def FakeTelemetryRun(machine, test_label, benchmark, profiler_args):
104      self.telemetry_run_args = [machine, test_label, benchmark, profiler_args]
105      self.call_telemetry_run = True
106      return 'Ran FakeTelemetryRun'
107
108    def FakeTelemetryCrosperfRun(machine, test_label, benchmark, test_args,
109                                 profiler_args):
110      self.telemetry_crosperf_args = [
111          machine, test_label, benchmark, test_args, profiler_args
112      ]
113      self.call_telemetry_crosperf_run = True
114      return 'Ran FakeTelemetryCrosperfRun'
115
116    def FakeTestThatRun(machine, test_label, benchmark, test_args,
117                        profiler_args):
118      self.test_that_args = [
119          machine, test_label, benchmark, test_args, profiler_args
120      ]
121      self.call_test_that_run = True
122      return 'Ran FakeTestThatRun'
123
124    self.runner.PinGovernorExecutionFrequencies = FakePinGovernor
125    self.runner.Telemetry_Run = FakeTelemetryRun
126    self.runner.Telemetry_Crosperf_Run = FakeTelemetryCrosperfRun
127    self.runner.Test_That_Run = FakeTestThatRun
128
129    machine = 'fake_machine'
130    test_args = ''
131    profiler_args = ''
132    reset()
133    self.runner.Run(machine, self.mock_label, self.telemetry_bench, test_args,
134                    profiler_args)
135    self.assertTrue(self.call_pin_governor)
136    self.assertTrue(self.call_telemetry_run)
137    self.assertFalse(self.call_test_that_run)
138    self.assertFalse(self.call_telemetry_crosperf_run)
139    self.assertEqual(
140        self.telemetry_run_args,
141        ['fake_machine', self.mock_label, self.telemetry_bench, ''])
142
143    reset()
144    self.runner.Run(machine, self.mock_label, self.test_that_bench, test_args,
145                    profiler_args)
146    self.assertTrue(self.call_pin_governor)
147    self.assertFalse(self.call_telemetry_run)
148    self.assertTrue(self.call_test_that_run)
149    self.assertFalse(self.call_telemetry_crosperf_run)
150    self.assertEqual(
151        self.test_that_args,
152        ['fake_machine', self.mock_label, self.test_that_bench, '', ''])
153
154    reset()
155    self.runner.Run(machine, self.mock_label, self.telemetry_crosperf_bench,
156                    test_args, profiler_args)
157    self.assertTrue(self.call_pin_governor)
158    self.assertFalse(self.call_telemetry_run)
159    self.assertFalse(self.call_test_that_run)
160    self.assertTrue(self.call_telemetry_crosperf_run)
161    self.assertEqual(self.telemetry_crosperf_args, [
162        'fake_machine', self.mock_label, self.telemetry_crosperf_bench, '', ''
163    ])
164
165  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommandWOutput')
166  def test_get_highest_static_frequency(self, mock_cros_runcmd):
167
168    self.mock_cmd_exec.CrosRunCommandWOutput = mock_cros_runcmd
169    mock_cros_runcmd.return_value = [0, '1666000 1333000 1000000', '']
170    freq = self.runner.GetHighestStaticFrequency('lumpy1.cros', '/tmp/chromeos')
171    self.assertEqual(freq, '1666000')
172
173    mock_cros_runcmd.return_value = [0, '1333000', '']
174    freq = self.runner.GetHighestStaticFrequency('lumpy1.cros', '/tmp/chromeos')
175    self.assertEqual(freq, '1333000')
176
177    mock_cros_runcmd.return_value = [0, '1661000 1333000 1000000', '']
178    freq = self.runner.GetHighestStaticFrequency('lumpy1.cros', '/tmp/chromeos')
179    self.assertEqual(freq, '1333000')
180
181  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand')
182  def test_pin_governor_execution_frequencies(self, mock_cros_runcmd):
183
184    def FakeGetHighestFreq(machine_name, chromeos_root):
185      if machine_name or chromeos_root:
186        pass
187      return '1666000'
188
189    self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd
190    self.runner.GetHighestStaticFrequency = FakeGetHighestFreq
191    self.runner.PinGovernorExecutionFrequencies('lumpy1.cros', '/tmp/chromeos')
192    self.assertEqual(mock_cros_runcmd.call_count, 1)
193    cmd = mock_cros_runcmd.call_args_list[0][0]
194    self.assertEqual(cmd, (
195        'set -e  && for f in '
196        '/sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq; do echo '
197        '1666000 > $f; done && for f in '
198        '/sys/devices/system/cpu/cpu*/cpufreq/scaling_min_freq; do echo '
199        '1666000 > $f; done && for f in '
200        '/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo '
201        'performance > $f; done',))
202
203  @mock.patch.object(time, 'sleep')
204  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand')
205  def test_reboot_machine(self, mock_cros_runcmd, mock_sleep):
206
207    def FakePinGovernor(machine_name, chromeos_root):
208      if machine_name or chromeos_root:
209        pass
210
211    self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd
212    self.runner.PinGovernorExecutionFrequencies = FakePinGovernor
213    self.runner.RebootMachine('lumpy1.cros', '/tmp/chromeos')
214    self.assertEqual(mock_cros_runcmd.call_count, 1)
215    self.assertEqual(mock_cros_runcmd.call_args_list[0][0], ('reboot && exit',))
216    self.assertEqual(mock_sleep.call_count, 1)
217    self.assertEqual(mock_sleep.call_args_list[0][0], (60,))
218
219  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand')
220  @mock.patch.object(command_executer.CommandExecuter,
221                     'ChrootRunCommandWOutput')
222  def test_test_that_run(self, mock_chroot_runcmd, mock_cros_runcmd):
223
224    def FakeRebootMachine(machine, chroot):
225      if machine or chroot:
226        pass
227
228    def FakeLogMsg(fd, termfd, msg, flush):
229      if fd or termfd or msg or flush:
230        pass
231
232    save_log_msg = self.real_logger.LogMsg
233    self.real_logger.LogMsg = FakeLogMsg
234    self.runner._logger = self.real_logger
235    self.runner.RebootMachine = FakeRebootMachine
236
237    raised_exception = False
238    try:
239      self.runner.Test_That_Run('lumpy1.cros', self.mock_label,
240                                self.test_that_bench, '', 'record -a -e cycles')
241    except:
242      raised_exception = True
243    self.assertTrue(raised_exception)
244
245    mock_chroot_runcmd.return_value = 0
246    self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd
247    self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd
248    res = self.runner.Test_That_Run('lumpy1.cros', self.mock_label,
249                                    self.test_that_bench, '--iterations=2', '')
250    self.assertEqual(mock_cros_runcmd.call_count, 1)
251    self.assertEqual(mock_chroot_runcmd.call_count, 1)
252    self.assertEqual(res, 0)
253    self.assertEqual(mock_cros_runcmd.call_args_list[0][0],
254                     ('rm -rf /usr/local/autotest/results/*',))
255    args_list = mock_chroot_runcmd.call_args_list[0][0]
256    args_dict = mock_chroot_runcmd.call_args_list[0][1]
257    self.assertEqual(len(args_list), 2)
258    self.assertEqual(args_list[0], '/tmp/chromeos')
259    self.assertEqual(args_list[1], ('/usr/bin/test_that --autotest_dir '
260                                    '~/trunk/src/third_party/autotest/files '
261                                    '--fast  --board=lumpy '
262                                    '--iterations=2 lumpy1.cros octane'))
263    self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term)
264    self.real_logger.LogMsg = save_log_msg
265
266  @mock.patch.object(os.path, 'isdir')
267  @mock.patch.object(command_executer.CommandExecuter,
268                     'ChrootRunCommandWOutput')
269  def test_telemetry_crosperf_run(self, mock_chroot_runcmd, mock_isdir):
270
271    mock_isdir.return_value = True
272    mock_chroot_runcmd.return_value = 0
273    self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd
274    profiler_args = ('--profiler=custom_perf --profiler_args=\'perf_options'
275                     '="record -a -e cycles,instructions"\'')
276    res = self.runner.Telemetry_Crosperf_Run('lumpy1.cros', self.mock_label,
277                                             self.telemetry_crosperf_bench, '',
278                                             profiler_args)
279    self.assertEqual(res, 0)
280    self.assertEqual(mock_chroot_runcmd.call_count, 1)
281    args_list = mock_chroot_runcmd.call_args_list[0][0]
282    args_dict = mock_chroot_runcmd.call_args_list[0][1]
283    self.assertEqual(args_list[0], '/tmp/chromeos')
284    self.assertEqual(args_list[1],
285                     ('/usr/bin/test_that --autotest_dir '
286                      '~/trunk/src/third_party/autotest/files '
287                      ' --board=lumpy --args=" run_local=False test=octane '
288                      'profiler=custom_perf profiler_args=\'record -a -e '
289                      'cycles,instructions\'" lumpy1.cros telemetry_Crosperf'))
290    self.assertEqual(args_dict['cros_sdk_options'],
291                     ('--no-ns-pid --chrome_root= '
292                      '--chrome_root_mount=/tmp/chrome_root '
293                      'FEATURES="-usersandbox" CHROME_ROOT=/tmp/chrome_root'))
294    self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term)
295    self.assertEqual(len(args_dict), 2)
296
297  @mock.patch.object(os.path, 'isdir')
298  @mock.patch.object(os.path, 'exists')
299  @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput')
300  def test_telemetry_run(self, mock_runcmd, mock_exists, mock_isdir):
301
302    def FakeLogMsg(fd, termfd, msg, flush):
303      if fd or termfd or msg or flush:
304        pass
305
306    save_log_msg = self.real_logger.LogMsg
307    self.real_logger.LogMsg = FakeLogMsg
308    mock_runcmd.return_value = 0
309
310    self.mock_cmd_exec.RunCommandWOutput = mock_runcmd
311    self.runner._logger = self.real_logger
312
313    profiler_args = ('--profiler=custom_perf --profiler_args=\'perf_options'
314                     '="record -a -e cycles,instructions"\'')
315
316    raises_exception = False
317    mock_isdir.return_value = False
318    try:
319      self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
320                                self.telemetry_bench, '')
321    except:
322      raises_exception = True
323    self.assertTrue(raises_exception)
324
325    raises_exception = False
326    mock_isdir.return_value = True
327    mock_exists.return_value = False
328    try:
329      self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
330                                self.telemetry_bench, '')
331    except:
332      raises_exception = True
333    self.assertTrue(raises_exception)
334
335    raises_exception = False
336    mock_isdir.return_value = True
337    mock_exists.return_value = True
338    try:
339      self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
340                                self.telemetry_bench, profiler_args)
341    except:
342      raises_exception = True
343    self.assertTrue(raises_exception)
344
345    test_flag.SetTestMode(True)
346    res = self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
347                                    self.telemetry_bench, '')
348    self.assertEqual(res, 0)
349    self.assertEqual(mock_runcmd.call_count, 1)
350    self.assertEqual(mock_runcmd.call_args_list[0][0], (
351        ('cd src/tools/perf && ./run_measurement '
352         '--browser=cros-chrome --output-format=csv '
353         '--remote=lumpy1.cros --identity /tmp/chromeos/src/scripts'
354         '/mod_for_test_scripts/ssh_keys/testing_rsa octane '),))
355
356    self.real_logger.LogMsg = save_log_msg
357
358
359if __name__ == '__main__':
360  unittest.main()
361