suite_runner_unittest.py revision b5cdd417bf92e40c9e77caf2e84f733bf1e51437 
1#!/usr/bin/python2
2#
3# Copyright 2014 Google Inc. All Rights Reserved.
4"""Unittest for suite_runner."""
5
6from __future__ import print_function
7
8import os.path
9import time
10
11import mock
12import unittest
13
14import suite_runner
15import label
16import test_flag
17
18from benchmark import Benchmark
19
20from cros_utils import command_executer
21from cros_utils import logger
22
23
24class SuiteRunnerTest(unittest.TestCase):
25  """Class of SuiteRunner test."""
26  real_logger = logger.GetLogger()
27
28  mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
29  mock_cmd_term = mock.Mock(spec=command_executer.CommandTerminator)
30  mock_logger = mock.Mock(spec=logger.Logger)
31  mock_label = label.MockLabel('lumpy', 'lumpy_chromeos_image', '/tmp/chromeos',
32                               'lumpy', ['lumpy1.cros', 'lumpy.cros2'], '', '',
33                               False, 'average', 'gcc', '')
34  telemetry_crosperf_bench = Benchmark('b1_test',  # name
35                                       'octane',  # test_name
36                                       '',  # test_args
37                                       3,  # iterations
38                                       False,  # rm_chroot_tmp
39                                       'record -e cycles',  # perf_args
40                                       'telemetry_Crosperf',  # suite
41                                       True)  # show_all_results
42
43  test_that_bench = Benchmark('b2_test',  # name
44                              'octane',  # test_name
45                              '',  # test_args
46                              3,  # iterations
47                              False,  # rm_chroot_tmp
48                              'record -e cycles')  # perf_args
49
50  telemetry_bench = Benchmark('b3_test',  # name
51                              'octane',  # test_name
52                              '',  # test_args
53                              3,  # iterations
54                              False,  # rm_chroot_tmp
55                              'record -e cycles',  # perf_args
56                              'telemetry',  # suite
57                              False)  # show_all_results
58
59  def __init__(self, *args, **kwargs):
60    super(SuiteRunnerTest, self).__init__(*args, **kwargs)
61    self.call_test_that_run = False
62    self.pin_governor_args = []
63    self.test_that_args = []
64    self.telemetry_run_args = []
65    self.telemetry_crosperf_args = []
66    self.call_telemetry_crosperf_run = False
67    self.call_pin_governor = False
68    self.call_telemetry_run = False
69
70  def setUp(self):
71    self.runner = suite_runner.SuiteRunner(
72        self.mock_logger, 'verbose', self.mock_cmd_exec, self.mock_cmd_term)
73
74  def test_get_profiler_args(self):
75    input_str = ('--profiler=custom_perf --profiler_args=\'perf_options'
76                 '="record -a -e cycles,instructions"\'')
77    output_str = ("profiler=custom_perf profiler_args='record -a -e "
78                  "cycles,instructions'")
79    res = suite_runner.GetProfilerArgs(input_str)
80    self.assertEqual(res, output_str)
81
82  def test_run(self):
83
84    def reset():
85      self.call_pin_governor = False
86      self.call_test_that_run = False
87      self.call_telemetry_run = False
88      self.call_telemetry_crosperf_run = False
89      self.pin_governor_args = []
90      self.test_that_args = []
91      self.telemetry_run_args = []
92      self.telemetry_crosperf_args = []
93
94    def FakePinGovernor(machine, chroot):
95      self.call_pin_governor = True
96      self.pin_governor_args = [machine, chroot]
97
98    def FakeTelemetryRun(machine, test_label, benchmark, profiler_args):
99      self.telemetry_run_args = [machine, test_label, benchmark, profiler_args]
100      self.call_telemetry_run = True
101      return 'Ran FakeTelemetryRun'
102
103    def FakeTelemetryCrosperfRun(machine, test_label, benchmark, test_args,
104                                 profiler_args):
105      self.telemetry_crosperf_args = [machine, test_label, benchmark, test_args,
106                                      profiler_args]
107      self.call_telemetry_crosperf_run = True
108      return 'Ran FakeTelemetryCrosperfRun'
109
110    def FakeTestThatRun(machine, test_label, benchmark,
111                        test_args, profiler_args):
112      self.test_that_args = [machine, test_label, benchmark,
113                             test_args, profiler_args
114                            ]
115      self.call_test_that_run = True
116      return 'Ran FakeTestThatRun'
117
118    self.runner.PinGovernorExecutionFrequencies = FakePinGovernor
119    self.runner.Telemetry_Run = FakeTelemetryRun
120    self.runner.Telemetry_Crosperf_Run = FakeTelemetryCrosperfRun
121    self.runner.Test_That_Run = FakeTestThatRun
122
123    machine = 'fake_machine'
124    test_args = ''
125    profiler_args = ''
126    reset()
127    self.runner.Run(machine, self.mock_label, self.telemetry_bench,
128                    test_args, profiler_args)
129    self.assertTrue(self.call_pin_governor)
130    self.assertTrue(self.call_telemetry_run)
131    self.assertFalse(self.call_test_that_run)
132    self.assertFalse(self.call_telemetry_crosperf_run)
133    self.assertEqual(
134        self.telemetry_run_args,
135        ['fake_machine', self.mock_label, self.telemetry_bench, ''])
136
137    reset()
138    self.runner.Run(machine, self.mock_label, self.test_that_bench,
139                    test_args, profiler_args)
140    self.assertTrue(self.call_pin_governor)
141    self.assertFalse(self.call_telemetry_run)
142    self.assertTrue(self.call_test_that_run)
143    self.assertFalse(self.call_telemetry_crosperf_run)
144    self.assertEqual(self.test_that_args, ['fake_machine', self.mock_label,
145                                           self.test_that_bench, '', ''])
146
147    reset()
148    self.runner.Run(machine, self.mock_label,
149                    self.telemetry_crosperf_bench, test_args,
150                    profiler_args)
151    self.assertTrue(self.call_pin_governor)
152    self.assertFalse(self.call_telemetry_run)
153    self.assertFalse(self.call_test_that_run)
154    self.assertTrue(self.call_telemetry_crosperf_run)
155    self.assertEqual(self.telemetry_crosperf_args,
156                     ['fake_machine', self.mock_label,
157                      self.telemetry_crosperf_bench, '', ''])
158
159  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommandWOutput')
160  def test_get_highest_static_frequency(self, mock_cros_runcmd):
161
162    self.mock_cmd_exec.CrosRunCommandWOutput = mock_cros_runcmd
163    mock_cros_runcmd.return_value = [0, '1666000 1333000 1000000', '']
164    freq = self.runner.GetHighestStaticFrequency('lumpy1.cros', '/tmp/chromeos')
165    self.assertEqual(freq, '1666000')
166
167    mock_cros_runcmd.return_value = [0, '1333000', '']
168    freq = self.runner.GetHighestStaticFrequency('lumpy1.cros', '/tmp/chromeos')
169    self.assertEqual(freq, '1333000')
170
171    mock_cros_runcmd.return_value = [0, '1661000 1333000 1000000', '']
172    freq = self.runner.GetHighestStaticFrequency('lumpy1.cros', '/tmp/chromeos')
173    self.assertEqual(freq, '1333000')
174
175  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand')
176  def test_pin_governor_execution_frequencies(self, mock_cros_runcmd):
177
178    def FakeGetHighestFreq(machine_name, chromeos_root):
179      if machine_name or chromeos_root:
180        pass
181      return '1666000'
182
183    self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd
184    self.runner.GetHighestStaticFrequency = FakeGetHighestFreq
185    self.runner.PinGovernorExecutionFrequencies('lumpy1.cros', '/tmp/chromeos')
186    self.assertEqual(mock_cros_runcmd.call_count, 1)
187    cmd = mock_cros_runcmd.call_args_list[0][0]
188    self.assertEqual(cmd, (
189        'set -e  && for f in '
190        '/sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq; do echo '
191        '1666000 > $f; done && for f in '
192        '/sys/devices/system/cpu/cpu*/cpufreq/scaling_min_freq; do echo '
193        '1666000 > $f; done && for f in '
194        '/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo '
195        'performance > $f; done',
196    ))
197
198  @mock.patch.object(time, 'sleep')
199  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand')
200  def test_reboot_machine(self, mock_cros_runcmd, mock_sleep):
201
202    def FakePinGovernor(machine_name, chromeos_root):
203      if machine_name or chromeos_root:
204        pass
205
206    self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd
207    self.runner.PinGovernorExecutionFrequencies = FakePinGovernor
208    self.runner.RebootMachine('lumpy1.cros', '/tmp/chromeos')
209    self.assertEqual(mock_cros_runcmd.call_count, 1)
210    self.assertEqual(mock_cros_runcmd.call_args_list[0][0], ('reboot && exit',))
211    self.assertEqual(mock_sleep.call_count, 1)
212    self.assertEqual(mock_sleep.call_args_list[0][0], (60,))
213
214  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand')
215  @mock.patch.object(command_executer.CommandExecuter,
216                     'ChrootRunCommandWOutput')
217  def test_test_that_run(self, mock_chroot_runcmd, mock_cros_runcmd):
218
219    def FakeRebootMachine(machine, chroot):
220      if machine or chroot:
221        pass
222
223    def FakeLogMsg(fd, termfd, msg, flush):
224      if fd or termfd or msg or flush:
225        pass
226
227    save_log_msg = self.real_logger._LogMsg
228    self.real_logger._LogMsg = FakeLogMsg
229    self.runner._logger = self.real_logger
230    self.runner.RebootMachine = FakeRebootMachine
231
232    raised_exception = False
233    try:
234      self.runner.Test_That_Run('lumpy1.cros', self.mock_label,
235                                self.test_that_bench, '', 'record -a -e cycles')
236    except:
237      raised_exception = True
238    self.assertTrue(raised_exception)
239
240    mock_chroot_runcmd.return_value = 0
241    self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd
242    self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd
243    res = self.runner.Test_That_Run('lumpy1.cros', self.mock_label,
244                                    self.test_that_bench, '--iterations=2', '')
245    self.assertEqual(mock_cros_runcmd.call_count, 1)
246    self.assertEqual(mock_chroot_runcmd.call_count, 1)
247    self.assertEqual(res, 0)
248    self.assertEqual(mock_cros_runcmd.call_args_list[0][0],
249                     ('rm -rf /usr/local/autotest/results/*',))
250    args_list = mock_chroot_runcmd.call_args_list[0][0]
251    args_dict = mock_chroot_runcmd.call_args_list[0][1]
252    self.assertEqual(len(args_list), 2)
253    self.assertEqual(args_list[0], '/tmp/chromeos')
254    self.assertEqual(args_list[1], ('/usr/bin/test_that --autotest_dir '
255                                    '~/trunk/src/third_party/autotest/files '
256                                    '--fast  --board=lumpy '
257                                    '--iterations=2 lumpy1.cros octane'))
258    self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term)
259    self.real_logger._LogMsg = save_log_msg
260
261  @mock.patch.object(os.path, 'isdir')
262  @mock.patch.object(command_executer.CommandExecuter,
263                     'ChrootRunCommandWOutput')
264  def test_telemetry_crosperf_run(self, mock_chroot_runcmd, mock_isdir):
265
266    mock_isdir.return_value = True
267    mock_chroot_runcmd.return_value = 0
268    self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd
269    profiler_args = ('--profiler=custom_perf --profiler_args=\'perf_options'
270                     '="record -a -e cycles,instructions"\'')
271    res = self.runner.Telemetry_Crosperf_Run('lumpy1.cros', self.mock_label,
272                                             self.telemetry_crosperf_bench, '',
273                                             profiler_args)
274    self.assertEqual(res, 0)
275    self.assertEqual(mock_chroot_runcmd.call_count, 1)
276    args_list = mock_chroot_runcmd.call_args_list[0][0]
277    args_dict = mock_chroot_runcmd.call_args_list[0][1]
278    self.assertEqual(args_list[0], '/tmp/chromeos')
279    self.assertEqual(args_list[1],
280                     ('/usr/bin/test_that --autotest_dir '
281                      '~/trunk/src/third_party/autotest/files '
282                      ' --board=lumpy --args=" run_local=False test=octane '
283                      'profiler=custom_perf profiler_args=\'record -a -e '
284                      'cycles,instructions\'" lumpy1.cros telemetry_Crosperf'))
285    self.assertEqual(args_dict['cros_sdk_options'],
286                     ('--no-ns-pid --chrome_root= '
287                      '--chrome_root_mount=/tmp/chrome_root '
288                      'FEATURES="-usersandbox" CHROME_ROOT=/tmp/chrome_root'))
289    self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term)
290    self.assertEqual(len(args_dict), 2)
291
292  @mock.patch.object(os.path, 'isdir')
293  @mock.patch.object(os.path, 'exists')
294  @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput')
295  def test_telemetry_run(self, mock_runcmd, mock_exists, mock_isdir):
296
297    def FakeLogMsg(fd, termfd, msg, flush):
298      if fd or termfd or msg or flush:
299        pass
300
301    save_log_msg = self.real_logger._LogMsg
302    self.real_logger._LogMsg = FakeLogMsg
303    mock_runcmd.return_value = 0
304
305    self.mock_cmd_exec.RunCommandWOutput = mock_runcmd
306    self.runner._logger = self.real_logger
307
308    profiler_args = ('--profiler=custom_perf --profiler_args=\'perf_options'
309                     '="record -a -e cycles,instructions"\'')
310
311    raises_exception = False
312    mock_isdir.return_value = False
313    try:
314      self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
315                                self.telemetry_bench, '')
316    except:
317      raises_exception = True
318    self.assertTrue(raises_exception)
319
320    raises_exception = False
321    mock_isdir.return_value = True
322    mock_exists.return_value = False
323    try:
324      self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
325                                self.telemetry_bench, '')
326    except:
327      raises_exception = True
328    self.assertTrue(raises_exception)
329
330    raises_exception = False
331    mock_isdir.return_value = True
332    mock_exists.return_value = True
333    try:
334      self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
335                                self.telemetry_bench, profiler_args)
336    except:
337      raises_exception = True
338    self.assertTrue(raises_exception)
339
340    test_flag.SetTestMode(True)
341    res = self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
342                                    self.telemetry_bench, '')
343    self.assertEqual(res, 0)
344    self.assertEqual(mock_runcmd.call_count, 1)
345    self.assertEqual(mock_runcmd.call_args_list[0][0], (
346        ('cd src/tools/perf && ./run_measurement '
347         '--browser=cros-chrome --output-format=csv '
348         '--remote=lumpy1.cros --identity /tmp/chromeos/src/scripts'
349         '/mod_for_test_scripts/ssh_keys/testing_rsa octane '),))
350
351    self.real_logger._LogMsg = save_log_msg
352
353
354if __name__ == '__main__':
355  unittest.main()
356