suite_runner_unittest.py revision f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe 
1#!/usr/bin/python
2#
3# Copyright 2014 Google Inc. All Rights Reserved.
4"""Unittest for machine_manager."""
5import os.path
6import time
7import hashlib
8
9import mock
10import unittest
11
12import suite_runner
13import machine_manager
14import image_checksummer
15import label
16import test_flag
17
18from benchmark import Benchmark
19from benchmark_run import MockBenchmarkRun
20
21from cros_utils import command_executer
22from cros_utils import logger
23
24
25class SuiteRunnerTest(unittest.TestCase):
26
27  real_logger = logger.GetLogger()
28
29  mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
30  mock_cmd_term = mock.Mock(spec=command_executer.CommandTerminator)
31  mock_logger = mock.Mock(spec=logger.Logger)
32  mock_label = label.MockLabel('lumpy', 'lumpy_chromeos_image', '/tmp/chromeos',
33                               'lumpy', ['lumpy1.cros', 'lumpy.cros2'], '', '',
34                               False, 'average', 'gcc', '')
35  telemetry_crosperf_bench = Benchmark('b1_test',  # name
36                                       'octane',  # test_name
37                                       '',  # test_args
38                                       3,  # iterations
39                                       False,  # rm_chroot_tmp
40                                       'record -e cycles',  # perf_args
41                                       'telemetry_Crosperf',  # suite
42                                       True)  # show_all_results
43
44  test_that_bench = Benchmark('b2_test',  # name
45                              'octane',  # test_name
46                              '',  # test_args
47                              3,  # iterations
48                              False,  # rm_chroot_tmp
49                              'record -e cycles')  # perf_args
50
51  telemetry_bench = Benchmark('b3_test',  # name
52                              'octane',  # test_name
53                              '',  # test_args
54                              3,  # iterations
55                              False,  # rm_chroot_tmp
56                              'record -e cycles',  # perf_args
57                              'telemetry',  # suite
58                              False)  # show_all_results
59
60  def setUp(self):
61    self.runner = suite_runner.SuiteRunner(
62        self.mock_logger, 'verbose', self.mock_cmd_exec, self.mock_cmd_term)
63
64  def test_get_profiler_args(self):
65    input_str = ('--profiler=custom_perf --profiler_args=\'perf_options'
66                 '="record -a -e cycles,instructions"\'')
67    output_str = ("profiler=custom_perf profiler_args='record -a -e "
68                  "cycles,instructions'")
69    res = suite_runner.GetProfilerArgs(input_str)
70    self.assertEqual(res, output_str)
71
72  def test_run(self):
73
74    def reset():
75      self.call_pin_governor = False
76      self.call_test_that_run = False
77      self.call_telemetry_run = False
78      self.call_telemetry_crosperf_run = False
79      self.pin_governor_args = []
80      self.test_that_args = []
81      self.telemetry_run_args = []
82      self.telemetry_crosperf_args = []
83
84    def FakePinGovernor(machine, chroot):
85      self.call_pin_governor = True
86      self.pin_governor_args = [machine, chroot]
87
88    def FakeTelemetryRun(machine, label, benchmark, profiler_args):
89      self.telemetry_run_args = [machine, label, benchmark, profiler_args]
90      self.call_telemetry_run = True
91      return 'Ran FakeTelemetryRun'
92
93    def FakeTelemetryCrosperfRun(machine, label, benchmark, test_args,
94                                 profiler_args):
95      self.telemetry_crosperf_args = [machine, label, benchmark, test_args,
96                                      profiler_args]
97      self.call_telemetry_crosperf_run = True
98      return 'Ran FakeTelemetryCrosperfRun'
99
100    def FakeTestThatRun(machine, label, benchmark, test_args, profiler_args):
101      self.test_that_args = [machine, label, benchmark, test_args, profiler_args
102                            ]
103      self.call_test_that_run = True
104      return 'Ran FakeTestThatRun'
105
106    self.runner.PinGovernorExecutionFrequencies = FakePinGovernor
107    self.runner.Telemetry_Run = FakeTelemetryRun
108    self.runner.Telemetry_Crosperf_Run = FakeTelemetryCrosperfRun
109    self.runner.Test_That_Run = FakeTestThatRun
110
111    machine = 'fake_machine'
112    test_args = ''
113    profiler_args = ''
114    reset()
115    res = self.runner.Run(machine, self.mock_label, self.telemetry_bench,
116                          test_args, profiler_args)
117    self.assertTrue(self.call_pin_governor)
118    self.assertTrue(self.call_telemetry_run)
119    self.assertFalse(self.call_test_that_run)
120    self.assertFalse(self.call_telemetry_crosperf_run)
121    self.assertEqual(
122        self.telemetry_run_args,
123        ['fake_machine', self.mock_label, self.telemetry_bench, ''])
124
125    reset()
126    res = self.runner.Run(machine, self.mock_label, self.test_that_bench,
127                          test_args, profiler_args)
128    self.assertTrue(self.call_pin_governor)
129    self.assertFalse(self.call_telemetry_run)
130    self.assertTrue(self.call_test_that_run)
131    self.assertFalse(self.call_telemetry_crosperf_run)
132    self.assertEqual(self.test_that_args, ['fake_machine', self.mock_label,
133                                           self.test_that_bench, '', ''])
134
135    reset()
136    res = self.runner.Run(machine, self.mock_label,
137                          self.telemetry_crosperf_bench, test_args,
138                          profiler_args)
139    self.assertTrue(self.call_pin_governor)
140    self.assertFalse(self.call_telemetry_run)
141    self.assertFalse(self.call_test_that_run)
142    self.assertTrue(self.call_telemetry_crosperf_run)
143    self.assertEqual(self.telemetry_crosperf_args,
144                     ['fake_machine', self.mock_label,
145                      self.telemetry_crosperf_bench, '', ''])
146
147  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommandWOutput')
148  def test_get_highest_static_frequency(self, mock_cros_runcmd):
149
150    self.mock_cmd_exec.CrosRunCommandWOutput = mock_cros_runcmd
151    mock_cros_runcmd.return_value = [0, '1666000 1333000 1000000', '']
152    freq = self.runner.GetHighestStaticFrequency('lumpy1.cros', '/tmp/chromeos')
153    self.assertEqual(freq, '1666000')
154
155    mock_cros_runcmd.return_value = [0, '1333000', '']
156    freq = self.runner.GetHighestStaticFrequency('lumpy1.cros', '/tmp/chromeos')
157    self.assertEqual(freq, '1333000')
158
159    mock_cros_runcmd.return_value = [0, '1661000 1333000 1000000', '']
160    freq = self.runner.GetHighestStaticFrequency('lumpy1.cros', '/tmp/chromeos')
161    self.assertEqual(freq, '1333000')
162
163  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand')
164  def test_pin_governor_execution_frequencies(self, mock_cros_runcmd):
165
166    def FakeGetHighestFreq(machine_name, chromeos_root):
167      return '1666000'
168
169    self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd
170    self.runner.GetHighestStaticFrequency = FakeGetHighestFreq
171    self.runner.PinGovernorExecutionFrequencies('lumpy1.cros', '/tmp/chromeos')
172    self.assertEqual(mock_cros_runcmd.call_count, 1)
173    cmd = mock_cros_runcmd.call_args_list[0][0]
174    self.assertEqual(cmd, (
175        'set -e  && for f in '
176                     '/sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq; do echo '
177                     '1666000 > $f; done && for f in '
178                     '/sys/devices/system/cpu/cpu*/cpufreq/scaling_min_freq; do echo '
179                     '1666000 > $f; done && for f in '
180                     '/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo '
181                     'performance > $f; done',
182    ))
183
184  @mock.patch.object(time, 'sleep')
185  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand')
186  def test_reboot_machine(self, mock_cros_runcmd, mock_sleep):
187
188    def FakePinGovernor(machine_name, chromeos_root):
189      pass
190
191    self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd
192    self.runner.PinGovernorExecutionFrequencies = FakePinGovernor
193    self.runner.RebootMachine('lumpy1.cros', '/tmp/chromeos')
194    self.assertEqual(mock_cros_runcmd.call_count, 1)
195    self.assertEqual(mock_cros_runcmd.call_args_list[0][0], ('reboot && exit',))
196    self.assertEqual(mock_sleep.call_count, 1)
197    self.assertEqual(mock_sleep.call_args_list[0][0], (60,))
198
199  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand')
200  @mock.patch.object(command_executer.CommandExecuter,
201                     'ChrootRunCommandWOutput')
202  def test_test_that_run(self, mock_chroot_runcmd, mock_cros_runcmd):
203
204    def FakeRebootMachine(machine, chroot):
205      pass
206
207    def FakeLogMsg(fd, termfd, msg, flush):
208      pass
209
210    save_log_msg = self.real_logger._LogMsg
211    self.real_logger._LogMsg = FakeLogMsg
212    self.runner._logger = self.real_logger
213    self.runner.RebootMachine = FakeRebootMachine
214
215    raised_exception = False
216    try:
217      self.runner.Test_That_Run('lumpy1.cros', self.mock_label,
218                                self.test_that_bench, '', 'record -a -e cycles')
219    except:
220      raised_exception = True
221    self.assertTrue(raised_exception)
222
223    mock_chroot_runcmd.return_value = 0
224    self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd
225    self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd
226    res = self.runner.Test_That_Run('lumpy1.cros', self.mock_label,
227                                    self.test_that_bench, '--iterations=2', '')
228    self.assertEqual(mock_cros_runcmd.call_count, 1)
229    self.assertEqual(mock_chroot_runcmd.call_count, 1)
230    self.assertEqual(res, 0)
231    self.assertEqual(mock_cros_runcmd.call_args_list[0][0],
232                     ('rm -rf /usr/local/autotest/results/*',))
233    args_list = mock_chroot_runcmd.call_args_list[0][0]
234    args_dict = mock_chroot_runcmd.call_args_list[0][1]
235    self.assertEqual(len(args_list), 2)
236    self.assertEqual(args_list[0], '/tmp/chromeos')
237    self.assertEqual(args_list[1], ('/usr/bin/test_that --autotest_dir '
238                                    '~/trunk/src/third_party/autotest/files '
239                                    '--fast  --board=lumpy '
240                                    '--iterations=2 lumpy1.cros octane'))
241    self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term)
242    self.real_logger._LogMsg = save_log_msg
243
244  @mock.patch.object(os.path, 'isdir')
245  @mock.patch.object(command_executer.CommandExecuter,
246                     'ChrootRunCommandWOutput')
247  def test_telemetry_crosperf_run(self, mock_chroot_runcmd, mock_isdir):
248
249    mock_isdir.return_value = True
250    mock_chroot_runcmd.return_value = 0
251    self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd
252    profiler_args = ('--profiler=custom_perf --profiler_args=\'perf_options'
253                     '="record -a -e cycles,instructions"\'')
254    res = self.runner.Telemetry_Crosperf_Run('lumpy1.cros', self.mock_label,
255                                             self.telemetry_crosperf_bench, '',
256                                             profiler_args)
257    self.assertEqual(res, 0)
258    self.assertEqual(mock_chroot_runcmd.call_count, 1)
259    args_list = mock_chroot_runcmd.call_args_list[0][0]
260    args_dict = mock_chroot_runcmd.call_args_list[0][1]
261    self.assertEqual(args_list[0], '/tmp/chromeos')
262    self.assertEqual(args_list[1],
263                     ('/usr/bin/test_that --autotest_dir '
264                      '~/trunk/src/third_party/autotest/files '
265                      ' --board=lumpy --args=" run_local=False test=octane '
266                      'profiler=custom_perf profiler_args=\'record -a -e '
267                      'cycles,instructions\'" lumpy1.cros telemetry_Crosperf'))
268    self.assertEqual(args_dict['cros_sdk_options'],
269                     ('--no-ns-pid --chrome_root= '
270                      '--chrome_root_mount=/tmp/chrome_root '
271                      'FEATURES="-usersandbox" CHROME_ROOT=/tmp/chrome_root'))
272    self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term)
273    self.assertEqual(len(args_dict), 2)
274
275  @mock.patch.object(os.path, 'isdir')
276  @mock.patch.object(os.path, 'exists')
277  @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput')
278  def test_telemetry_run(self, mock_runcmd, mock_exists, mock_isdir):
279
280    def FakeLogMsg(fd, termfd, msg, flush):
281      pass
282
283    save_log_msg = self.real_logger._LogMsg
284    self.real_logger._LogMsg = FakeLogMsg
285    mock_runcmd.return_value = 0
286
287    self.mock_cmd_exec.RunCommandWOutput = mock_runcmd
288    self.runner._logger = self.real_logger
289
290    profiler_args = ('--profiler=custom_perf --profiler_args=\'perf_options'
291                     '="record -a -e cycles,instructions"\'')
292
293    raises_exception = False
294    mock_isdir.return_value = False
295    try:
296      self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
297                                self.telemetry_bench, '')
298    except:
299      raises_exception = True
300    self.assertTrue(raises_exception)
301
302    raises_exception = False
303    mock_isdir.return_value = True
304    mock_exists.return_value = False
305    try:
306      self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
307                                self.telemetry_bench, '')
308    except:
309      raises_exception = True
310    self.assertTrue(raises_exception)
311
312    raises_exception = False
313    mock_isdir.return_value = True
314    mock_exists.return_value = True
315    try:
316      self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
317                                self.telemetry_bench, profiler_args)
318    except:
319      raises_exception = True
320    self.assertTrue(raises_exception)
321
322    test_flag.SetTestMode(True)
323    res = self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
324                                    self.telemetry_bench, '')
325    self.assertEqual(res, 0)
326    self.assertEqual(mock_runcmd.call_count, 1)
327    self.assertEqual(mock_runcmd.call_args_list[0][0], (
328        ('cd src/tools/perf && ./run_measurement '
329         '--browser=cros-chrome --output-format=csv '
330         '--remote=lumpy1.cros --identity /tmp/chromeos/src/scripts'
331         '/mod_for_test_scripts/ssh_keys/testing_rsa octane '),))
332
333    self.real_logger._LogMsg = save_log_msg
334
335
336if __name__ == '__main__':
337  unittest.main()
338