suite_runner_unittest.py revision 3fb928efb42405e386da41adf4bd674e86e23022
1#!/usr/bin/env python2 2# 3# Copyright 2014 Google Inc. All Rights Reserved. 4"""Unittest for suite_runner.""" 5 6from __future__ import print_function 7 8import os.path 9import time 10 11import mock 12import unittest 13 14import suite_runner 15import label 16import test_flag 17 18from benchmark import Benchmark 19 20from cros_utils import command_executer 21from cros_utils import logger 22 23 24class SuiteRunnerTest(unittest.TestCase): 25 """Class of SuiteRunner test.""" 26 real_logger = logger.GetLogger() 27 28 mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) 29 mock_cmd_term = mock.Mock(spec=command_executer.CommandTerminator) 30 mock_logger = mock.Mock(spec=logger.Logger) 31 mock_label = label.MockLabel('lumpy', 'lumpy_chromeos_image', '', 32 '/tmp/chromeos', 'lumpy', 33 ['lumpy1.cros', 'lumpy.cros2'], '', '', False, 34 'average', 'gcc', '') 35 telemetry_crosperf_bench = Benchmark( 36 'b1_test', # name 37 'octane', # test_name 38 '', # test_args 39 3, # iterations 40 False, # rm_chroot_tmp 41 'record -e cycles', # perf_args 42 'telemetry_Crosperf', # suite 43 True) # show_all_results 44 45 test_that_bench = Benchmark( 46 'b2_test', # name 47 'octane', # test_name 48 '', # test_args 49 3, # iterations 50 False, # rm_chroot_tmp 51 'record -e cycles') # perf_args 52 53 telemetry_bench = Benchmark( 54 'b3_test', # name 55 'octane', # test_name 56 '', # test_args 57 3, # iterations 58 False, # rm_chroot_tmp 59 'record -e cycles', # perf_args 60 'telemetry', # suite 61 False) # show_all_results 62 63 def __init__(self, *args, **kwargs): 64 super(SuiteRunnerTest, self).__init__(*args, **kwargs) 65 self.call_test_that_run = False 66 self.pin_governor_args = [] 67 self.test_that_args = [] 68 self.telemetry_run_args = [] 69 self.telemetry_crosperf_args = [] 70 self.call_telemetry_crosperf_run = False 71 self.call_pin_governor = False 72 self.call_telemetry_run = False 73 74 def setUp(self): 75 self.runner = suite_runner.SuiteRunner(self.mock_logger, 'verbose', 76 self.mock_cmd_exec, 77 self.mock_cmd_term) 78 79 def test_get_profiler_args(self): 80 input_str = ('--profiler=custom_perf --profiler_args=\'perf_options' 81 '="record -a -e cycles,instructions"\'') 82 output_str = ("profiler=custom_perf profiler_args='record -a -e " 83 "cycles,instructions'") 84 res = suite_runner.GetProfilerArgs(input_str) 85 self.assertEqual(res, output_str) 86 87 def test_run(self): 88 89 def reset(): 90 self.call_pin_governor = False 91 self.call_test_that_run = False 92 self.call_telemetry_run = False 93 self.call_telemetry_crosperf_run = False 94 self.pin_governor_args = [] 95 self.test_that_args = [] 96 self.telemetry_run_args = [] 97 self.telemetry_crosperf_args = [] 98 99 def FakePinGovernor(machine, chroot): 100 self.call_pin_governor = True 101 self.pin_governor_args = [machine, chroot] 102 103 def FakeTelemetryRun(machine, test_label, benchmark, profiler_args): 104 self.telemetry_run_args = [machine, test_label, benchmark, profiler_args] 105 self.call_telemetry_run = True 106 return 'Ran FakeTelemetryRun' 107 108 def FakeTelemetryCrosperfRun(machine, test_label, benchmark, test_args, 109 profiler_args): 110 self.telemetry_crosperf_args = [ 111 machine, test_label, benchmark, test_args, profiler_args 112 ] 113 self.call_telemetry_crosperf_run = True 114 return 'Ran FakeTelemetryCrosperfRun' 115 116 def FakeTestThatRun(machine, test_label, benchmark, test_args, 117 profiler_args): 118 self.test_that_args = [ 119 machine, test_label, benchmark, test_args, profiler_args 120 ] 121 self.call_test_that_run = True 122 return 'Ran FakeTestThatRun' 123 124 self.runner.PinGovernorExecutionFrequencies = FakePinGovernor 125 self.runner.Telemetry_Run = FakeTelemetryRun 126 self.runner.Telemetry_Crosperf_Run = FakeTelemetryCrosperfRun 127 self.runner.Test_That_Run = FakeTestThatRun 128 129 machine = 'fake_machine' 130 test_args = '' 131 profiler_args = '' 132 reset() 133 self.runner.Run(machine, self.mock_label, self.telemetry_bench, test_args, 134 profiler_args) 135 self.assertTrue(self.call_pin_governor) 136 self.assertTrue(self.call_telemetry_run) 137 self.assertFalse(self.call_test_that_run) 138 self.assertFalse(self.call_telemetry_crosperf_run) 139 self.assertEqual( 140 self.telemetry_run_args, 141 ['fake_machine', self.mock_label, self.telemetry_bench, '']) 142 143 reset() 144 self.runner.Run(machine, self.mock_label, self.test_that_bench, test_args, 145 profiler_args) 146 self.assertTrue(self.call_pin_governor) 147 self.assertFalse(self.call_telemetry_run) 148 self.assertTrue(self.call_test_that_run) 149 self.assertFalse(self.call_telemetry_crosperf_run) 150 self.assertEqual( 151 self.test_that_args, 152 ['fake_machine', self.mock_label, self.test_that_bench, '', '']) 153 154 reset() 155 self.runner.Run(machine, self.mock_label, self.telemetry_crosperf_bench, 156 test_args, profiler_args) 157 self.assertTrue(self.call_pin_governor) 158 self.assertFalse(self.call_telemetry_run) 159 self.assertFalse(self.call_test_that_run) 160 self.assertTrue(self.call_telemetry_crosperf_run) 161 self.assertEqual(self.telemetry_crosperf_args, [ 162 'fake_machine', self.mock_label, self.telemetry_crosperf_bench, '', '' 163 ]) 164 165 @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommandWOutput') 166 def test_get_highest_static_frequency(self, mock_cros_runcmd): 167 168 self.mock_cmd_exec.CrosRunCommandWOutput = mock_cros_runcmd 169 mock_cros_runcmd.return_value = [0, '1666000 1333000 1000000', ''] 170 freq = self.runner.GetHighestStaticFrequency('lumpy1.cros', '/tmp/chromeos') 171 self.assertEqual(freq, '1666000') 172 173 mock_cros_runcmd.return_value = [0, '1333000', ''] 174 freq = self.runner.GetHighestStaticFrequency('lumpy1.cros', '/tmp/chromeos') 175 self.assertEqual(freq, '1333000') 176 177 mock_cros_runcmd.return_value = [0, '1661000 1333000 1000000', ''] 178 freq = self.runner.GetHighestStaticFrequency('lumpy1.cros', '/tmp/chromeos') 179 self.assertEqual(freq, '1333000') 180 181 @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand') 182 def test_pin_governor_execution_frequencies(self, mock_cros_runcmd): 183 184 def FakeGetHighestFreq(machine_name, chromeos_root): 185 if machine_name or chromeos_root: 186 pass 187 return '1666000' 188 189 self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd 190 self.runner.GetHighestStaticFrequency = FakeGetHighestFreq 191 self.runner.PinGovernorExecutionFrequencies('lumpy1.cros', '/tmp/chromeos') 192 self.assertEqual(mock_cros_runcmd.call_count, 1) 193 cmd = mock_cros_runcmd.call_args_list[0][0] 194 self.assertEqual(cmd, ( 195 'set -e && for f in ' 196 '/sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq; do echo ' 197 '1666000 > $f; done && for f in ' 198 '/sys/devices/system/cpu/cpu*/cpufreq/scaling_min_freq; do echo ' 199 '1666000 > $f; done && for f in ' 200 '/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo ' 201 'performance > $f; done',)) 202 203 @mock.patch.object(time, 'sleep') 204 @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand') 205 def test_reboot_machine(self, mock_cros_runcmd, mock_sleep): 206 207 def FakePinGovernor(machine_name, chromeos_root): 208 if machine_name or chromeos_root: 209 pass 210 211 self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd 212 self.runner.PinGovernorExecutionFrequencies = FakePinGovernor 213 self.runner.RebootMachine('lumpy1.cros', '/tmp/chromeos') 214 self.assertEqual(mock_cros_runcmd.call_count, 1) 215 self.assertEqual(mock_cros_runcmd.call_args_list[0][0], ('reboot && exit',)) 216 self.assertEqual(mock_sleep.call_count, 1) 217 self.assertEqual(mock_sleep.call_args_list[0][0], (60,)) 218 219 @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand') 220 @mock.patch.object(command_executer.CommandExecuter, 221 'ChrootRunCommandWOutput') 222 def test_test_that_run(self, mock_chroot_runcmd, mock_cros_runcmd): 223 224 def FakeRebootMachine(machine, chroot): 225 if machine or chroot: 226 pass 227 228 def FakeLogMsg(fd, termfd, msg, flush=True): 229 if fd or termfd or msg or flush: 230 pass 231 232 save_log_msg = self.real_logger.LogMsg 233 self.real_logger.LogMsg = FakeLogMsg 234 self.runner.logger = self.real_logger 235 self.runner.RebootMachine = FakeRebootMachine 236 237 raised_exception = False 238 try: 239 self.runner.Test_That_Run('lumpy1.cros', self.mock_label, 240 self.test_that_bench, '', 'record -a -e cycles') 241 except SystemExit: 242 raised_exception = True 243 self.assertTrue(raised_exception) 244 245 mock_chroot_runcmd.return_value = 0 246 self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd 247 self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd 248 res = self.runner.Test_That_Run('lumpy1.cros', self.mock_label, 249 self.test_that_bench, '--iterations=2', '') 250 self.assertEqual(mock_cros_runcmd.call_count, 1) 251 self.assertEqual(mock_chroot_runcmd.call_count, 1) 252 self.assertEqual(res, 0) 253 self.assertEqual(mock_cros_runcmd.call_args_list[0][0], 254 ('rm -rf /usr/local/autotest/results/*',)) 255 args_list = mock_chroot_runcmd.call_args_list[0][0] 256 args_dict = mock_chroot_runcmd.call_args_list[0][1] 257 self.assertEqual(len(args_list), 2) 258 self.assertEqual(args_list[0], '/tmp/chromeos') 259 self.assertEqual(args_list[1], ('/usr/bin/test_that ' 260 '--fast --board=lumpy ' 261 '--iterations=2 lumpy1.cros octane')) 262 self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) 263 self.real_logger.LogMsg = save_log_msg 264 265 @mock.patch.object(os.path, 'isdir') 266 @mock.patch.object(command_executer.CommandExecuter, 267 'ChrootRunCommandWOutput') 268 def test_telemetry_crosperf_run(self, mock_chroot_runcmd, mock_isdir): 269 270 mock_isdir.return_value = True 271 mock_chroot_runcmd.return_value = 0 272 self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd 273 profiler_args = ('--profiler=custom_perf --profiler_args=\'perf_options' 274 '="record -a -e cycles,instructions"\'') 275 res = self.runner.Telemetry_Crosperf_Run('lumpy1.cros', self.mock_label, 276 self.telemetry_crosperf_bench, '', 277 profiler_args) 278 self.assertEqual(res, 0) 279 self.assertEqual(mock_chroot_runcmd.call_count, 1) 280 args_list = mock_chroot_runcmd.call_args_list[0][0] 281 args_dict = mock_chroot_runcmd.call_args_list[0][1] 282 self.assertEqual(args_list[0], '/tmp/chromeos') 283 self.assertEqual(args_list[1], 284 ('/usr/bin/test_that --autotest_dir ' 285 '~/trunk/src/third_party/autotest/files ' 286 ' --board=lumpy --args=" run_local=False test=octane ' 287 'profiler=custom_perf profiler_args=\'record -a -e ' 288 'cycles,instructions\'" lumpy1.cros telemetry_Crosperf')) 289 self.assertEqual(args_dict['cros_sdk_options'], 290 ('--no-ns-pid --chrome_root= ' 291 '--chrome_root_mount=/tmp/chrome_root ' 292 'FEATURES="-usersandbox" CHROME_ROOT=/tmp/chrome_root')) 293 self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term) 294 self.assertEqual(len(args_dict), 2) 295 296 @mock.patch.object(os.path, 'isdir') 297 @mock.patch.object(os.path, 'exists') 298 @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput') 299 def test_telemetry_run(self, mock_runcmd, mock_exists, mock_isdir): 300 301 def FakeLogMsg(fd, termfd, msg, flush=True): 302 if fd or termfd or msg or flush: 303 pass 304 305 save_log_msg = self.real_logger.LogMsg 306 self.real_logger.LogMsg = FakeLogMsg 307 mock_runcmd.return_value = 0 308 309 self.mock_cmd_exec.RunCommandWOutput = mock_runcmd 310 self.runner.logger = self.real_logger 311 312 profiler_args = ('--profiler=custom_perf --profiler_args=\'perf_options' 313 '="record -a -e cycles,instructions"\'') 314 315 raises_exception = False 316 mock_isdir.return_value = False 317 try: 318 self.runner.Telemetry_Run('lumpy1.cros', self.mock_label, 319 self.telemetry_bench, '') 320 except SystemExit: 321 raises_exception = True 322 self.assertTrue(raises_exception) 323 324 raises_exception = False 325 mock_isdir.return_value = True 326 mock_exists.return_value = False 327 try: 328 self.runner.Telemetry_Run('lumpy1.cros', self.mock_label, 329 self.telemetry_bench, '') 330 except SystemExit: 331 raises_exception = True 332 self.assertTrue(raises_exception) 333 334 raises_exception = False 335 mock_isdir.return_value = True 336 mock_exists.return_value = True 337 try: 338 self.runner.Telemetry_Run('lumpy1.cros', self.mock_label, 339 self.telemetry_bench, profiler_args) 340 except SystemExit: 341 raises_exception = True 342 self.assertTrue(raises_exception) 343 344 test_flag.SetTestMode(True) 345 res = self.runner.Telemetry_Run('lumpy1.cros', self.mock_label, 346 self.telemetry_bench, '') 347 self.assertEqual(res, 0) 348 self.assertEqual(mock_runcmd.call_count, 1) 349 self.assertEqual(mock_runcmd.call_args_list[0][0], ( 350 ('cd src/tools/perf && ./run_measurement ' 351 '--browser=cros-chrome --output-format=csv ' 352 '--remote=lumpy1.cros --identity /tmp/chromeos/src/scripts' 353 '/mod_for_test_scripts/ssh_keys/testing_rsa octane '),)) 354 355 self.real_logger.LogMsg = save_log_msg 356 357 358if __name__ == '__main__': 359 unittest.main() 360