benchmark_run_unittest.py revision 7057cf67ba1dbdd4387f53e5fe47b43c955b1a53
1#!/usr/bin/python 2 3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Testing of benchmark_run.""" 8 9import mock 10import unittest 11import inspect 12 13from cros_utils import logger 14 15import benchmark_run 16 17from suite_runner import MockSuiteRunner 18from suite_runner import SuiteRunner 19from label import MockLabel 20from benchmark import Benchmark 21from machine_manager import MockMachineManager 22from machine_manager import MachineManager 23from machine_manager import MockCrosMachine 24from results_cache import MockResultsCache 25from results_cache import CacheConditions 26from results_cache import Result 27from results_cache import ResultsCache 28 29 30class BenchmarkRunTest(unittest.TestCase): 31 """ 32 Unit tests for the BenchmarkRun class and all of its methods. 33 """ 34 35 def setUp(self): 36 self.test_benchmark = Benchmark("page_cycler.netsim.top_10", # name 37 "page_cycler.netsim.top_10", # test_name 38 "", # test_args 39 1, # iterations 40 False, # rm_chroot_tmp 41 "", # perf_args 42 suite="telemetry_Crosperf") # suite 43 44 self.test_label = MockLabel("test1", "image1", "/tmp/test_benchmark_run", 45 "x86-alex", "chromeos2-row1-rack4-host9.cros", 46 image_args="", cache_dir="", cache_only=False, 47 log_level="average", compiler="gcc") 48 49 50 self.test_cache_conditions = [CacheConditions.CACHE_FILE_EXISTS, 51 CacheConditions.CHECKSUMS_MATCH] 52 53 self.mock_logger = logger.GetLogger(log_dir="", mock=True) 54 55 self.mock_machine_manager = mock.Mock(spec=MachineManager) 56 57 def testDryRun(self): 58 my_label = MockLabel("test1", "image1", "/tmp/test_benchmark_run", 59 "x86-alex", "chromeos2-row1-rack4-host9.cros", 60 image_args="", cache_dir="", cache_only=False, 61 log_level="average", compiler="gcc") 62 63 logging_level = "average" 64 m = MockMachineManager("/tmp/chromeos_root", 0, logging_level, '') 65 m.AddMachine("chromeos2-row1-rack4-host9.cros") 66 bench = Benchmark("page_cycler.netsim.top_10", # name 67 "page_cycler.netsim.top_10", # test_name 68 "", # test_args 69 1, # iterations 70 False, # rm_chroot_tmp 71 "", # perf_args 72 suite="telemetry_Crosperf") # suite 73 b = benchmark_run.MockBenchmarkRun("test run", 74 bench, 75 my_label, 76 1, 77 [], 78 m, 79 logger.GetLogger(), 80 logging_level, 81 "") 82 b.cache = MockResultsCache() 83 b.suite_runner = MockSuiteRunner() 84 b.start() 85 86 # Make sure the arguments to BenchmarkRun.__init__ have not changed 87 # since the last time this test was updated: 88 args_list = ['self', 'name', 'benchmark', 'label', 'iteration', 89 'cache_conditions', 'machine_manager', 'logger_to_use', 90 'log_level', 'share_cache'] 91 arg_spec = inspect.getargspec(benchmark_run.BenchmarkRun.__init__) 92 self.assertEqual(len(arg_spec.args), len(args_list)) 93 self.assertEqual (arg_spec.args, args_list) 94 95 96 def test_init(self): 97 # Nothing really worth testing here; just field assignments. 98 pass 99 100 def test_read_cache(self): 101 # Nothing really worth testing here, either. 102 pass 103 104 def test_run(self): 105 br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark, 106 self.test_label, 1, self.test_cache_conditions, 107 self.mock_machine_manager, 108 self.mock_logger, 109 "average", "") 110 111 def MockLogOutput(msg, print_to_console=False): 112 "Helper function for test_run." 113 self.log_output.append(msg) 114 115 def MockLogError(msg, print_to_console=False): 116 "Helper function for test_run." 117 self.log_error.append(msg) 118 119 def MockRecordStatus(msg): 120 "Helper function for test_run." 121 self.status.append(msg) 122 123 def FakeReadCache(): 124 "Helper function for test_run." 125 br.cache = mock.Mock(spec=ResultsCache) 126 self.called_ReadCache = True 127 return 0 128 129 def FakeReadCacheSucceed(): 130 "Helper function for test_run." 131 br.cache = mock.Mock(spec=ResultsCache) 132 br.result = mock.Mock(spec=Result) 133 br.result.out = "result.out stuff" 134 br.result.err = "result.err stuff" 135 br.result.retval = 0 136 self.called_ReadCache = True 137 return 0 138 139 def FakeReadCacheException(): 140 "Helper function for test_run." 141 raise Exception("This is an exception test; it is supposed to happen") 142 143 def FakeAcquireMachine(): 144 "Helper function for test_run." 145 mock_machine = MockCrosMachine ('chromeos1-row3-rack5-host7.cros', 146 'chromeos', 'average') 147 return mock_machine 148 149 def FakeRunTest(_machine): 150 "Helper function for test_run." 151 mock_result = mock.Mock(spec=Result) 152 mock_result.retval = 0 153 return mock_result 154 155 def FakeRunTestFail(_machine): 156 "Helper function for test_run." 157 mock_result = mock.Mock(spec=Result) 158 mock_result.retval = 1 159 return mock_result 160 161 def ResetTestValues(): 162 "Helper function for test_run." 163 self.log_output = [] 164 self.log_error = [] 165 self.status = [] 166 br.result = None 167 self.called_ReadCache = False 168 169 # Assign all the fake functions to the appropriate objects. 170 br._logger.LogOutput = MockLogOutput 171 br._logger.LogError = MockLogError 172 br.timeline.Record = MockRecordStatus 173 br.ReadCache = FakeReadCache 174 br.RunTest = FakeRunTest 175 br.AcquireMachine = FakeAcquireMachine 176 177 # First test: No cache hit, all goes well. 178 ResetTestValues() 179 br.run() 180 self.assertTrue (self.called_ReadCache) 181 self.assertEqual (self.log_output, 182 ['test_run: No cache hit.', 183 'Releasing machine: chromeos1-row3-rack5-host7.cros', 184 'Released machine: chromeos1-row3-rack5-host7.cros']) 185 self.assertEqual (len(self.log_error), 0) 186 self.assertEqual (self.status, ['WAITING', 'SUCCEEDED']) 187 188 # Second test: No cached result found; test run was "terminated" for some 189 # reason. 190 ResetTestValues() 191 br.terminated = True 192 br.run() 193 self.assertTrue (self.called_ReadCache) 194 self.assertEqual (self.log_output, 195 ['test_run: No cache hit.', 196 'Releasing machine: chromeos1-row3-rack5-host7.cros', 197 'Released machine: chromeos1-row3-rack5-host7.cros']) 198 self.assertEqual (len(self.log_error), 0) 199 self.assertEqual (self.status, ['WAITING']) 200 201 # Third test. No cached result found; RunTest failed for some reason. 202 ResetTestValues() 203 br.terminated = False 204 br.RunTest = FakeRunTestFail 205 br.run() 206 self.assertTrue (self.called_ReadCache) 207 self.assertEqual (self.log_output, 208 ['test_run: No cache hit.', 209 'Releasing machine: chromeos1-row3-rack5-host7.cros', 210 'Released machine: chromeos1-row3-rack5-host7.cros']) 211 self.assertEqual (len(self.log_error), 0) 212 self.assertEqual (self.status, ['WAITING', 'FAILED']) 213 214 # Fourth test: ReadCache found a cached result. 215 ResetTestValues() 216 br.RunTest = FakeRunTest 217 br.ReadCache = FakeReadCacheSucceed 218 br.run() 219 self.assertTrue (self.called_ReadCache) 220 self.assertEqual (self.log_output, 221 ['test_run: Cache hit.', 222 'result.out stuff', 223 'Releasing machine: chromeos1-row3-rack5-host7.cros', 224 'Released machine: chromeos1-row3-rack5-host7.cros']) 225 self.assertEqual (self.log_error, ['result.err stuff']) 226 self.assertEqual (self.status, ['SUCCEEDED']) 227 228 # Fifth test: ReadCache generates an exception; does the try/finally block 229 # work? 230 ResetTestValues() 231 br.ReadCache = FakeReadCacheException 232 br.machine = FakeAcquireMachine() 233 br.run() 234 self.assertEqual (self.log_output, 235 ['Releasing machine: chromeos1-row3-rack5-host7.cros', 236 'Released machine: chromeos1-row3-rack5-host7.cros']) 237 self.assertEqual (self.log_error, 238 ["Benchmark run: 'test_run' failed: This is an exception test; it is supposed to happen"]) 239 self.assertEqual (self.status, ['FAILED']) 240 241 242 def test_terminate_pass(self): 243 br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark, 244 self.test_label, 1, self.test_cache_conditions, 245 self.mock_machine_manager, 246 self.mock_logger, 247 "average", "") 248 249 def GetLastEventPassed(): 250 "Helper function for test_terminate_pass" 251 return benchmark_run.STATUS_SUCCEEDED 252 253 def RecordStub(status): 254 "Helper function for test_terminate_pass" 255 self.status = status 256 257 self.status = benchmark_run.STATUS_SUCCEEDED 258 self.assertFalse (br.terminated) 259 self.assertFalse (br.suite_runner._ct.IsTerminated()) 260 261 br.timeline.GetLastEvent = GetLastEventPassed 262 br.timeline.Record = RecordStub 263 264 br.Terminate() 265 266 self.assertTrue (br.terminated) 267 self.assertTrue (br.suite_runner._ct.IsTerminated()) 268 self.assertEqual (self.status, benchmark_run.STATUS_FAILED) 269 270 271 272 def test_terminate_fail(self): 273 br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark, 274 self.test_label, 1, self.test_cache_conditions, 275 self.mock_machine_manager, 276 self.mock_logger, 277 "average", "") 278 279 def GetLastEventFailed(): 280 "Helper function for test_terminate_fail" 281 return benchmark_run.STATUS_FAILED 282 283 def RecordStub(status): 284 "Helper function for test_terminate_fail" 285 self.status = status 286 287 self.status = benchmark_run.STATUS_SUCCEEDED 288 self.assertFalse (br.terminated) 289 self.assertFalse (br.suite_runner._ct.IsTerminated()) 290 291 br.timeline.GetLastEvent = GetLastEventFailed 292 br.timeline.Record = RecordStub 293 294 br.Terminate() 295 296 self.assertTrue (br.terminated) 297 self.assertTrue (br.suite_runner._ct.IsTerminated()) 298 self.assertEqual (self.status, benchmark_run.STATUS_SUCCEEDED) 299 300 301 def test_acquire_machine(self): 302 br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark, 303 self.test_label, 1, self.test_cache_conditions, 304 self.mock_machine_manager, 305 self.mock_logger, 306 "average", "") 307 308 309 br.terminated = True 310 self.assertRaises (Exception, br.AcquireMachine) 311 312 br.terminated = False 313 mock_machine = MockCrosMachine ('chromeos1-row3-rack5-host7.cros', 314 'chromeos', 'average') 315 self.mock_machine_manager.AcquireMachine.return_value = mock_machine 316 317 machine = br.AcquireMachine() 318 self.assertEqual (machine.name, 'chromeos1-row3-rack5-host7.cros') 319 320 321 def test_get_extra_autotest_args(self): 322 br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark, 323 self.test_label, 1, self.test_cache_conditions, 324 self.mock_machine_manager, 325 self.mock_logger, 326 "average", "") 327 328 def MockLogError(err_msg): 329 "Helper function for test_get_extra_autotest_args" 330 self.err_msg = err_msg 331 332 self.mock_logger.LogError = MockLogError 333 334 result = br._GetExtraAutotestArgs() 335 self.assertEqual(result, "") 336 337 self.test_benchmark.perf_args = "record -e cycles" 338 result = br._GetExtraAutotestArgs() 339 self.assertEqual(result, 340"--profiler=custom_perf --profiler_args='perf_options=\"record -a -e cycles\"'") 341 342 self.test_benchmark.suite = "telemetry" 343 result = br._GetExtraAutotestArgs() 344 self.assertEqual(result, "") 345 self.assertEqual(self.err_msg, "Telemetry does not support profiler.") 346 347 self.test_benchmark.perf_args = "record -e cycles" 348 self.test_benchmark.suite = "test_that" 349 result = br._GetExtraAutotestArgs() 350 self.assertEqual(result, "") 351 self.assertEqual(self.err_msg, "test_that does not support profiler.") 352 353 self.test_benchmark.perf_args = "junk args" 354 self.test_benchmark.suite = "telemetry_Crosperf" 355 self.assertRaises(Exception, br._GetExtraAutotestArgs) 356 357 358 @mock.patch.object(SuiteRunner, 'Run') 359 @mock.patch.object(Result, 'CreateFromRun') 360 def test_run_test(self, mock_result, mock_runner): 361 br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark, 362 self.test_label, 1, self.test_cache_conditions, 363 self.mock_machine_manager, 364 self.mock_logger, 365 "average", "") 366 367 self.status = [] 368 def MockRecord(status): 369 self.status.append(status) 370 371 br.timeline.Record = MockRecord 372 mock_machine = MockCrosMachine ('chromeos1-row3-rack5-host7.cros', 373 'chromeos', 'average') 374 mock_runner.return_value = [0, "{'Score':100}", ""] 375 376 br.RunTest(mock_machine) 377 378 self.assertTrue(br.run_completed) 379 self.assertEqual (self.status, [ benchmark_run.STATUS_IMAGING, 380 benchmark_run.STATUS_RUNNING]) 381 382 self.assertEqual (br.machine_manager.ImageMachine.call_count, 1) 383 br.machine_manager.ImageMachine.assert_called_with (mock_machine, 384 self.test_label) 385 self.assertEqual (mock_runner.call_count, 1) 386 mock_runner.assert_called_with (mock_machine.name, br.label, 387 br.benchmark, "", br.profiler_args) 388 389 self.assertEqual (mock_result.call_count, 1) 390 mock_result.assert_called_with (self.mock_logger, 'average', 391 self.test_label, None, "{'Score':100}", 392 "", 0, False, 'page_cycler.netsim.top_10', 393 'telemetry_Crosperf') 394 395 396 397 def test_set_cache_conditions(self): 398 br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark, 399 self.test_label, 1, self.test_cache_conditions, 400 self.mock_machine_manager, 401 self.mock_logger, 402 "average", "") 403 404 phony_cache_conditions = [ 123, 456, True, False ] 405 406 self.assertEqual(br.cache_conditions, self.test_cache_conditions) 407 408 br.SetCacheConditions (phony_cache_conditions) 409 self.assertEqual(br.cache_conditions, phony_cache_conditions) 410 411 br.SetCacheConditions(self.test_cache_conditions) 412 self.assertEqual(br.cache_conditions, self.test_cache_conditions) 413 414 415if __name__ == "__main__": 416 unittest.main() 417