experiment_runner_unittest.py revision 7057cf67ba1dbdd4387f53e5fe47b43c955b1a53
1#!/usr/bin/python 2# 3# Copyright 2014 Google Inc. All Rights Reserved 4import StringIO 5import getpass 6import os 7import time 8 9import mock 10import unittest 11 12import experiment_runner 13import experiment_status 14import machine_manager 15import config 16import test_flag 17from experiment_file import ExperimentFile 18from experiment_factory import ExperimentFactory 19from results_report import TextResultsReport 20from results_report import HTMLResultsReport 21from results_cache import Result 22 23from cros_utils import logger 24from cros_utils import command_executer 25from cros_utils.email_sender import EmailSender 26from cros_utils.file_utils import FileUtils 27 28EXPERIMENT_FILE_1 = """ 29 board: parrot 30 remote: chromeos-parrot1.cros chromreos-parrot2.cros 31 32 benchmark: kraken { 33 suite: telemetry_Crosperf 34 iterations: 3 35 } 36 37 image1 { 38 chromeos_root: /usr/local/google/chromeos 39 chromeos_image: /usr/local/google/chromeos/src/build/images/parrot/latest/cros_image1.bin 40 } 41 42 image2 { 43 chromeos_image: /usr/local/google/chromeos/src/build/imaages/parrot/latest/cros_image2.bin 44 } 45 """ 46 47class FakeLogger(object): 48 49 def __init__(self): 50 self.LogOutputCount = 0 51 self.LogErrorCount = 0 52 self.output_msgs = [] 53 self.error_msgs = [] 54 self.dot_count = 0 55 self.LogStartDotsCount = 0 56 self.LogEndDotsCount = 0 57 self.LogAppendDotCount = 0 58 59 def LogOutput(self, msg): 60 self.LogOutputCount += 1 61 self.output_msgs.append(msg) 62 63 def LogError(self, msg): 64 self.LogErrorCount += 1 65 self.error_msgs.append(msg) 66 67 def LogStartDots(self): 68 self.LogStartDotsCount += 1 69 self.dot_count += 1 70 71 def LogAppendDot(self): 72 self.LogAppendDotCount += 1 73 self.dot_count += 1 74 75 def LogEndDots(self): 76 self.LogEndDotsCount += 1 77 78 def Reset(self): 79 self.LogOutputCount = 0 80 self.LogErrorCount = 0 81 self.output_msgs = [] 82 self.error_msgs = [] 83 self.dot_count = 0 84 self.LogStartDotsCount = 0 85 self.LogEndDotsCount = 0 86 self.LogAppendDotCount = 0 87 88class ExperimentRunnerTest(unittest.TestCase): 89 90 run_counter = 0 91 mock_logger = FakeLogger() 92 mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) 93 94 def make_fake_experiment(self): 95 test_flag.SetTestMode(True) 96 experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1)) 97 experiment = ExperimentFactory().GetExperiment(experiment_file, 98 working_directory="", 99 log_dir="") 100 return experiment 101 102 @mock.patch.object (machine_manager.MachineManager, 'AddMachine') 103 @mock.patch.object (os.path, 'isfile') 104 def setUp(self, mock_isfile, mock_addmachine): 105 mock_isfile.return_value = True 106 self.exp = self.make_fake_experiment() 107 108 109 def test_init(self): 110 er = experiment_runner.ExperimentRunner(self.exp, json_report=False, 111 using_schedv2=False, 112 log=self.mock_logger, 113 cmd_exec=self.mock_cmd_exec) 114 self.assertFalse (er._terminated) 115 self.assertEqual (er.STATUS_TIME_DELAY, 10) 116 117 self.exp.log_level = "verbose" 118 er = experiment_runner.ExperimentRunner(self.exp, json_report=False, 119 using_schedv2=False, 120 log=self.mock_logger, 121 cmd_exec=self.mock_cmd_exec) 122 self.assertEqual (er.STATUS_TIME_DELAY, 30) 123 124 125 126 @mock.patch.object(experiment_status.ExperimentStatus, 'GetStatusString') 127 @mock.patch.object(experiment_status.ExperimentStatus, 'GetProgressString') 128 def test_run(self, mock_progress_string, mock_status_string): 129 130 self.run_count = 0 131 self.is_complete_count = 0 132 133 def reset(): 134 self.run_count = 0 135 self.is_complete_count = 0 136 137 def FakeRun(): 138 self.run_count += 1 139 return 0 140 141 def FakeIsComplete(): 142 self.is_complete_count += 1 143 if self.is_complete_count < 3: 144 return False 145 else: 146 return True 147 148 self.mock_logger.Reset() 149 self.exp.Run = FakeRun 150 self.exp.IsComplete = FakeIsComplete 151 152 # Test 1: log_level == "quiet" 153 self.exp.log_level = "quiet" 154 er = experiment_runner.ExperimentRunner(self.exp, json_report=False, 155 using_schedv2=False, 156 log = self.mock_logger, 157 cmd_exec =self.mock_cmd_exec) 158 er.STATUS_TIME_DELAY = 2 159 mock_status_string.return_value = "Fake status string" 160 er._Run(self.exp) 161 self.assertEqual(self.run_count, 1) 162 self.assertTrue(self.is_complete_count > 0) 163 self.assertEqual(self.mock_logger.LogStartDotsCount, 1) 164 self.assertEqual(self.mock_logger.LogAppendDotCount, 1) 165 self.assertEqual(self.mock_logger.LogEndDotsCount, 1) 166 self.assertEqual(self.mock_logger.dot_count, 2) 167 self.assertEqual(mock_progress_string.call_count, 0) 168 self.assertEqual(mock_status_string.call_count, 2) 169 self.assertEqual(self.mock_logger.output_msgs, 170 ['==============================', 'Fake status string', 171 '==============================']) 172 self.assertEqual(len(self.mock_logger.error_msgs), 0) 173 174 # Test 2: log_level == "average" 175 self.mock_logger.Reset() 176 reset() 177 self.exp.log_level = "average" 178 mock_status_string.call_count = 0 179 er = experiment_runner.ExperimentRunner(self.exp, json_report=False, 180 using_schedv2=False, 181 log=self.mock_logger, 182 cmd_exec=self.mock_cmd_exec) 183 er.STATUS_TIME_DELAY = 2 184 mock_status_string.return_value = "Fake status string" 185 er._Run(self.exp) 186 self.assertEqual(self.run_count, 1) 187 self.assertTrue(self.is_complete_count > 0) 188 self.assertEqual(self.mock_logger.LogStartDotsCount, 1) 189 self.assertEqual(self.mock_logger.LogAppendDotCount, 1) 190 self.assertEqual(self.mock_logger.LogEndDotsCount, 1) 191 self.assertEqual(self.mock_logger.dot_count, 2) 192 self.assertEqual(mock_progress_string.call_count, 0) 193 self.assertEqual(mock_status_string.call_count, 2) 194 self.assertEqual(self.mock_logger.output_msgs, 195 ['==============================', 'Fake status string', 196 '==============================']) 197 self.assertEqual(len(self.mock_logger.error_msgs), 0) 198 199 200 # Test 3: log_level == "verbose" 201 self.mock_logger.Reset() 202 reset() 203 self.exp.log_level = "verbose" 204 mock_status_string.call_count = 0 205 er = experiment_runner.ExperimentRunner(self.exp, json_report=False, 206 using_schedv2=False, 207 log=self.mock_logger, 208 cmd_exec=self.mock_cmd_exec) 209 er.STATUS_TIME_DELAY = 2 210 mock_status_string.return_value = "Fake status string" 211 mock_progress_string.return_value = "Fake progress string" 212 er._Run(self.exp) 213 self.assertEqual(self.run_count, 1) 214 self.assertTrue(self.is_complete_count > 0) 215 self.assertEqual(self.mock_logger.LogStartDotsCount, 0) 216 self.assertEqual(self.mock_logger.LogAppendDotCount, 0) 217 self.assertEqual(self.mock_logger.LogEndDotsCount, 0) 218 self.assertEqual(self.mock_logger.dot_count, 0) 219 self.assertEqual(mock_progress_string.call_count, 2) 220 self.assertEqual(mock_status_string.call_count, 2) 221 self.assertEqual(self.mock_logger.output_msgs, 222 ['==============================', 223 'Fake progress string', 'Fake status string', 224 '==============================', 225 '==============================', 226 'Fake progress string', 'Fake status string', 227 '==============================']) 228 self.assertEqual(len(self.mock_logger.error_msgs), 0) 229 230 231 @mock.patch.object(TextResultsReport, 'GetReport') 232 def test_print_table(self, mock_report): 233 self.mock_logger.Reset() 234 mock_report.return_value = "This is a fake experiment report." 235 er = experiment_runner.ExperimentRunner(self.exp, json_report=False, 236 using_schedv2=False, 237 log=self.mock_logger, 238 cmd_exec=self.mock_cmd_exec) 239 er._PrintTable(self.exp) 240 self.assertEqual(mock_report.call_count, 1) 241 self.assertEqual(self.mock_logger.output_msgs, 242 [ 'This is a fake experiment report.' ]) 243 244 245 @mock.patch.object(HTMLResultsReport, 'GetReport') 246 @mock.patch.object(TextResultsReport, 'GetReport') 247 @mock.patch.object(EmailSender, 'Attachment') 248 @mock.patch.object(EmailSender, 'SendEmail') 249 @mock.patch.object(getpass, 'getuser') 250 def test_email(self, mock_getuser, mock_emailer, mock_attachment, 251 mock_text_report, mock_html_report): 252 253 mock_getuser.return_value = "john.smith@google.com" 254 mock_text_report.return_value = "This is a fake text report." 255 mock_html_report.return_value = "This is a fake html report." 256 257 self.mock_logger.Reset() 258 config.AddConfig("no_email", True) 259 self.exp.email_to = ["jane.doe@google.com"] 260 er = experiment_runner.ExperimentRunner(self.exp, json_report=False, 261 using_schedv2=False, 262 log=self.mock_logger, 263 cmd_exec=self.mock_cmd_exec) 264 # Test 1. Config:no_email; exp.email_to set ==> no email sent 265 er._Email(self.exp) 266 self.assertEqual(mock_getuser.call_count, 0) 267 self.assertEqual(mock_emailer.call_count, 0) 268 self.assertEqual(mock_attachment.call_count, 0) 269 self.assertEqual(mock_text_report.call_count, 0) 270 self.assertEqual(mock_html_report.call_count, 0) 271 272 # Test 2. Config: email. exp.email_to set; cache hit. => send email 273 self.mock_logger.Reset() 274 config.AddConfig("no_email", False) 275 for r in self.exp.benchmark_runs: 276 r.cache_hit = True 277 er._Email(self.exp) 278 self.assertEqual(mock_getuser.call_count, 1) 279 self.assertEqual(mock_emailer.call_count, 1) 280 self.assertEqual(mock_attachment.call_count, 1) 281 self.assertEqual(mock_text_report.call_count, 1) 282 self.assertEqual(mock_html_report.call_count, 1) 283 self.assertEqual(len(mock_emailer.call_args), 2) 284 self.assertEqual(mock_emailer.call_args[0], 285 (['john.smith@google.com', 'jane.doe@google.com'], 286 ': image1 vs. image2', 287 "<pre style='font-size: 13px'>This is a fake text " 288 "report.\nResults are stored in _results.\n</pre>")) 289 self.assertTrue(type(mock_emailer.call_args[1]) is dict) 290 self.assertEqual(len(mock_emailer.call_args[1]), 2) 291 self.assertTrue('attachments' in mock_emailer.call_args[1].keys()) 292 self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html') 293 294 mock_attachment.assert_called_with('report.html', 295 'This is a fake html report.') 296 297 # Test 3. Config: email; exp.mail_to set; no cache hit. => send email 298 self.mock_logger.Reset() 299 mock_getuser.reset_mock() 300 mock_emailer.reset_mock() 301 mock_attachment.reset_mock() 302 mock_text_report.reset_mock() 303 mock_html_report.reset_mock() 304 config.AddConfig("no_email", False) 305 for r in self.exp.benchmark_runs: 306 r.cache_hit = False 307 er._Email(self.exp) 308 self.assertEqual(mock_getuser.call_count, 1) 309 self.assertEqual(mock_emailer.call_count, 1) 310 self.assertEqual(mock_attachment.call_count, 1) 311 self.assertEqual(mock_text_report.call_count, 1) 312 self.assertEqual(mock_html_report.call_count, 1) 313 self.assertEqual(len(mock_emailer.call_args), 2) 314 self.assertEqual(mock_emailer.call_args[0], 315 (['john.smith@google.com', 'jane.doe@google.com'], 316 ': image1 vs. image2', 317 "<pre style='font-size: 13px'>This is a fake text " 318 "report.\nResults are stored in _results.\n</pre>")) 319 self.assertTrue(type(mock_emailer.call_args[1]) is dict) 320 self.assertEqual(len(mock_emailer.call_args[1]), 2) 321 self.assertTrue('attachments' in mock_emailer.call_args[1].keys()) 322 self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html') 323 324 mock_attachment.assert_called_with('report.html', 325 'This is a fake html report.') 326 327 # Test 4. Config: email; exp.mail_to = None; no cache hit. => send email 328 self.mock_logger.Reset() 329 mock_getuser.reset_mock() 330 mock_emailer.reset_mock() 331 mock_attachment.reset_mock() 332 mock_text_report.reset_mock() 333 mock_html_report.reset_mock() 334 self.exp.email_to = [] 335 er._Email(self.exp) 336 self.assertEqual(mock_getuser.call_count, 1) 337 self.assertEqual(mock_emailer.call_count, 1) 338 self.assertEqual(mock_attachment.call_count, 1) 339 self.assertEqual(mock_text_report.call_count, 1) 340 self.assertEqual(mock_html_report.call_count, 1) 341 self.assertEqual(len(mock_emailer.call_args), 2) 342 self.assertEqual(mock_emailer.call_args[0], 343 (['john.smith@google.com'], 344 ': image1 vs. image2', 345 "<pre style='font-size: 13px'>This is a fake text " 346 "report.\nResults are stored in _results.\n</pre>")) 347 self.assertTrue(type(mock_emailer.call_args[1]) is dict) 348 self.assertEqual(len(mock_emailer.call_args[1]), 2) 349 self.assertTrue('attachments' in mock_emailer.call_args[1].keys()) 350 self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html') 351 352 mock_attachment.assert_called_with('report.html', 353 'This is a fake html report.') 354 355 # Test 5. Config: email; exp.mail_to = None; cache hit => no email sent 356 self.mock_logger.Reset() 357 mock_getuser.reset_mock() 358 mock_emailer.reset_mock() 359 mock_attachment.reset_mock() 360 mock_text_report.reset_mock() 361 mock_html_report.reset_mock() 362 for r in self.exp.benchmark_runs: 363 r.cache_hit = True 364 er._Email(self.exp) 365 self.assertEqual(mock_getuser.call_count, 0) 366 self.assertEqual(mock_emailer.call_count, 0) 367 self.assertEqual(mock_attachment.call_count, 0) 368 self.assertEqual(mock_text_report.call_count, 0) 369 self.assertEqual(mock_html_report.call_count, 0) 370 371 @mock.patch.object(FileUtils, 'RmDir') 372 @mock.patch.object(FileUtils, 'MkDirP') 373 @mock.patch.object(FileUtils, 'WriteFile') 374 @mock.patch.object(HTMLResultsReport, 'GetReport') 375 @mock.patch.object(TextResultsReport, 'GetReport') 376 @mock.patch.object(Result, 'CopyResultsTo') 377 @mock.patch.object(Result, 'CleanUp') 378 def test_store_results(self, mock_cleanup, mock_copy, mock_text_report, 379 mock_report, mock_writefile, mock_mkdir, mock_rmdir): 380 381 self.mock_logger.Reset() 382 self.exp.results_directory='/usr/local/crosperf-results' 383 bench_run = self.exp.benchmark_runs[5] 384 bench_path = '/usr/local/crosperf-results/' + filter(str.isalnum, 385 bench_run.name) 386 self.assertEqual (len(self.exp.benchmark_runs), 6) 387 388 er = experiment_runner.ExperimentRunner(self.exp, json_report=False, 389 using_schedv2=False, 390 log=self.mock_logger, 391 cmd_exec=self.mock_cmd_exec) 392 393 # Test 1. Make sure nothing is done if _terminated is true. 394 er._terminated = True 395 er._StoreResults(self.exp) 396 self.assertEqual(mock_cleanup.call_count, 0) 397 self.assertEqual(mock_copy.call_count, 0) 398 self.assertEqual(mock_report.call_count, 0) 399 self.assertEqual(mock_writefile.call_count, 0) 400 self.assertEqual(mock_mkdir.call_count, 0) 401 self.assertEqual(mock_rmdir.call_count, 0) 402 self.assertEqual(self.mock_logger.LogOutputCount, 0) 403 404 # Test 2. _terminated is false; everything works properly. 405 fake_result = Result(self.mock_logger, self.exp.labels[0], "average", 406 "daisy1") 407 for r in self.exp.benchmark_runs: 408 r.result = fake_result 409 er._terminated = False 410 er._StoreResults(self.exp) 411 self.assertEqual(mock_cleanup.call_count, 6) 412 mock_cleanup.called_with(bench_run.benchmark.rm_chroot_tmp) 413 self.assertEqual(mock_copy.call_count, 6) 414 mock_copy.called_with(bench_path) 415 self.assertEqual(mock_writefile.call_count, 3) 416 self.assertEqual (len(mock_writefile.call_args_list), 3) 417 first_args = mock_writefile.call_args_list[0] 418 second_args = mock_writefile.call_args_list[1] 419 self.assertEqual(first_args[0][0], 420 '/usr/local/crosperf-results/experiment.exp') 421 self.assertEqual(second_args[0][0], 422 '/usr/local/crosperf-results/results.html') 423 self.assertEqual(mock_mkdir.call_count, 1) 424 mock_mkdir.called_with('/usr/local/crosperf-results') 425 self.assertEqual(mock_rmdir.call_count, 1) 426 mock_rmdir.called_with('/usr/local/crosperf-results') 427 self.assertEqual(self.mock_logger.LogOutputCount, 4) 428 self.assertEqual(self.mock_logger.output_msgs, 429 ['Storing experiment file in /usr/local/crosperf-results.', 430 'Storing results report in /usr/local/crosperf-results.', 431 'Storing email message body in /usr/local/crosperf-results.', 432 'Storing results of each benchmark run.']) 433 434 435 436if __name__ == '__main__': 437 unittest.main() 438