1#!/usr/bin/python 2# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import datetime as datetime_base 7from datetime import datetime 8import mock 9import time 10import unittest 11 12import common 13 14from autotest_lib.server.cros.dynamic_suite import constants 15from autotest_lib.site_utils import run_suite 16from autotest_lib.site_utils import diagnosis_utils 17 18 19class ResultCollectorUnittest(unittest.TestCase): 20 """Runsuite unittest""" 21 22 JOB_MAX_RUNTIME_MINS = 10 23 24 def setUp(self): 25 """Set up test.""" 26 self.afe = mock.MagicMock() 27 self.tko = mock.MagicMock() 28 29 30 def _build_view(self, test_idx, test_name, subdir, status, afe_job_id, 31 job_name='fake_job_name', reason='fake reason', 32 job_keyvals=None, test_started_time=None, 33 test_finished_time=None, invalidates_test_idx=None, 34 job_started_time=None, job_finished_time=None): 35 """Build a test view using the given fields. 36 37 @param test_idx: An integer representing test_idx. 38 @param test_name: A string, e.g. 'dummy_Pass' 39 @param subdir: A string representing the subdir field of the test view. 40 e.g. 'dummy_Pass'. 41 @param status: A string representing the test status. 42 e.g. 'FAIL', 'PASS' 43 @param afe_job_id: An integer representing the afe job id. 44 @param job_name: A string representing the job name. 45 @param reason: A string representing the reason field of the test view. 46 @param job_keyvals: A dictionary stroing the job keyvals. 47 @param test_started_time: A string, e.g. '2014-04-12 12:35:33' 48 @param test_finished_time: A string, e.g. '2014-04-12 12:35:33' 49 @param invalidates_test_idx: An integer, representing the idx of the 50 test that has been retried. 51 @param job_started_time: A string, e.g. '2014-04-12 12:35:33' 52 @param job_finished_time: A string, e.g. '2014-04-12 12:35:33' 53 54 @reutrn: A dictionary representing a test view. 55 56 """ 57 if job_keyvals is None: 58 job_keyvals = {} 59 return {'test_idx': test_idx, 'test_name': test_name, 'subdir':subdir, 60 'status': status, 'afe_job_id': afe_job_id, 61 'job_name': job_name, 'reason': reason, 62 'job_keyvals': job_keyvals, 63 'test_started_time': test_started_time, 64 'test_finished_time': test_finished_time, 65 'invalidates_test_idx': invalidates_test_idx, 66 'job_started_time': job_started_time, 67 'job_finished_time': job_finished_time} 68 69 70 def _mock_tko_get_detailed_test_views(self, test_views, 71 missing_results=[]): 72 """Mock tko method get_detailed_test_views call. 73 74 @param test_views: A list of test views that will be returned 75 by get_detailed_test_views. 76 """ 77 return_values = {} 78 for v in test_views: 79 views_of_job = return_values.setdefault( 80 ('get_detailed_test_views', v['afe_job_id']), []) 81 views_of_job.append(v) 82 for job_id in missing_results: 83 views_of_job = return_values.setdefault( 84 ('get_detailed_test_views', job_id), []) 85 86 def side_effect(*args, **kwargs): 87 """Maps args and kwargs to the mocked return values.""" 88 key = (kwargs['call'], kwargs['afe_job_id']) 89 return return_values[key] 90 91 self.tko.run = mock.MagicMock(side_effect=side_effect) 92 93 94 def _mock_afe_get_jobs(self, suite_job_id, child_job_ids): 95 """Mock afe get_jobs call. 96 97 @param suite_job_id: The afe job id of the suite job. 98 @param child_job_ids: A list of job ids of the child jobs. 99 100 """ 101 suite_job = mock.MagicMock() 102 suite_job.id = suite_job_id 103 suite_job.max_runtime_mins = 10 104 suite_job.parent_job = None 105 106 return_values = {suite_job_id: []} 107 for job_id in child_job_ids: 108 new_job = mock.MagicMock() 109 new_job.id = job_id 110 new_job.name = 'test.%d' % job_id 111 new_job.max_runtime_mins = self.JOB_MAX_RUNTIME_MINS 112 new_job.parent_job = suite_job 113 return_values[suite_job_id].append(new_job) 114 115 def side_effect(*args, **kwargs): 116 """Maps args and kwargs to the mocked return values.""" 117 if kwargs.get('id') == suite_job_id: 118 return [suite_job] 119 return return_values[kwargs['parent_job_id']] 120 121 self.afe.get_jobs = mock.MagicMock(side_effect=side_effect) 122 123 124 def testFetchSuiteTestView(self): 125 """Test that it fetches the correct suite test views.""" 126 suite_job_id = 100 127 suite_name = 'dummy' 128 build = 'R23-1.1.1.1' 129 server_job_view = self._build_view( 130 10, 'SERVER_JOB', '----', 'GOOD', suite_job_id) 131 test_to_ignore = self._build_view( 132 11, 'dummy_Pass', '101-user/host/dummy_Pass', 133 'GOOD', suite_job_id) 134 test_to_include = self._build_view( 135 12, 'dummy_Pass.bluetooth', None, 'TEST_NA', suite_job_id) 136 test_missing = self._build_view( 137 13, 'dummy_Missing', None, 'ABORT', suite_job_id) 138 self._mock_afe_get_jobs(suite_job_id, []) 139 self._mock_tko_get_detailed_test_views( 140 [server_job_view, test_to_ignore, test_to_include, 141 test_missing]) 142 collector = run_suite.ResultCollector( 143 'fake_server', self.afe, self.tko, 144 build='fake/build', board='fake', suite_name='dummy', 145 suite_job_id=suite_job_id) 146 collector._missing_results = { 147 test_missing['test_name']: [14, 15], 148 } 149 suite_views = collector._fetch_relevant_test_views_of_suite() 150 suite_views = sorted(suite_views, key=lambda view: view['test_idx']) 151 # Verify that SERVER_JOB is renamed to 'Suite job' 152 self.assertEqual(suite_views[0].get_testname(), 153 run_suite.TestView.SUITE_JOB) 154 # Verify that the test with a subidr is not included. 155 self.assertEqual(suite_views[0]['test_idx'], 10) 156 self.assertEqual(suite_views[1]['test_idx'], 12) 157 self.assertEqual(suite_views[1]['afe_job_id'], suite_job_id) 158 # Verify that the test with missing results had it's AFE job id 159 # replaced. 160 self.assertEqual(suite_views[2]['test_idx'], 13) 161 self.assertEqual(suite_views[2]['afe_job_id'], 14) 162 163 164 def testFetchTestViewOfChildJobs(self): 165 """Test that it fetches the correct child test views.""" 166 build = 'lumpy-release/R36-5788.0.0' 167 board = 'lumpy' 168 suite_name = 'my_suite' 169 suite_job_id = 100 170 invalid_job_id = 101 171 invalid_job_name = '%s/%s/test_Pass' % (build, suite_name) 172 good_job_id = 102 173 good_job_name = '%s/%s/test_Pass' % (build, suite_name) 174 bad_job_id = 103 175 bad_job_name = '%s/%s/test_ServerJobFail' % (build, suite_name) 176 missing_job_id = 104 177 178 invalid_test = self._build_view( 179 19, 'test_Pass_Old', 'fake/subdir', 180 'FAIL', invalid_job_id, invalid_job_name) 181 good_job_server_job = self._build_view( 182 20, 'SERVER_JOB', '----', 'GOOD', good_job_id, good_job_name) 183 good_job_test = self._build_view( 184 21, 'test_Pass', 'fake/subdir', 'GOOD', 185 good_job_id, good_job_name, 186 job_keyvals={'retry_original_job_id': invalid_job_id}) 187 bad_job_server_job = self._build_view( 188 22, 'SERVER_JOB', '----', 'FAIL', bad_job_id, bad_job_name) 189 bad_job_test = self._build_view( 190 23, 'test_ServerJobFail', 'fake/subdir', 'GOOD', 191 bad_job_id, bad_job_name) 192 self._mock_tko_get_detailed_test_views( 193 [good_job_server_job, good_job_test, 194 bad_job_server_job, bad_job_test, invalid_test], 195 [missing_job_id]) 196 self._mock_afe_get_jobs(suite_job_id, 197 [good_job_id, bad_job_id, missing_job_id]) 198 collector = run_suite.ResultCollector( 199 'fake_server', self.afe, self.tko, 200 build, board, suite_name, suite_job_id) 201 child_views, retry_counts, missing_results = ( 202 collector._fetch_test_views_of_child_jobs()) 203 # child_views should contain tests 21, 22, 23 204 child_views = sorted(child_views, key=lambda view: view['test_idx']) 205 # Verify that the SERVER_JOB has been renamed properly 206 self.assertEqual(child_views[1].get_testname(), 207 'test_ServerJobFail_SERVER_JOB') 208 self.assertEqual(missing_results, {'test.104': [104]}) 209 # Verify that failed SERVER_JOB and actual invalid tests are included, 210 expected = [good_job_test['test_idx'], bad_job_server_job['test_idx'], 211 bad_job_test['test_idx']] 212 child_view_ids = [v['test_idx'] for v in child_views] 213 self.assertEqual(child_view_ids, expected) 214 self.afe.get_jobs.assert_called_once_with( 215 parent_job_id=suite_job_id) 216 # Verify the retry_counts is calculated correctly 217 self.assertEqual(len(retry_counts), 1) 218 self.assertEqual(retry_counts[21], 1) 219 220 221 def testGenerateLinks(self): 222 """Test that it generates correct web and buildbot links.""" 223 suite_job_id = 100 224 suite_name = 'my_suite' 225 build = 'lumpy-release/R36-5788.0.0' 226 board = 'lumpy' 227 fake_job = mock.MagicMock() 228 fake_job.parent = suite_job_id 229 suite_job_view = run_suite.TestView( 230 self._build_view( 231 20, 'Suite job', '----', 'GOOD', suite_job_id), 232 fake_job, suite_name, build, 'chromeos-test') 233 good_test = run_suite.TestView( 234 self._build_view( 235 21, 'test_Pass', 'fake/subdir', 'GOOD', 101), 236 fake_job, suite_name, build, 'chromeos-test') 237 bad_test = run_suite.TestView( 238 self._build_view( 239 23, 'test_Fail', 'fake/subdir', 'FAIL', 102), 240 fake_job, suite_name, build, 'chromeos-test') 241 242 collector = run_suite.ResultCollector( 243 'fake_server', self.afe, self.tko, 244 build, board, suite_name, suite_job_id, user='chromeos-test') 245 collector._suite_views = [suite_job_view] 246 collector._test_views = [suite_job_view, good_test, bad_test] 247 collector._max_testname_width = max( 248 [len(v.get_testname()) for v in collector._test_views]) + 3 249 collector._generate_web_and_buildbot_links() 250 URL_PATTERN = run_suite._URL_PATTERN 251 # expected_web_links is list of (anchor, url) tuples we 252 # are expecting. 253 expected_web_links = [ 254 (v.get_testname(), 255 URL_PATTERN % ('fake_server', 256 '%s-%s' % (v['afe_job_id'], 'chromeos-test'))) 257 for v in collector._test_views] 258 # Verify web links are generated correctly. 259 for i in range(len(collector._web_links)): 260 expect = expected_web_links[i] 261 self.assertEqual(collector._web_links[i].anchor, expect[0]) 262 self.assertEqual(collector._web_links[i].url, expect[1]) 263 264 expected_buildbot_links = [ 265 (v.get_testname(), 266 URL_PATTERN % ('fake_server', 267 '%s-%s' % (v['afe_job_id'], 'chromeos-test'))) 268 for v in collector._test_views if v['status'] != 'GOOD'] 269 # Verify buildbot links are generated correctly. 270 for i in range(len(collector._buildbot_links)): 271 expect = expected_buildbot_links[i] 272 self.assertEqual(collector._buildbot_links[i].anchor, expect[0]) 273 self.assertEqual(collector._buildbot_links[i].url, expect[1]) 274 self.assertEqual(collector._buildbot_links[i].retry_count, 0) 275 # Assert that a wmatrix retry dashboard link is created. 276 self.assertNotEqual( 277 collector._buildbot_links[i].GenerateWmatrixRetryLink(),'') 278 279 280 def _end_to_end_test_helper( 281 self, include_bad_test=False, include_warn_test=False, 282 include_experimental_bad_test=False, include_timeout_test=False, 283 include_self_aborted_test=False, 284 include_aborted_by_suite_test=False, 285 include_good_retry=False, include_bad_retry=False, 286 suite_job_timed_out=False, suite_job_status='GOOD'): 287 """A helper method for testing ResultCollector end-to-end. 288 289 This method mocks the retrieving of required test views, 290 and call ResultCollector.run() to collect the results. 291 292 @param include_bad_test: 293 If True, include a view of a test which has status 'FAIL'. 294 @param include_warn_test: 295 If True, include a view of a test which has status 'WARN' 296 @param include_experimental_bad_test: 297 If True, include a view of an experimental test 298 which has status 'FAIL'. 299 @param include_timeout_test: 300 If True, include a view of a test which was aborted before 301 started. 302 @param include_self_aborted_test: 303 If True, include a view of test which was aborted after 304 started and hit hits own timeout. 305 @param include_self_aborted_by_suite_test: 306 If True, include a view of test which was aborted after 307 started but has not hit its own timeout. 308 @param include_good_retry: 309 If True, include a test that passed after retry. 310 @param include_bad_retry: 311 If True, include a test that failed after retry. 312 @param suite_job_status: One of 'GOOD' 'FAIL' 'ABORT' 'RUNNING' 313 314 @returns: A ResultCollector instance. 315 """ 316 suite_job_id = 100 317 good_job_id = 101 318 bad_job_id = 102 319 warn_job_id = 102 320 experimental_bad_job_id = 102 321 timeout_job_id = 100 322 self_aborted_job_id = 104 323 aborted_by_suite_job_id = 105 324 good_retry_job_id = 106 325 bad_retry_job_id = 107 326 invalid_job_id_1 = 90 327 invalid_job_id_2 = 91 328 suite_name = 'dummy' 329 build = 'lumpy-release/R27-3888.0.0' 330 suite_job_keyvals = { 331 constants.DOWNLOAD_STARTED_TIME: '2014-04-29 13:14:20', 332 constants.PAYLOAD_FINISHED_TIME: '2014-04-29 13:14:25', 333 constants.ARTIFACT_FINISHED_TIME: '2014-04-29 13:14:30'} 334 335 suite_job_started_time = '2014-04-29 13:14:37' 336 if suite_job_timed_out: 337 suite_job_keyvals['aborted_by'] = 'test_user' 338 suite_job_finished_time = '2014-04-29 13:25:37' 339 suite_job_status = 'ABORT' 340 else: 341 suite_job_finished_time = '2014-04-29 13:23:37' 342 343 server_job_view = self._build_view( 344 10, 'SERVER_JOB', '----', suite_job_status, suite_job_id, 345 'lumpy-release/R27-3888.0.0-test_suites/control.dummy', 346 '', suite_job_keyvals, '2014-04-29 13:14:37', 347 '2014-04-29 13:20:27', job_started_time=suite_job_started_time, 348 job_finished_time=suite_job_finished_time) 349 good_test = self._build_view( 350 11, 'dummy_Pass', '101-user/host/dummy_Pass', 'GOOD', 351 good_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Pass', 352 '', {}, '2014-04-29 13:15:35', '2014-04-29 13:15:36') 353 bad_test = self._build_view( 354 12, 'dummy_Fail.Fail', '102-user/host/dummy_Fail.Fail', 'FAIL', 355 bad_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Fail', 356 'always fail', {}, '2014-04-29 13:16:00', 357 '2014-04-29 13:16:02') 358 warn_test = self._build_view( 359 13, 'dummy_Fail.Warn', '102-user/host/dummy_Fail.Warn', 'WARN', 360 warn_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Warn', 361 'always warn', {}, '2014-04-29 13:16:00', 362 '2014-04-29 13:16:02') 363 experimental_bad_test = self._build_view( 364 14, 'experimental_dummy_Fail.Fail', 365 '102-user/host/dummy_Fail.Fail', 'FAIL', 366 experimental_bad_job_id, 367 'lumpy-release/R27-3888.0.0/dummy/experimental_dummy_Fail.Fail', 368 'always fail', {'experimental': 'True'}, '2014-04-29 13:16:06', 369 '2014-04-29 13:16:07') 370 timeout_test = self._build_view( 371 15, 'dummy_Timeout', '', 'ABORT', 372 timeout_job_id, 373 'lumpy-release/R27-3888.0.0/dummy/dummy_Timeout', 374 'child job did not run', {}, '2014-04-29 13:15:37', 375 '2014-04-29 13:15:38') 376 self_aborted_test = self._build_view( 377 16, 'dummy_Abort', '104-user/host/dummy_Abort', 'ABORT', 378 self_aborted_job_id, 379 'lumpy-release/R27-3888.0.0/dummy/dummy_Abort', 380 'child job aborted', {'aborted_by': 'test_user'}, 381 '2014-04-29 13:15:39', '2014-04-29 13:15:40', 382 job_started_time='2014-04-29 13:15:39', 383 job_finished_time='2014-04-29 13:25:40') 384 aborted_by_suite = self._build_view( 385 17, 'dummy_AbortBySuite', '105-user/host/dummy_AbortBySuite', 386 'RUNNING', aborted_by_suite_job_id, 387 'lumpy-release/R27-3888.0.0/dummy/dummy_Abort', 388 'aborted by suite', {'aborted_by': 'test_user'}, 389 '2014-04-29 13:15:39', '2014-04-29 13:15:40', 390 job_started_time='2014-04-29 13:15:39', 391 job_finished_time='2014-04-29 13:15:40') 392 good_retry = self._build_view( 393 18, 'dummy_RetryPass', '106-user/host/dummy_RetryPass', 'GOOD', 394 good_retry_job_id, 395 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass', 396 '', {'retry_original_job_id': invalid_job_id_1}, 397 '2014-04-29 13:15:37', 398 '2014-04-29 13:15:38', invalidates_test_idx=1) 399 bad_retry = self._build_view( 400 19, 'dummy_RetryFail', '107-user/host/dummy_RetryFail', 'FAIL', 401 bad_retry_job_id, 402 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail', 403 'retry failed', {'retry_original_job_id': invalid_job_id_2}, 404 '2014-04-29 13:15:39', '2014-04-29 13:15:40', 405 invalidates_test_idx=2) 406 invalid_test_1 = self._build_view( 407 1, 'dummy_RetryPass', '90-user/host/dummy_RetryPass', 'GOOD', 408 invalid_job_id_1, 409 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass', 410 'original test failed', {}, '2014-04-29 13:10:00', 411 '2014-04-29 13:10:01') 412 invalid_test_2 = self._build_view( 413 2, 'dummy_RetryFail', '91-user/host/dummy_RetryFail', 'FAIL', 414 invalid_job_id_2, 415 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail', 416 'original test failed', {}, 417 '2014-04-29 13:10:03', '2014-04-29 13:10:04') 418 419 test_views = [server_job_view, good_test] 420 child_jobs = set([good_job_id]) 421 if include_bad_test: 422 test_views.append(bad_test) 423 child_jobs.add(bad_job_id) 424 if include_warn_test: 425 test_views.append(warn_test) 426 child_jobs.add(warn_job_id) 427 if include_experimental_bad_test: 428 test_views.append(experimental_bad_test) 429 child_jobs.add(experimental_bad_job_id) 430 if include_timeout_test: 431 test_views.append(timeout_test) 432 if include_self_aborted_test: 433 test_views.append(self_aborted_test) 434 child_jobs.add(self_aborted_job_id) 435 if include_good_retry: 436 test_views.extend([good_retry, invalid_test_1]) 437 child_jobs.add(good_retry_job_id) 438 if include_bad_retry: 439 test_views.extend([bad_retry, invalid_test_2]) 440 child_jobs.add(bad_retry_job_id) 441 if include_aborted_by_suite_test: 442 test_views.append(aborted_by_suite) 443 child_jobs.add(aborted_by_suite_job_id) 444 self._mock_tko_get_detailed_test_views(test_views) 445 self._mock_afe_get_jobs(suite_job_id, child_jobs) 446 collector = run_suite.ResultCollector( 447 'fake_server', self.afe, self.tko, 448 'lumpy-release/R36-5788.0.0', 'lumpy', 'dummy', suite_job_id) 449 collector.run() 450 return collector 451 452 453 def testEndToEndSuitePass(self): 454 """Test it returns code OK when all test pass.""" 455 collector = self._end_to_end_test_helper() 456 self.assertEqual(collector.return_code, run_suite.RETURN_CODES.OK) 457 458 459 def testEndToEndExperimentalTestFails(self): 460 """Test that it returns code OK when only experimental test fails.""" 461 collector = self._end_to_end_test_helper( 462 include_experimental_bad_test=True) 463 self.assertEqual(collector.return_code, run_suite.RETURN_CODES.OK) 464 465 466 def testEndToEndSuiteWarn(self): 467 """Test it returns code WARNING when there is a test that warns.""" 468 collector = self._end_to_end_test_helper(include_warn_test=True) 469 self.assertEqual(collector.return_code, run_suite.RETURN_CODES.WARNING) 470 471 472 def testEndToEndSuiteFail(self): 473 """Test it returns code ERROR when there is a test that fails.""" 474 # Test that it returns ERROR when there is test that fails. 475 collector = self._end_to_end_test_helper(include_bad_test=True) 476 self.assertEqual(collector.return_code, run_suite.RETURN_CODES.ERROR) 477 478 # Test that it returns ERROR when both experimental and non-experimental 479 # test fail. 480 collector = self._end_to_end_test_helper( 481 include_bad_test=True, include_warn_test=True, 482 include_experimental_bad_test=True) 483 self.assertEqual(collector.return_code, run_suite.RETURN_CODES.ERROR) 484 485 collector = self._end_to_end_test_helper(include_self_aborted_test=True) 486 self.assertEqual(collector.return_code, run_suite.RETURN_CODES.ERROR) 487 488 489 def testEndToEndSuiteJobFail(self): 490 """Test it returns code SUITE_FAILURE when only the suite job failed.""" 491 collector = self._end_to_end_test_helper(suite_job_status='ABORT') 492 self.assertEqual( 493 collector.return_code, run_suite.RETURN_CODES.INFRA_FAILURE) 494 495 collector = self._end_to_end_test_helper(suite_job_status='ERROR') 496 self.assertEqual( 497 collector.return_code, run_suite.RETURN_CODES.INFRA_FAILURE) 498 499 500 def testEndToEndRetry(self): 501 """Test it returns correct code when a test was retried.""" 502 collector = self._end_to_end_test_helper(include_good_retry=True) 503 self.assertEqual( 504 collector.return_code, run_suite.RETURN_CODES.WARNING) 505 506 collector = self._end_to_end_test_helper(include_good_retry=True, 507 include_self_aborted_test=True) 508 self.assertEqual( 509 collector.return_code, run_suite.RETURN_CODES.ERROR) 510 511 collector = self._end_to_end_test_helper(include_good_retry=True, 512 include_bad_test=True) 513 self.assertEqual( 514 collector.return_code, run_suite.RETURN_CODES.ERROR) 515 516 collector = self._end_to_end_test_helper(include_bad_retry=True) 517 self.assertEqual( 518 collector.return_code, run_suite.RETURN_CODES.ERROR) 519 520 521 def testEndToEndSuiteTimeout(self): 522 """Test it returns correct code when a child job timed out.""" 523 # a child job timed out before started, none failed. 524 collector = self._end_to_end_test_helper(include_timeout_test=True) 525 self.assertEqual( 526 collector.return_code, run_suite.RETURN_CODES.SUITE_TIMEOUT) 527 528 # a child job timed out before started, and one test failed. 529 collector = self._end_to_end_test_helper( 530 include_bad_test=True, include_timeout_test=True) 531 self.assertEqual(collector.return_code, run_suite.RETURN_CODES.ERROR) 532 533 # a child job timed out before started, and one test warned. 534 collector = self._end_to_end_test_helper( 535 include_warn_test=True, include_timeout_test=True) 536 self.assertEqual(collector.return_code, 537 run_suite.RETURN_CODES.SUITE_TIMEOUT) 538 539 # a child job timed out before started, and one test was retried. 540 collector = self._end_to_end_test_helper(include_good_retry=True, 541 include_timeout_test=True) 542 self.assertEqual( 543 collector.return_code, run_suite.RETURN_CODES.SUITE_TIMEOUT) 544 545 # a child jot was aborted because suite timed out. 546 collector = self._end_to_end_test_helper( 547 include_aborted_by_suite_test=True) 548 self.assertEqual( 549 collector.return_code, run_suite.RETURN_CODES.OK) 550 551 # suite job timed out. 552 collector = self._end_to_end_test_helper(suite_job_timed_out=True) 553 self.assertEqual( 554 collector.return_code, run_suite.RETURN_CODES.SUITE_TIMEOUT) 555 556 557class LogLinkUnittests(unittest.TestCase): 558 """Test the LogLink""" 559 560 def testGenerateBuildbotLinks(self): 561 """Test LogLink GenerateBuildbotLinks""" 562 log_link_a = run_suite.LogLink('mock_anchor', 'mock_server', 563 'mock_job_string', 564 bug_info=('mock_bug_id', 1), 565 reason='mock_reason', 566 retry_count=1, 567 testname='mock_testname') 568 # Generate a bug link and a log link when bug_info is present 569 self.assertTrue(len(list(log_link_a.GenerateBuildbotLinks())) == 2) 570 571 log_link_b = run_suite.LogLink('mock_anchor', 'mock_server', 572 'mock_job_string_b', 573 reason='mock_reason', 574 retry_count=1, 575 testname='mock_testname') 576 # Generate a log link when there is no bug_info 577 self.assertTrue(len(list(log_link_b.GenerateBuildbotLinks())) == 1) 578 579 580class SimpleTimerUnittests(unittest.TestCase): 581 """Test the simple timer.""" 582 583 def testPoll(self): 584 """Test polling the timer.""" 585 interval_hours = 0.0001 586 t = diagnosis_utils.SimpleTimer(interval_hours=interval_hours) 587 deadline = t.deadline 588 self.assertTrue(deadline is not None and 589 t.interval_hours == interval_hours) 590 min_deadline = (datetime.now() + 591 datetime_base.timedelta(hours=interval_hours)) 592 time.sleep(interval_hours * 3600) 593 self.assertTrue(t.poll()) 594 self.assertTrue(t.deadline >= min_deadline) 595 596 597 def testBadInterval(self): 598 """Test a bad interval.""" 599 t = diagnosis_utils.SimpleTimer(interval_hours=-1) 600 self.assertTrue(t.deadline is None and t.poll() == False) 601 t._reset() 602 self.assertTrue(t.deadline is None and t.poll() == False) 603 604 605class ArgumentParserUnittests(unittest.TestCase): 606 """Tests for argument parser.""" 607 608 @unittest.expectedFailure 609 def test_crbug_658013(self): 610 """crbug.com/658013 611 612 Expected failure due to http://bugs.python.org/issue9334 613 """ 614 parser = run_suite.make_parser() 615 args = [ 616 '--board', 'heli', 617 '--build', 'trybot-heli-paladin/R56-8918.0.0-b1601', 618 '--suite_name', 'test_that_wrapper', 619 '--pool', 'suites', 620 '--max_runtime_mins', '20', 621 '--suite_args', '-b heli -i trybot-heli-paladin/R56-8918.0.0-b1601 :lab: suite:bvt-inline', 622 ] 623 624 def error_handler(msg): # pylint: disable=missing-docstring 625 self.fail('Argument parsing failed: ' + msg) 626 627 parser.error = error_handler 628 got = parser.parse_args(args) 629 self.assertEqual( 630 got.board, 'heli') 631 self.assertEqual( 632 got.build, 'trybot-heli-paladin/R56-8918.0.0-b1601') 633 self.assertEqual( 634 got.suite_args, 635 '-b heli -i trybot-heli-paladin/R56-8918.0.0-b1601 :lab: suite:bvt-inline') 636 637 def test_crbug_658013b(self): 638 """crbug.com/658013 639 640 Unambiguous behavior. 641 """ 642 parser = run_suite.make_parser() 643 args = [ 644 '--board=heli', 645 '--build=trybot-heli-paladin/R56-8918.0.0-b1601', 646 '--suite_name=test_that_wrapper', 647 '--pool=suites', 648 '--max_runtime_mins=20', 649 '--suite_args=-b heli -i trybot-heli-paladin/R56-8918.0.0-b1601 :lab: suite:bvt-inline', 650 ] 651 652 def error_handler(msg): # pylint: disable=missing-docstring 653 self.fail('Argument parsing failed: ' + msg) 654 655 parser.error = error_handler 656 got = parser.parse_args(args) 657 self.assertEqual( 658 got.board, 'heli') 659 self.assertEqual( 660 got.build, 'trybot-heli-paladin/R56-8918.0.0-b1601') 661 self.assertEqual( 662 got.suite_args, 663 '-b heli -i trybot-heli-paladin/R56-8918.0.0-b1601 :lab: suite:bvt-inline') 664 665 666if __name__ == '__main__': 667 unittest.main() 668