12a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Copyright (c) 2012 The Chromium Authors. All rights reserved. 22a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 32a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# found in the LICENSE file. 42a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 52a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import logging 62a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import os 77dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochimport re 82a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 92a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from pylib import constants 107dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochfrom pylib import pexpect 112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from pylib.base import base_test_result 122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from pylib.base import base_test_runner 13cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)from pylib.device import device_errors 141e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)from pylib.perf import perf_control 152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 16c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 17fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdochdef _TestSuiteRequiresMockTestServer(suite_name): 182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Returns True if the test suite requires mock test server.""" 192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) tests_require_net_test_server = ['unit_tests', 'net_unittests', 202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 'content_unittests', 212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 'content_browsertests'] 22fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch return (suite_name in 232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) tests_require_net_test_server) 242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 251e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)def _TestSuiteRequiresHighPerfMode(suite_name): 261e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) """Returns True if the test suite requires high performance mode.""" 271e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) return 'perftests' in suite_name 282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)class TestRunner(base_test_runner.BaseTestRunner): 30a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) def __init__(self, test_options, device, test_package): 317dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """Single test suite attached to a single device. 327dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 337dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Args: 34a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) test_options: A GTestOptions object. 357dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch device: Device to run the tests. 36fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch test_package: An instance of TestPackage class. 377dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """ 38a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 39a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) super(TestRunner, self).__init__(device, test_options.tool, 40a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) test_options.push_deps, 41a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) test_options.cleanup_test_files) 42a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 43fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self.test_package = test_package 44fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self.test_package.tool = self.tool 45a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) self._test_arguments = test_options.test_arguments 46a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 47a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) timeout = test_options.timeout 487dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if timeout == 0: 497dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch timeout = 60 507dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # On a VM (e.g. chromium buildbots), this timeout is way too small. 517dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if os.environ.get('BUILDBOT_SLAVENAME'): 527dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch timeout = timeout * 2 53a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 54fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self._timeout = timeout * self.tool.GetTimeoutScale() 55f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name): 56a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch self._perf_controller = perf_control.PerfControl(self.device) 572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 58c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) #override 597d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) def InstallTestPackage(self): 60a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch self.test_package.Install(self.device) 61fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch 627d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) #override 637d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) def PushDataDeps(self): 64cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.device.WaitUntilFullyBooted(timeout=20) 652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.tool.CopyFiles() 667dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if os.path.exists(constants.ISOLATE_DEPS_DIR): 677dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # TODO(frankf): linux_dumper_unittest_helper needs to be in the same dir 687dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # as breakpad_unittests exe. Find a better way to do this. 69fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch if self.test_package.suite_name == 'breakpad_unittests': 707dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch device_dir = constants.TEST_EXECUTABLE_DIR 71cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) else: 72cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) device_dir = self.device.GetExternalStoragePath() 737dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch for p in os.listdir(constants.ISOLATE_DEPS_DIR): 74116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.device.PushChangedFiles( 757dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch os.path.join(constants.ISOLATE_DEPS_DIR, p), 767dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch os.path.join(device_dir, p)) 772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 787dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch def _ParseTestOutput(self, p): 797dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """Process the test output. 807dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 817dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Args: 827dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch p: An instance of pexpect spawn class. 837dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 847dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Returns: 857dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch A TestRunResults object. 867dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """ 877dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch results = base_test_result.TestRunResults() 887dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 897dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # Test case statuses. 907dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch re_run = re.compile('\[ RUN \] ?(.*)\r\n') 917dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch re_fail = re.compile('\[ FAILED \] ?(.*)\r\n') 927dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch re_ok = re.compile('\[ OK \] ?(.*?) .*\r\n') 937dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 947dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # Test run statuses. 957dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch re_passed = re.compile('\[ PASSED \] ?(.*)\r\n') 967dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n') 977dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # Signal handlers are installed before starting tests 987dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # to output the CRASHED marker when a crash happens. 997dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch re_crash = re.compile('\[ CRASHED \](.*)\r\n') 1007dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1017dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch log = '' 1027dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch try: 1037dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch while True: 1047dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name = None 1057dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch found = p.expect([re_run, re_passed, re_runner_fail], 106fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch timeout=self._timeout) 1077dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if found == 1: # re_passed 1087dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch break 1097dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch elif found == 2: # re_runner_fail 1107dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch break 1117dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch else: # re_run 1127dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name = p.match.group(1).replace('\r', '') 113fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch found = p.expect([re_ok, re_fail, re_crash], timeout=self._timeout) 1147dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch log = p.before.replace('\r', '') 1157dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if found == 0: # re_ok 1167dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if full_test_name == p.match.group(1).replace('\r', ''): 1177dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch results.AddResult(base_test_result.BaseTestResult( 1187dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name, base_test_result.ResultType.PASS, 1197dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch log=log)) 1207dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch elif found == 2: # re_crash 1217dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch results.AddResult(base_test_result.BaseTestResult( 1227dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name, base_test_result.ResultType.CRASH, 1237dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch log=log)) 1247dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch break 1257dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch else: # re_fail 1267dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch results.AddResult(base_test_result.BaseTestResult( 1277dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name, base_test_result.ResultType.FAIL, log=log)) 1287dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch except pexpect.EOF: 1297dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch logging.error('Test terminated - EOF') 1307dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # We're here because either the device went offline, or the test harness 1317dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # crashed without outputting the CRASHED marker (crbug.com/175538). 132cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if not self.device.IsOnline(): 133cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) raise device_errors.DeviceUnreachableError( 1345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'Device %s went offline.' % str(self.device)) 1357dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if full_test_name: 1367dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch results.AddResult(base_test_result.BaseTestResult( 1377dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name, base_test_result.ResultType.CRASH, 1387dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch log=p.before.replace('\r', ''))) 1397dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch except pexpect.TIMEOUT: 1407dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch logging.error('Test terminated after %d second timeout.', 141fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self._timeout) 1427dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if full_test_name: 1437dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch results.AddResult(base_test_result.BaseTestResult( 1447dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name, base_test_result.ResultType.TIMEOUT, 1457dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch log=p.before.replace('\r', ''))) 1467dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch finally: 1477dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch p.close() 1487dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 149a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch ret_code = self.test_package.GetGTestReturnCode(self.device) 1507dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if ret_code: 1517dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch logging.critical( 1527dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s', 1537dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch ret_code, p.before, p.after) 1547dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1557dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch return results 1567dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) #override 1582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def RunTest(self, test): 1592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) test_results = base_test_result.TestRunResults() 1602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if not test: 1612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return test_results, None 1622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) try: 164a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch self.test_package.ClearApplicationState(self.device) 1657dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch self.test_package.CreateCommandLineFileOnDevice( 166a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch self.device, test, self._test_arguments) 167fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch test_results = self._ParseTestOutput( 168a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch self.test_package.SpawnTestProcess(self.device)) 1692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) finally: 1702a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.CleanupSpawningServerState() 1712a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Calculate unknown test results. 1722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) all_tests = set(test.split(':')) 1732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) all_tests_ran = set([t.GetName() for t in test_results.GetAll()]) 1742a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) unknown_tests = all_tests - all_tests_ran 1752a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) test_results.AddResults( 1762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) [base_test_result.BaseTestResult(t, base_test_result.ResultType.UNKNOWN) 1772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for t in unknown_tests]) 1782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) retry = ':'.join([t.GetName() for t in test_results.GetNotPass()]) 1792a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return test_results, retry 1802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) #override 1822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def SetUp(self): 1832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Sets up necessary test enviroment for the test suite.""" 1842a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) super(TestRunner, self).SetUp() 185fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch if _TestSuiteRequiresMockTestServer(self.test_package.suite_name): 1862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.LaunchChromeTestServerSpawner() 1871e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name): 1881e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) self._perf_controller.SetHighPerfMode() 1892a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.tool.SetupEnvironment() 1902a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1912a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) #override 1922a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def TearDown(self): 1932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Cleans up the test enviroment for the test suite.""" 1941e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name): 1956d86b77056ed63eb6871182f42a9fd5f07550f90Torne (Richard Coles) self._perf_controller.SetDefaultPerfMode() 196a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch self.test_package.ClearApplicationState(self.device) 1972a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.tool.CleanUpEnvironment() 1982a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) super(TestRunner, self).TearDown() 199