test_runner.py revision 1e9bf3e0803691d0a228da41fc608347b6db4340
12a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Copyright (c) 2012 The Chromium Authors. All rights reserved. 22a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 32a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# found in the LICENSE file. 42a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 52a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import logging 62a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import os 77dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochimport re 82a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 92a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from pylib import android_commands 102a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from pylib import constants 117dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochfrom pylib import pexpect 122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from pylib.base import base_test_result 132a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from pylib.base import base_test_runner 141e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)from pylib.perf import perf_control 152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 16c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 17fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdochdef _TestSuiteRequiresMockTestServer(suite_name): 182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Returns True if the test suite requires mock test server.""" 192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) tests_require_net_test_server = ['unit_tests', 'net_unittests', 202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 'content_unittests', 212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 'content_browsertests'] 22fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch return (suite_name in 232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) tests_require_net_test_server) 242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 251e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)def _TestSuiteRequiresHighPerfMode(suite_name): 261e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) """Returns True if the test suite requires high performance mode.""" 271e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) return 'perftests' in suite_name 282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)class TestRunner(base_test_runner.BaseTestRunner): 30a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) def __init__(self, test_options, device, test_package): 317dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """Single test suite attached to a single device. 327dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 337dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Args: 34a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) test_options: A GTestOptions object. 357dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch device: Device to run the tests. 36fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch test_package: An instance of TestPackage class. 377dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """ 38a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 39a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) super(TestRunner, self).__init__(device, test_options.tool, 40a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) test_options.push_deps, 41a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) test_options.cleanup_test_files) 42a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 43fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self.test_package = test_package 44fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self.test_package.tool = self.tool 45a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) self._test_arguments = test_options.test_arguments 46a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 47a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) timeout = test_options.timeout 487dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if timeout == 0: 497dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch timeout = 60 507dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # On a VM (e.g. chromium buildbots), this timeout is way too small. 517dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if os.environ.get('BUILDBOT_SLAVENAME'): 527dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch timeout = timeout * 2 53a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 54fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self._timeout = timeout * self.tool.GetTimeoutScale() 551e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) self._perf_controller = perf_control.PerfControl(self.adb) 562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 57c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) #override 587d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) def InstallTestPackage(self): 59fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self.test_package.Install(self.adb) 60fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch 61fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch def GetAllTests(self): 62fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch """Install test package and get a list of all tests.""" 63fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self.test_package.Install(self.adb) 64fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch return self.test_package.GetAllTests(self.adb) 657d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) 667d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) #override 677d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) def PushDataDeps(self): 687dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch self.adb.WaitForSdCardReady(20) 692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.tool.CopyFiles() 707dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if os.path.exists(constants.ISOLATE_DEPS_DIR): 717dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch device_dir = self.adb.GetExternalStorage() 727dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # TODO(frankf): linux_dumper_unittest_helper needs to be in the same dir 737dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # as breakpad_unittests exe. Find a better way to do this. 74fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch if self.test_package.suite_name == 'breakpad_unittests': 757dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch device_dir = constants.TEST_EXECUTABLE_DIR 767dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch for p in os.listdir(constants.ISOLATE_DEPS_DIR): 777dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch self.adb.PushIfNeeded( 787dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch os.path.join(constants.ISOLATE_DEPS_DIR, p), 797dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch os.path.join(device_dir, p)) 802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 817dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch def _ParseTestOutput(self, p): 827dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """Process the test output. 837dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 847dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Args: 857dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch p: An instance of pexpect spawn class. 867dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 877dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Returns: 887dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch A TestRunResults object. 897dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """ 907dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch results = base_test_result.TestRunResults() 917dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 927dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # Test case statuses. 937dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch re_run = re.compile('\[ RUN \] ?(.*)\r\n') 947dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch re_fail = re.compile('\[ FAILED \] ?(.*)\r\n') 957dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch re_ok = re.compile('\[ OK \] ?(.*?) .*\r\n') 967dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 977dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # Test run statuses. 987dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch re_passed = re.compile('\[ PASSED \] ?(.*)\r\n') 997dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n') 1007dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # Signal handlers are installed before starting tests 1017dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # to output the CRASHED marker when a crash happens. 1027dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch re_crash = re.compile('\[ CRASHED \](.*)\r\n') 1037dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1047dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch log = '' 1057dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch try: 1067dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch while True: 1077dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name = None 1087dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch found = p.expect([re_run, re_passed, re_runner_fail], 109fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch timeout=self._timeout) 1107dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if found == 1: # re_passed 1117dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch break 1127dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch elif found == 2: # re_runner_fail 1137dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch break 1147dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch else: # re_run 1157dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name = p.match.group(1).replace('\r', '') 116fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch found = p.expect([re_ok, re_fail, re_crash], timeout=self._timeout) 1177dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch log = p.before.replace('\r', '') 1187dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if found == 0: # re_ok 1197dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if full_test_name == p.match.group(1).replace('\r', ''): 1207dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch results.AddResult(base_test_result.BaseTestResult( 1217dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name, base_test_result.ResultType.PASS, 1227dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch log=log)) 1237dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch elif found == 2: # re_crash 1247dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch results.AddResult(base_test_result.BaseTestResult( 1257dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name, base_test_result.ResultType.CRASH, 1267dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch log=log)) 1277dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch break 1287dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch else: # re_fail 1297dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch results.AddResult(base_test_result.BaseTestResult( 1307dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name, base_test_result.ResultType.FAIL, log=log)) 1317dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch except pexpect.EOF: 1327dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch logging.error('Test terminated - EOF') 1337dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # We're here because either the device went offline, or the test harness 1347dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # crashed without outputting the CRASHED marker (crbug.com/175538). 1357dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if not self.adb.IsOnline(): 1367dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch raise android_commands.errors.DeviceUnresponsiveError( 1377dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 'Device %s went offline.' % self.device) 1387dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if full_test_name: 1397dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch results.AddResult(base_test_result.BaseTestResult( 1407dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name, base_test_result.ResultType.CRASH, 1417dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch log=p.before.replace('\r', ''))) 1427dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch except pexpect.TIMEOUT: 1437dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch logging.error('Test terminated after %d second timeout.', 144fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self._timeout) 1457dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if full_test_name: 1467dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch results.AddResult(base_test_result.BaseTestResult( 1477dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch full_test_name, base_test_result.ResultType.TIMEOUT, 1487dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch log=p.before.replace('\r', ''))) 1497dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch finally: 1507dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch p.close() 1517dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 152fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch ret_code = self.test_package.GetGTestReturnCode(self.adb) 1537dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if ret_code: 1547dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch logging.critical( 1557dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s', 1567dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch ret_code, p.before, p.after) 1577dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1587dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch return results 1597dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) #override 1612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def RunTest(self, test): 1622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) test_results = base_test_result.TestRunResults() 1632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if not test: 1642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return test_results, None 1652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) try: 167fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self.test_package.ClearApplicationState(self.adb) 1687dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch self.test_package.CreateCommandLineFileOnDevice( 169fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self.adb, test, self._test_arguments) 170fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch test_results = self._ParseTestOutput( 171fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch self.test_package.SpawnTestProcess(self.adb)) 1722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) finally: 1732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.CleanupSpawningServerState() 1742a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Calculate unknown test results. 1752a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) all_tests = set(test.split(':')) 1762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) all_tests_ran = set([t.GetName() for t in test_results.GetAll()]) 1772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) unknown_tests = all_tests - all_tests_ran 1782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) test_results.AddResults( 1792a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) [base_test_result.BaseTestResult(t, base_test_result.ResultType.UNKNOWN) 1802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for t in unknown_tests]) 1812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) retry = ':'.join([t.GetName() for t in test_results.GetNotPass()]) 1822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return test_results, retry 1832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1842a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) #override 1852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def SetUp(self): 1862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Sets up necessary test enviroment for the test suite.""" 1872a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) super(TestRunner, self).SetUp() 188fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch if _TestSuiteRequiresMockTestServer(self.test_package.suite_name): 1892a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.LaunchChromeTestServerSpawner() 1901e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name): 1911e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) self._perf_controller.SetHighPerfMode() 1922a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.tool.SetupEnvironment() 1932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) #override 1952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def TearDown(self): 1962a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Cleans up the test enviroment for the test suite.""" 1971e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name): 1981e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) self._perf_controller.RestoreOriginalPerfMode() 199c2db58bd994c04d98e4ee2cd7565b71548655fe3Ben Murdoch self.test_package.ClearApplicationState(self.adb) 2002a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.tool.CleanUpEnvironment() 2012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) super(TestRunner, self).TearDown() 202