13240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch# Copyright 2013 The Chromium Authors. All rights reserved. 23240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch# Use of this source code is governed by a BSD-style license that can be 33240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch# found in the LICENSE file. 43240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 53240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch"""Runs host-driven tests on a particular device.""" 63240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 73240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochimport logging 83240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochimport sys 93240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochimport time 103240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochimport traceback 113240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 123240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochfrom pylib.base import base_test_result 133240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochfrom pylib.base import base_test_runner 145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from pylib.host_driven import test_case 153240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochfrom pylib.instrumentation import test_result 163240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 173240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 183240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochclass HostDrivenExceptionTestResult(test_result.InstrumentationTestResult): 193240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Test result corresponding to a python exception in a host-driven test.""" 203240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 213240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch def __init__(self, test_name, start_date_ms, exc_info): 223240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Constructs a HostDrivenExceptionTestResult object. 233240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 243240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Args: 253240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_name: name of the test which raised an exception. 263240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch start_date_ms: the starting time for the test. 273240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch exc_info: exception info, ostensibly from sys.exc_info(). 283240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 293240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch exc_type, exc_value, exc_traceback = exc_info 303240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch trace_info = ''.join(traceback.format_exception(exc_type, exc_value, 313240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch exc_traceback)) 323240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch log_msg = 'Exception:\n' + trace_info 333240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch duration_ms = (int(time.time()) * 1000) - start_date_ms 343240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 353240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch super(HostDrivenExceptionTestResult, self).__init__( 363240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_name, 373240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch base_test_result.ResultType.FAIL, 383240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch start_date_ms, 393240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch duration_ms, 403240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch log=str(exc_type) + ' ' + log_msg) 413240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 423240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 433240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochclass HostDrivenTestRunner(base_test_runner.BaseTestRunner): 443240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Orchestrates running a set of host-driven tests. 453240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 463240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Any Python exceptions in the tests are caught and translated into a failed 473240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch result, rather than being re-raised on the main thread. 483240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 493240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 503240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch #override 513551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) def __init__(self, device, shard_index, tool, push_deps, 523240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch cleanup_test_files): 533240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Creates a new HostDrivenTestRunner. 543240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 553240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Args: 563240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch device: Attached android device. 573240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch shard_index: Shard index. 583240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch tool: Name of the Valgrind tool. 593240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch push_deps: If True, push all dependencies to the device. 603240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch cleanup_test_files: Whether or not to cleanup test files on device. 613240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 623240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 633551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) super(HostDrivenTestRunner, self).__init__(device, tool, push_deps, 643551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) cleanup_test_files) 653240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 663240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch # The shard index affords the ability to create unique port numbers (e.g. 673240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch # DEFAULT_PORT + shard_index) if the test so wishes. 683240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch self.shard_index = shard_index 693240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 703240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch #override 713240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch def RunTest(self, test): 723240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Sets up and runs a test case. 733240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 743240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Args: 753240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test: An object which is ostensibly a subclass of HostDrivenTestCase. 763240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 773240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Returns: 783240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch A TestRunResults object which contains the result produced by the test 793240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch and, in the case of a failure, the test that should be retried. 803240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 813240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 823240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch assert isinstance(test, test_case.HostDrivenTestCase) 833240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 843240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch start_date_ms = int(time.time()) * 1000 853240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch exception_raised = False 863240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 873240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch try: 88a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch test.SetUp(self.device.old_interface.GetDevice(), self.shard_index, 89a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch self._push_deps, self._cleanup_test_files) 903240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch except Exception: 913240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch logging.exception( 923240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 'Caught exception while trying to run SetUp() for test: ' + 933240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test.tagged_name) 943240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch # Tests whose SetUp() method has failed are likely to fail, or at least 953240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch # yield invalid results. 963240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch exc_info = sys.exc_info() 973240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch results = base_test_result.TestRunResults() 983240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch results.AddResult(HostDrivenExceptionTestResult( 993240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test.tagged_name, start_date_ms, exc_info)) 1003240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return results, test 1013240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1023240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch try: 1033240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch results = test.Run() 1043240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch except Exception: 1053240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch # Setting this lets TearDown() avoid stomping on our stack trace from 1063240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch # Run() should TearDown() also raise an exception. 1073240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch exception_raised = True 1083240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch logging.exception('Caught exception while trying to run test: ' + 1093240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test.tagged_name) 1103240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch exc_info = sys.exc_info() 1113240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch results = base_test_result.TestRunResults() 1123240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch results.AddResult(HostDrivenExceptionTestResult( 1133240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test.tagged_name, start_date_ms, exc_info)) 1143240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1153240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch try: 1163240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test.TearDown() 1173240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch except Exception: 1183240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch logging.exception( 1193240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 'Caught exception while trying run TearDown() for test: ' + 1203240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test.tagged_name) 1213240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch if not exception_raised: 1223240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch # Don't stomp the error during the test if TearDown blows up. This is a 1233240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch # trade-off: if the test fails, this will mask any problem with TearDown 1243240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch # until the test is fixed. 1253240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch exc_info = sys.exc_info() 1263240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch results = base_test_result.TestRunResults() 1273240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch results.AddResult(HostDrivenExceptionTestResult( 1283240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test.tagged_name, start_date_ms, exc_info)) 1293240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1303240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch if not results.DidRunPass(): 1313240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return results, test 1323240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch else: 1333240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return results, None 134