setup.py revision 3240926e260ce088908e02ac07a6cf7b0c0cbf44
13240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch# Copyright 2013 The Chromium Authors. All rights reserved. 23240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch# Use of this source code is governed by a BSD-style license that can be 33240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch# found in the LICENSE file. 43240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 53240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch"""Setup for instrumentation host-driven tests.""" 63240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 73240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochimport logging 83240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochimport os 93240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochimport sys 103240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochimport types 113240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 123240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochimport test_case 133240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochimport test_info_collection 143240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochimport test_runner 153240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 163240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 173240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochdef _GetPythonFiles(root, files): 183240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Returns all files from |files| that end in 'Test.py'. 193240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 203240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Args: 213240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch root: A directory name with python files. 223240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch files: A list of file names. 233240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 243240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Returns: 253240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch A list with all python files that match the testing naming scheme. 263240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 273240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return [os.path.join(root, f) for f in files if f.endswith('Test.py')] 283240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 293240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 303240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochdef _InferImportNameFromFile(python_file): 313240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Given a file, infer the import name for that file. 323240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 333240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Example: /usr/foo/bar/baz.py -> baz. 343240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 353240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Args: 363240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch python_file: Path to the Python file, ostensibly to import later. 373240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 383240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Returns: 393240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch The module name for the given file. 403240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 413240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return os.path.splitext(os.path.basename(python_file))[0] 423240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 433240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 443240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochdef _GetTestModules(host_driven_test_root, is_official_build): 453240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Retrieve a list of python modules that match the testing naming scheme. 463240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 473240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Walks the location of host-driven tests, imports them, and provides the list 483240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch of imported modules to the caller. 493240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 503240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Args: 513240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch host_driven_test_root: The path to walk, looking for the 523240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch pythonDrivenTests or host_driven_tests directory 533240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch is_official_build: Whether to run only those tests marked 'official' 543240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 553240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Returns: 563240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch A list of python modules under |host_driven_test_root| which match the 573240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch testing naming scheme. Each module should define one or more classes that 583240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch derive from HostDrivenTestCase. 593240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 603240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch # By default run all host-driven tests under pythonDrivenTests or 613240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch # host_driven_tests. 623240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch host_driven_test_file_list = [] 633240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch for root, _, files in os.walk(host_driven_test_root): 643240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch if (root.endswith('host_driven_tests') or 653240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch root.endswith('pythonDrivenTests') or 663240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch (is_official_build and (root.endswith('pythonDrivenTests/official') or 673240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch root.endswith('host_driven_tests/official')))): 683240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch host_driven_test_file_list += _GetPythonFiles(root, files) 693240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch host_driven_test_file_list.sort() 703240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 713240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_module_list = [_GetModuleFromFile(test_file) 723240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch for test_file in host_driven_test_file_list] 733240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return test_module_list 743240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 753240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 763240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochdef _GetModuleFromFile(python_file): 773240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Gets the python module associated with a file by importing it. 783240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 793240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Args: 803240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch python_file: File to import. 813240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 823240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Returns: 833240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch The module object. 843240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 853240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch sys.path.append(os.path.dirname(python_file)) 863240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch import_name = _InferImportNameFromFile(python_file) 873240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return __import__(import_name) 883240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 893240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 903240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochdef _GetTestsFromClass(test_case_class, **kwargs): 913240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Returns one test object for each test method in |test_case_class|. 923240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 933240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Test methods are methods on the class which begin with 'test'. 943240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 953240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Args: 963240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_case_class: Class derived from HostDrivenTestCase which contains zero 973240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch or more test methods. 983240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch kwargs: Keyword args to pass into the constructor of test cases. 993240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1003240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Returns: 1013240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch A list of test case objects, each initialized for a particular test method. 1023240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 1033240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_names = [m for m in dir(test_case_class) 1043240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch if _IsTestMethod(m, test_case_class)] 1053240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return [test_case_class(name, **kwargs) for name in test_names] 1063240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1073240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1083240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochdef _GetTestsFromModule(test_module, **kwargs): 1093240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Gets a list of test objects from |test_module|. 1103240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1113240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Args: 1123240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_module: Module from which to get the set of test methods. 1133240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch kwargs: Keyword args to pass into the constructor of test cases. 1143240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1153240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Returns: 1163240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch A list of test case objects each initialized for a particular test method 1173240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch defined in |test_module|. 1183240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 1193240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1203240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch tests = [] 1213240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch for name in dir(test_module): 1223240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch attr = getattr(test_module, name) 1233240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch if _IsTestCaseClass(attr): 1243240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch tests.extend(_GetTestsFromClass(attr, **kwargs)) 1253240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return tests 1263240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1273240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1283240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochdef _IsTestCaseClass(test_class): 1293240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return (type(test_class) is types.TypeType and 1303240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch issubclass(test_class, test_case.HostDrivenTestCase) and 1313240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_class is not test_case.HostDrivenTestCase) 1323240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1333240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1343240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochdef _IsTestMethod(attrname, test_case_class): 1353240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Checks whether this is a valid test method. 1363240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1373240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Args: 1383240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch attrname: The method name. 1393240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_case_class: The test case class. 1403240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1413240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Returns: 1423240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch True if test_case_class.'attrname' is callable and it starts with 'test'; 1433240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch False otherwise. 1443240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 1453240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch attr = getattr(test_case_class, attrname) 1463240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return callable(attr) and attrname.startswith('test') 1473240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1483240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1493240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochdef _GetAllTests(test_root, is_official_build, **kwargs): 1503240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Retrieve a list of host-driven tests defined under |test_root|. 1513240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1523240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Args: 1533240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_root: Path which contains host-driven test files. 1543240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch is_official_build: Whether this is an official build. 1553240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch kwargs: Keyword args to pass into the constructor of test cases. 1563240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1573240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Returns: 1583240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch List of test case objects, one for each available test method. 1593240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 1603240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch if not test_root: 1613240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return [] 1623240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch all_tests = [] 1633240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_module_list = _GetTestModules(test_root, is_official_build) 1643240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch for module in test_module_list: 1653240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch all_tests.extend(_GetTestsFromModule(module, **kwargs)) 1663240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return all_tests 1673240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1683240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1693240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochdef InstrumentationSetup(host_driven_test_root, official_build, 1703240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch instrumentation_options): 1713240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """Creates a list of host-driven instrumentation tests and a runner factory. 1723240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1733240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Args: 1743240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch host_driven_test_root: Directory where the host-driven tests are. 1753240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch official_build: True if this is an official build. 1763240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch instrumentation_options: An InstrumentationOptions object. 1773240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1783240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch Returns: 1793240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch A tuple of (TestRunnerFactory, tests). 1803240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch """ 1813240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1823240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_collection = test_info_collection.TestInfoCollection() 1833240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch all_tests = _GetAllTests( 1843240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch host_driven_test_root, official_build, 1853240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch instrumentation_options=instrumentation_options) 1863240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_collection.AddTests(all_tests) 1873240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1883240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch available_tests = test_collection.GetAvailableTests( 1893240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch instrumentation_options.annotations, 1903240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch instrumentation_options.exclude_annotations, 1913240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch instrumentation_options.test_filter) 1923240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch logging.debug('All available tests: ' + str( 1933240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch [t.tagged_name for t in available_tests])) 1943240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 1953240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch def TestRunnerFactory(device, shard_index): 1963240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return test_runner.HostDrivenTestRunner( 1973240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch device, shard_index, 1983240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch instrumentation_options.tool, 1993240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch instrumentation_options.build_type, 2003240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch instrumentation_options.push_deps, 2013240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch instrumentation_options.cleanup_test_files) 2023240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 2033240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch return (TestRunnerFactory, available_tests) 204