run_tests.py revision 5f1c94371a64b3196d4be9466099bb892df9b88e
15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Copyright 2012 The Chromium Authors. All rights reserved. 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# found in the LICENSE file. 45c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import logging 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import unittest 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 85c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from telemetry import decorators 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from telemetry.core import browser_finder 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from telemetry.core import browser_options 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from telemetry.core import command_line 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from telemetry.core import discover 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from telemetry.unittest import json_results 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from telemetry.unittest import output_formatter 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class Config(object): 185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __init__(self, top_level_dir, test_dirs, output_formatters): 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._top_level_dir = top_level_dir 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._test_dirs = tuple(test_dirs) 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._output_formatters = tuple(output_formatters) 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) @property 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def top_level_dir(self): 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self._top_level_dir 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) @property 285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_dirs(self): 295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self._test_dirs 305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) @property 32f91f5fa1608c2cdd9af1842fb5dadbe78275be2aBo Liu def output_formatters(self): 335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self._output_formatters 3451b2906e11752df6c18351cf520e30522d3b53a1Torne (Richard Coles) 355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 369e12abdf8c3a23d52091ea54ebb6a04d327f9300Torne (Richard Coles)def Discover(start_dir, top_level_dir=None, pattern='test*.py'): 375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) loader = unittest.defaultTestLoader 385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) loader.suiteClass = output_formatter.TestSuite 395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_suites = [] 415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) modules = discover.DiscoverModules(start_dir, top_level_dir, pattern) 425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for module in modules: 435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if hasattr(module, 'suite'): 445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) suite = module.suite() 455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else: 465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) suite = loader.loadTestsFromModule(module) 479e12abdf8c3a23d52091ea54ebb6a04d327f9300Torne (Richard Coles) if suite.countTestCases(): 485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_suites.append(suite) 499e12abdf8c3a23d52091ea54ebb6a04d327f9300Torne (Richard Coles) return test_suites 505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)def FilterSuite(suite, predicate): 535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) new_suite = suite.__class__() 545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for test in suite: 555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if isinstance(test, unittest.TestSuite): 565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) subsuite = FilterSuite(test, predicate) 579e12abdf8c3a23d52091ea54ebb6a04d327f9300Torne (Richard Coles) if subsuite.countTestCases(): 585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) new_suite.addTest(subsuite) 599e12abdf8c3a23d52091ea54ebb6a04d327f9300Torne (Richard Coles) else: 605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) assert isinstance(test, unittest.TestCase) 615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if predicate(test): 625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) new_suite.addTest(test) 635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return new_suite 6551b2906e11752df6c18351cf520e30522d3b53a1Torne (Richard Coles) 66 67def DiscoverTests(search_dirs, top_level_dir, possible_browser, 68 selected_tests=None, run_disabled_tests=False): 69 def IsTestSelected(test): 70 if selected_tests: 71 found = False 72 for name in selected_tests: 73 if name in test.id(): 74 found = True 75 if not found: 76 return False 77 if run_disabled_tests: 78 return True 79 # pylint: disable=W0212 80 if not hasattr(test, '_testMethodName'): 81 return True 82 method = getattr(test, test._testMethodName) 83 return decorators.IsEnabled(method, possible_browser) 84 85 wrapper_suite = output_formatter.TestSuite() 86 for search_dir in search_dirs: 87 wrapper_suite.addTests(Discover(search_dir, top_level_dir, '*_unittest.py')) 88 return FilterSuite(wrapper_suite, IsTestSelected) 89 90 91def RestoreLoggingLevel(func): 92 def _LoggingRestoreWrapper(*args, **kwargs): 93 # Cache the current logging level, this needs to be done before calling 94 # parser.parse_args, which changes logging level based on verbosity 95 # setting. 96 logging_level = logging.getLogger().getEffectiveLevel() 97 try: 98 return func(*args, **kwargs) 99 finally: 100 # Restore logging level, which may be changed in parser.parse_args. 101 logging.getLogger().setLevel(logging_level) 102 103 return _LoggingRestoreWrapper 104 105 106config = None 107 108 109class RunTestsCommand(command_line.OptparseCommand): 110 """Run unit tests""" 111 112 usage = '[test_name ...] [<options>]' 113 114 @classmethod 115 def CreateParser(cls): 116 options = browser_options.BrowserFinderOptions() 117 options.browser_type = 'any' 118 parser = options.CreateParser('%%prog %s' % cls.usage) 119 return parser 120 121 @classmethod 122 def AddCommandLineArgs(cls, parser): 123 parser.add_option('--repeat-count', type='int', default=1, 124 help='Repeats each a provided number of times.') 125 parser.add_option('-d', '--also-run-disabled-tests', 126 dest='run_disabled_tests', 127 action='store_true', default=False, 128 help='Ignore @Disabled and @Enabled restrictions.') 129 json_results.AddOptions(parser) 130 131 @classmethod 132 def ProcessCommandLineArgs(cls, parser, args): 133 if args.verbosity == 0: 134 logging.getLogger().setLevel(logging.WARN) 135 136 try: 137 possible_browser = browser_finder.FindBrowser(args) 138 except browser_finder.BrowserFinderException, ex: 139 parser.error(ex) 140 141 if not possible_browser: 142 parser.error('No browser found of type %s. Cannot run tests.\n' 143 'Re-run with --browser=list to see ' 144 'available browser types.' % args.browser_type) 145 146 json_results.ValidateArgs(parser, args) 147 148 def Run(self, args): 149 possible_browser = browser_finder.FindBrowser(args) 150 test_suite = DiscoverTests( 151 config.test_dirs, config.top_level_dir, possible_browser, 152 args.positional_args, args.run_disabled_tests) 153 runner = output_formatter.TestRunner() 154 result = runner.run( 155 test_suite, config.output_formatters, args.repeat_count, args) 156 157 json_results.WriteandUploadResultsIfNecessary(args, test_suite, result) 158 159 return len(result.failures_and_errors) 160 161 @classmethod 162 @RestoreLoggingLevel 163 def main(cls, args=None): 164 return super(RunTestsCommand, cls).main(args) 165