1# Copyright 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6import unittest
7
8from telemetry import decorators
9from telemetry.core import browser_finder
10from telemetry.core import browser_options
11from telemetry.core import command_line
12from telemetry.core import discover
13from telemetry.unittest import json_results
14from telemetry.unittest import progress_reporter
15
16
17class Config(object):
18  def __init__(self, top_level_dir, test_dirs, progress_reporters):
19    self._top_level_dir = top_level_dir
20    self._test_dirs = tuple(test_dirs)
21    self._progress_reporters = tuple(progress_reporters)
22
23  @property
24  def top_level_dir(self):
25    return self._top_level_dir
26
27  @property
28  def test_dirs(self):
29    return self._test_dirs
30
31  @property
32  def progress_reporters(self):
33    return self._progress_reporters
34
35
36def Discover(start_dir, top_level_dir=None, pattern='test*.py'):
37  loader = unittest.defaultTestLoader
38  loader.suiteClass = progress_reporter.TestSuite
39
40  test_suites = []
41  modules = discover.DiscoverModules(start_dir, top_level_dir, pattern)
42  for module in modules:
43    if hasattr(module, 'suite'):
44      suite = module.suite()
45    else:
46      suite = loader.loadTestsFromModule(module)
47    if suite.countTestCases():
48      test_suites.append(suite)
49  return test_suites
50
51
52def FilterSuite(suite, predicate):
53  new_suite = suite.__class__()
54  for test in suite:
55    if isinstance(test, unittest.TestSuite):
56      subsuite = FilterSuite(test, predicate)
57      if subsuite.countTestCases():
58        new_suite.addTest(subsuite)
59    else:
60      assert isinstance(test, unittest.TestCase)
61      if predicate(test):
62        new_suite.addTest(test)
63
64  return new_suite
65
66
67def DiscoverTests(search_dirs, top_level_dir, possible_browser,
68                  selected_tests=None, selected_tests_are_exact=False,
69                  run_disabled_tests=False):
70  def IsTestSelected(test):
71    if selected_tests:
72      found = False
73      for name in selected_tests:
74        if selected_tests_are_exact:
75          if name == test.id():
76            found = True
77        else:
78          if name in test.id():
79            found = True
80      if not found:
81        return False
82    if run_disabled_tests:
83      return True
84    # pylint: disable=W0212
85    if not hasattr(test, '_testMethodName'):
86      return True
87    method = getattr(test, test._testMethodName)
88    return decorators.IsEnabled(method, possible_browser)
89
90  wrapper_suite = progress_reporter.TestSuite()
91  for search_dir in search_dirs:
92    wrapper_suite.addTests(Discover(search_dir, top_level_dir, '*_unittest.py'))
93  return FilterSuite(wrapper_suite, IsTestSelected)
94
95
96def RestoreLoggingLevel(func):
97  def _LoggingRestoreWrapper(*args, **kwargs):
98    # Cache the current logging level, this needs to be done before calling
99    # parser.parse_args, which changes logging level based on verbosity
100    # setting.
101    logging_level = logging.getLogger().getEffectiveLevel()
102    try:
103      return func(*args, **kwargs)
104    finally:
105      # Restore logging level, which may be changed in parser.parse_args.
106      logging.getLogger().setLevel(logging_level)
107
108  return _LoggingRestoreWrapper
109
110
111config = None
112
113
114class RunTestsCommand(command_line.OptparseCommand):
115  """Run unit tests"""
116
117  usage = '[test_name ...] [<options>]'
118
119  @classmethod
120  def CreateParser(cls):
121    options = browser_options.BrowserFinderOptions()
122    options.browser_type = 'any'
123    parser = options.CreateParser('%%prog %s' % cls.usage)
124    return parser
125
126  @classmethod
127  def AddCommandLineArgs(cls, parser):
128    parser.add_option('--repeat-count', type='int', default=1,
129                      help='Repeats each a provided number of times.')
130    parser.add_option('-d', '--also-run-disabled-tests',
131                      dest='run_disabled_tests',
132                      action='store_true', default=False,
133                      help='Ignore @Disabled and @Enabled restrictions.')
134    parser.add_option('--retry-limit', type='int',
135                      help='Retry each failure up to N times'
136                           ' to de-flake things.')
137    parser.add_option('--exact-test-filter', action='store_true', default=False,
138                      help='Treat test filter as exact matches (default is '
139                           'substring matches).')
140    json_results.AddOptions(parser)
141
142  @classmethod
143  def ProcessCommandLineArgs(cls, parser, args):
144    if args.verbosity == 0:
145      logging.getLogger().setLevel(logging.WARN)
146
147    # We retry failures by default unless we're running a list of tests
148    # explicitly.
149    if args.retry_limit is None and not args.positional_args:
150      args.retry_limit = 3
151
152    try:
153      possible_browser = browser_finder.FindBrowser(args)
154    except browser_finder.BrowserFinderException, ex:
155      parser.error(ex)
156
157    if not possible_browser:
158      parser.error('No browser found of type %s. Cannot run tests.\n'
159                   'Re-run with --browser=list to see '
160                   'available browser types.' % args.browser_type)
161
162    json_results.ValidateArgs(parser, args)
163
164  def Run(self, args):
165    possible_browser = browser_finder.FindBrowser(args)
166
167    test_suite, result = self.RunOneSuite(possible_browser, args)
168
169    results = [result]
170
171    failed_tests = json_results.FailedTestNames(test_suite, result)
172    retry_limit = args.retry_limit
173
174    while retry_limit and failed_tests:
175      args.positional_args = failed_tests
176      args.exact_test_filter = True
177
178      _, result = self.RunOneSuite(possible_browser, args)
179      results.append(result)
180
181      failed_tests = json_results.FailedTestNames(test_suite, result)
182      retry_limit -= 1
183
184    full_results = json_results.FullResults(args, test_suite, results)
185    json_results.WriteFullResultsIfNecessary(args, full_results)
186
187    err_occurred, err_str = json_results.UploadFullResultsIfNecessary(
188        args, full_results)
189    if err_occurred:
190      for line in err_str.splitlines():
191        logging.error(line)
192      return 1
193
194    return json_results.ExitCodeFromFullResults(full_results)
195
196  def RunOneSuite(self, possible_browser, args):
197    test_suite = DiscoverTests(config.test_dirs, config.top_level_dir,
198                               possible_browser, args.positional_args,
199                               args.exact_test_filter, args.run_disabled_tests)
200    runner = progress_reporter.TestRunner()
201    result = runner.run(test_suite, config.progress_reporters,
202                        args.repeat_count, args)
203    return test_suite, result
204
205  @classmethod
206  @RestoreLoggingLevel
207  def main(cls, args=None):
208    return super(RunTestsCommand, cls).main(args)
209