1# Copyright 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import fnmatch
6import logging
7import os
8import sys
9
10from telemetry.core import util
11from telemetry.core import platform as platform_module
12from telemetry import decorators
13from telemetry.internal.browser import browser_finder
14from telemetry.internal.browser import browser_finder_exceptions
15from telemetry.internal.browser import browser_options
16from telemetry.internal.platform import android_device
17from telemetry.internal.util import binary_manager
18from telemetry.internal.util import command_line
19from telemetry.internal.util import ps_util
20from telemetry.testing import browser_test_case
21from telemetry.testing import options_for_unittests
22
23from py_utils import cloud_storage
24from py_utils import xvfb
25
26import typ
27
28
29class RunTestsCommand(command_line.OptparseCommand):
30  """Run unit tests"""
31
32  usage = '[test_name ...] [<options>]'
33  xvfb_process = None
34
35  def __init__(self):
36    super(RunTestsCommand, self).__init__()
37    self.stream = sys.stdout
38
39  @classmethod
40  def CreateParser(cls):
41    options = browser_options.BrowserFinderOptions()
42    options.browser_type = 'any'
43    parser = options.CreateParser('%%prog %s' % cls.usage)
44    return parser
45
46  @classmethod
47  def AddCommandLineArgs(cls, parser, _):
48    parser.add_option('--start-xvfb', action='store_true',
49                      default=False, help='Start Xvfb display if needed.')
50    parser.add_option('--disable-cloud-storage-io', action='store_true',
51                      default=False, help=('Disable cloud storage IO when '
52                                           'tests are run in parallel.'))
53    parser.add_option('--repeat-count', type='int', default=1,
54                      help='Repeats each a provided number of times.')
55    parser.add_option('--no-browser', action='store_true', default=False,
56                      help='Don\'t require an actual browser to run the tests.')
57    parser.add_option('-d', '--also-run-disabled-tests',
58                      dest='run_disabled_tests',
59                      action='store_true', default=False,
60                      help='Ignore @Disabled and @Enabled restrictions.')
61    parser.add_option('--exact-test-filter', action='store_true', default=False,
62                      help='Treat test filter as exact matches (default is '
63                           'substring matches).')
64    parser.add_option('--client-config', dest='client_configs',
65                      action='append', default=[])
66    parser.add_option('--disable-logging-config', action='store_true',
67                      default=False, help='Configure logging (default on)')
68    parser.add_option('--skip', metavar='glob', default=[],
69                      action='append', help=(
70                          'Globs of test names to skip (defaults to '
71                          '%(default)s).'))
72    typ.ArgumentParser.add_option_group(parser,
73                                        "Options for running the tests",
74                                        running=True,
75                                        skip=['-d', '-v', '--verbose'])
76    typ.ArgumentParser.add_option_group(parser,
77                                        "Options for reporting the results",
78                                        reporting=True)
79
80  @classmethod
81  def ProcessCommandLineArgs(cls, parser, args, _):
82    # We retry failures by default unless we're running a list of tests
83    # explicitly.
84    if not args.retry_limit and not args.positional_args:
85      args.retry_limit = 3
86
87    if args.no_browser:
88      return
89
90    if args.start_xvfb and xvfb.ShouldStartXvfb():
91      cls.xvfb_process = xvfb.StartXvfb()
92      # Work around Mesa issues on Linux. See
93      # https://github.com/catapult-project/catapult/issues/3074
94      args.browser_options.AppendExtraBrowserArgs('--disable-gpu')
95
96    try:
97      possible_browser = browser_finder.FindBrowser(args)
98    except browser_finder_exceptions.BrowserFinderException, ex:
99      parser.error(ex)
100
101    if not possible_browser:
102      parser.error('No browser found of type %s. Cannot run tests.\n'
103                   'Re-run with --browser=list to see '
104                   'available browser types.' % args.browser_type)
105
106  @classmethod
107  def main(cls, args=None, stream=None):  # pylint: disable=arguments-differ
108    # We override the superclass so that we can hook in the 'stream' arg.
109    parser = cls.CreateParser()
110    cls.AddCommandLineArgs(parser, None)
111    options, positional_args = parser.parse_args(args)
112    options.positional_args = positional_args
113
114    try:
115      # Must initialize the DependencyManager before calling
116      # browser_finder.FindBrowser(args)
117      binary_manager.InitDependencyManager(options.client_configs)
118      cls.ProcessCommandLineArgs(parser, options, None)
119
120      obj = cls()
121      if stream is not None:
122        obj.stream = stream
123      return obj.Run(options)
124    finally:
125      if cls.xvfb_process:
126        cls.xvfb_process.kill()
127
128  def Run(self, args):
129    runner = typ.Runner()
130    if self.stream:
131      runner.host.stdout = self.stream
132
133    if args.no_browser:
134      possible_browser = None
135      platform = platform_module.GetHostPlatform()
136    else:
137      possible_browser = browser_finder.FindBrowser(args)
138      platform = possible_browser.platform
139
140    fetch_reference_chrome_binary = False
141    # Fetch all binaries needed by telemetry before we run the benchmark.
142    if possible_browser and possible_browser.browser_type == 'reference':
143      fetch_reference_chrome_binary = True
144    binary_manager.FetchBinaryDependencies(
145        platform, args.client_configs, fetch_reference_chrome_binary)
146
147    # Telemetry seems to overload the system if we run one test per core,
148    # so we scale things back a fair amount. Many of the telemetry tests
149    # are long-running, so there's a limit to how much parallelism we
150    # can effectively use for now anyway.
151    #
152    # It should be possible to handle multiple devices if we adjust the
153    # browser_finder code properly, but for now we only handle one on ChromeOS.
154    if platform.GetOSName() == 'chromeos':
155      runner.args.jobs = 1
156    elif platform.GetOSName() == 'android':
157      android_devs = android_device.FindAllAvailableDevices(args)
158      runner.args.jobs = len(android_devs)
159      if runner.args.jobs == 0:
160        raise RuntimeError("No Android device found")
161      print 'Running tests with %d Android device(s).' % runner.args.jobs
162    elif platform.GetOSVersionName() == 'xp':
163      # For an undiagnosed reason, XP falls over with more parallelism.
164      # See crbug.com/388256
165      runner.args.jobs = max(int(args.jobs) // 4, 1)
166    else:
167      runner.args.jobs = max(int(args.jobs) // 2, 1)
168
169    runner.args.skip = args.skip
170    runner.args.metadata = args.metadata
171    runner.args.passthrough = args.passthrough
172    runner.args.path = args.path
173    runner.args.retry_limit = args.retry_limit
174    runner.args.test_results_server = args.test_results_server
175    runner.args.test_type = args.test_type
176    runner.args.top_level_dir = args.top_level_dir
177    runner.args.write_full_results_to = args.write_full_results_to
178    runner.args.write_trace_to = args.write_trace_to
179    runner.args.list_only = args.list_only
180    runner.args.shard_index = args.shard_index
181    runner.args.total_shards = args.total_shards
182
183    runner.args.path.append(util.GetUnittestDataDir())
184
185    # Always print out these info for the ease of debugging.
186    runner.args.timing = True
187    runner.args.verbose = 3
188
189    runner.classifier = GetClassifier(args, possible_browser)
190    runner.context = args
191    runner.setup_fn = _SetUpProcess
192    runner.teardown_fn = _TearDownProcess
193    runner.win_multiprocessing = typ.WinMultiprocessing.importable
194    try:
195      ret, _, _ = runner.run()
196    except KeyboardInterrupt:
197      print >> sys.stderr, "interrupted, exiting"
198      ret = 130
199    return ret
200
201
202def _SkipMatch(name, skipGlobs):
203  return any(fnmatch.fnmatch(name, glob) for glob in skipGlobs)
204
205
206def GetClassifier(args, possible_browser):
207
208  def ClassifyTestWithoutBrowser(test_set, test):
209    name = test.id()
210    if _SkipMatch(name, args.skip):
211      test_set.tests_to_skip.append(
212          typ.TestInput(name, 'skipped because matched --skip'))
213      return
214    if (not args.positional_args
215        or _MatchesSelectedTest(name, args.positional_args,
216                                  args.exact_test_filter)):
217      # TODO(telemetry-team): Make sure that all telemetry unittest that invokes
218      # actual browser are subclasses of browser_test_case.BrowserTestCase
219      # (crbug.com/537428)
220      if issubclass(test.__class__, browser_test_case.BrowserTestCase):
221        test_set.tests_to_skip.append(typ.TestInput(
222            name, msg='Skip the test because it requires a browser.'))
223      else:
224        test_set.parallel_tests.append(typ.TestInput(name))
225
226  def ClassifyTestWithBrowser(test_set, test):
227    name = test.id()
228    if _SkipMatch(name, args.skip):
229      test_set.tests_to_skip.append(
230          typ.TestInput(name, 'skipped because matched --skip'))
231      return
232    if (not args.positional_args
233        or _MatchesSelectedTest(name, args.positional_args,
234                                args.exact_test_filter)):
235      assert hasattr(test, '_testMethodName')
236      method = getattr(
237          test, test._testMethodName)  # pylint: disable=protected-access
238      should_skip, reason = decorators.ShouldSkip(method, possible_browser)
239      if should_skip and not args.run_disabled_tests:
240        test_set.tests_to_skip.append(typ.TestInput(name, msg=reason))
241      elif decorators.ShouldBeIsolated(method, possible_browser):
242        test_set.isolated_tests.append(typ.TestInput(name))
243      else:
244        test_set.parallel_tests.append(typ.TestInput(name))
245
246  if possible_browser:
247    return ClassifyTestWithBrowser
248  else:
249    return ClassifyTestWithoutBrowser
250
251
252def _MatchesSelectedTest(name, selected_tests, selected_tests_are_exact):
253  if not selected_tests:
254    return False
255  if selected_tests_are_exact:
256    return any(name in selected_tests)
257  else:
258    return any(test in name for test in selected_tests)
259
260
261def _SetUpProcess(child, context): # pylint: disable=unused-argument
262  ps_util.EnableListingStrayProcessesUponExitHook()
263  # Make sure that we don't invokes cloud storage I/Os when we run the tests in
264  # parallel.
265  # TODO(nednguyen): always do this once telemetry tests in Chromium is updated
266  # to prefetch files.
267  # (https://github.com/catapult-project/catapult/issues/2192)
268  args = context
269  if args.disable_cloud_storage_io:
270    os.environ[cloud_storage.DISABLE_CLOUD_STORAGE_IO] = '1'
271  if binary_manager.NeedsInit():
272    # Typ doesn't keep the DependencyManager initialization in the child
273    # processes.
274    binary_manager.InitDependencyManager(context.client_configs)
275  # We need to reset the handlers in case some other parts of telemetry already
276  # set it to make this work.
277  if not args.disable_logging_config:
278    logging.getLogger().handlers = []
279    logging.basicConfig(
280        level=logging.INFO,
281        format='(%(levelname)s) %(asctime)s pid=%(process)d'
282               '  %(module)s.%(funcName)s:%(lineno)d'
283               '  %(message)s')
284  if args.remote_platform_options.device == 'android':
285    android_devices = android_device.FindAllAvailableDevices(args)
286    if not android_devices:
287      raise RuntimeError("No Android device found")
288    android_devices.sort(key=lambda device: device.name)
289    args.remote_platform_options.device = (
290        android_devices[child.worker_num-1].guid)
291  options_for_unittests.Push(args)
292
293
294def _TearDownProcess(child, context): # pylint: disable=unused-argument
295  # It's safe to call teardown_browser even if we did not start any browser
296  # in any of the tests.
297  browser_test_case.teardown_browser()
298  options_for_unittests.Pop()
299
300
301if __name__ == '__main__':
302  ret_code = RunTestsCommand.main()
303  sys.exit(ret_code)
304