1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Parses the command line, discovers the appropriate tests, and runs them.
6
7Handles test configuration, but all the logic for
8actually running the test is in Test and PageRunner."""
9
10import hashlib
11import inspect
12import json
13import os
14import sys
15
16from telemetry import benchmark
17from telemetry import decorators
18from telemetry.core import browser_finder
19from telemetry.core import browser_options
20from telemetry.core import command_line
21from telemetry.core import discover
22from telemetry.core import environment
23from telemetry.core import util
24from telemetry.page import page_set
25from telemetry.page import page_test
26from telemetry.page import profile_creator
27from telemetry.util import find_dependencies
28
29
30class Deps(find_dependencies.FindDependenciesCommand):
31  """Prints all dependencies"""
32
33  def Run(self, args):
34    main_module = sys.modules['__main__']
35    args.positional_args.append(os.path.realpath(main_module.__file__))
36    return super(Deps, self).Run(args)
37
38
39class Help(command_line.OptparseCommand):
40  """Display help information about a command"""
41
42  usage = '[command]'
43
44  def Run(self, args):
45    if len(args.positional_args) == 1:
46      commands = _MatchingCommands(args.positional_args[0])
47      if len(commands) == 1:
48        command = commands[0]
49        parser = command.CreateParser()
50        command.AddCommandLineArgs(parser)
51        parser.print_help()
52        return 0
53
54    print >> sys.stderr, ('usage: %s [command] [<options>]' % _ScriptName())
55    print >> sys.stderr, 'Available commands are:'
56    for command in _Commands():
57      print >> sys.stderr, '  %-10s %s' % (
58          command.Name(), command.Description())
59    print >> sys.stderr, ('"%s help <command>" to see usage information '
60                          'for a specific command.' % _ScriptName())
61    return 0
62
63
64class List(command_line.OptparseCommand):
65  """Lists the available tests"""
66
67  usage = '[test_name] [<options>]'
68
69  @classmethod
70  def CreateParser(cls):
71    options = browser_options.BrowserFinderOptions()
72    parser = options.CreateParser('%%prog %s %s' % (cls.Name(), cls.usage))
73    return parser
74
75  @classmethod
76  def AddCommandLineArgs(cls, parser):
77    parser.add_option('-j', '--json-output-file', type='string')
78    parser.add_option('-n', '--num-shards', type='int', default=1)
79
80  @classmethod
81  def ProcessCommandLineArgs(cls, parser, args):
82    if not args.positional_args:
83      args.tests = _Tests()
84    elif len(args.positional_args) == 1:
85      args.tests = _MatchTestName(args.positional_args[0], exact_matches=False)
86    else:
87      parser.error('Must provide at most one test name.')
88
89  def Run(self, args):
90    if args.json_output_file:
91      possible_browser = browser_finder.FindBrowser(args)
92      if args.browser_type in (
93          'exact', 'release', 'release_x64', 'debug', 'debug_x64', 'canary'):
94        args.browser_type = 'reference'
95        possible_reference_browser = browser_finder.FindBrowser(args)
96      else:
97        possible_reference_browser = None
98      with open(args.json_output_file, 'w') as f:
99        f.write(_GetJsonTestList(possible_browser, possible_reference_browser,
100                                 args.tests, args.num_shards))
101    else:
102      _PrintTestList(args.tests)
103    return 0
104
105
106class Run(command_line.OptparseCommand):
107  """Run one or more tests (default)"""
108
109  usage = 'test_name [page_set] [<options>]'
110
111  @classmethod
112  def CreateParser(cls):
113    options = browser_options.BrowserFinderOptions()
114    parser = options.CreateParser('%%prog %s %s' % (cls.Name(), cls.usage))
115    return parser
116
117  @classmethod
118  def AddCommandLineArgs(cls, parser):
119    benchmark.AddCommandLineArgs(parser)
120
121    # Allow tests to add their own command line options.
122    matching_tests = []
123    for arg in sys.argv[1:]:
124      matching_tests += _MatchTestName(arg)
125
126    if matching_tests:
127      # TODO(dtu): After move to argparse, add command-line args for all tests
128      # to subparser. Using subparsers will avoid duplicate arguments.
129      matching_test = matching_tests.pop()
130      matching_test.AddCommandLineArgs(parser)
131      # The test's options override the defaults!
132      matching_test.SetArgumentDefaults(parser)
133
134  @classmethod
135  def ProcessCommandLineArgs(cls, parser, args):
136    if not args.positional_args:
137      _PrintTestList(_Tests())
138      sys.exit(-1)
139
140    input_test_name = args.positional_args[0]
141    matching_tests = _MatchTestName(input_test_name)
142    if not matching_tests:
143      print >> sys.stderr, 'No test named "%s".' % input_test_name
144      print >> sys.stderr
145      _PrintTestList(_Tests())
146      sys.exit(-1)
147
148    if len(matching_tests) > 1:
149      print >> sys.stderr, 'Multiple tests named "%s".' % input_test_name
150      print >> sys.stderr, 'Did you mean one of these?'
151      print >> sys.stderr
152      _PrintTestList(matching_tests)
153      sys.exit(-1)
154
155    test_class = matching_tests.pop()
156    if issubclass(test_class, page_test.PageTest):
157      if len(args.positional_args) < 2:
158        parser.error('Need to specify a page set for "%s".' % test_class.Name())
159      if len(args.positional_args) > 2:
160        parser.error('Too many arguments.')
161      page_set_name = args.positional_args[1]
162      page_set_class = _MatchPageSetName(page_set_name)
163      if page_set_class is None:
164        parser.error("Page set %s not found. Available sets:\n%s" %
165                     (page_set_name, _AvailablePageSetNamesString()))
166
167      class TestWrapper(benchmark.Benchmark):
168        test = test_class
169
170        @classmethod
171        def CreatePageSet(cls, options):
172          return page_set_class()
173
174      test_class = TestWrapper
175    else:
176      if len(args.positional_args) > 1:
177        parser.error('Too many arguments.')
178
179    assert issubclass(test_class, benchmark.Benchmark), (
180        'Trying to run a non-Benchmark?!')
181
182    benchmark.ProcessCommandLineArgs(parser, args)
183    test_class.ProcessCommandLineArgs(parser, args)
184
185    cls._test = test_class
186
187  def Run(self, args):
188    return min(255, self._test().Run(args))
189
190
191def _ScriptName():
192  return os.path.basename(sys.argv[0])
193
194
195def _Commands():
196  """Generates a list of all classes in this file that subclass Command."""
197  for _, cls in inspect.getmembers(sys.modules[__name__]):
198    if not inspect.isclass(cls):
199      continue
200    if not issubclass(cls, command_line.Command):
201      continue
202    yield cls
203
204def _MatchingCommands(string):
205  return [command for command in _Commands()
206         if command.Name().startswith(string)]
207
208@decorators.Cache
209def _Tests():
210  tests = []
211  for base_dir in config.base_paths:
212    tests += discover.DiscoverClasses(base_dir, base_dir, benchmark.Benchmark,
213                                      index_by_class_name=True).values()
214    page_tests = discover.DiscoverClasses(base_dir, base_dir,
215                                          page_test.PageTest,
216                                          index_by_class_name=True).values()
217    tests += [test_class for test_class in page_tests
218              if not issubclass(test_class, profile_creator.ProfileCreator)]
219  return tests
220
221
222# TODO(ariblue): Use discover.py's abstracted _MatchName class (in pending CL
223# 432543003) and eliminate _MatchPageSetName and _MatchTestName.
224def _MatchPageSetName(input_name):
225  page_sets = []
226  for base_dir in config.base_paths:
227    page_sets += discover.DiscoverClasses(base_dir, base_dir, page_set.PageSet,
228                                          index_by_class_name=True).values()
229  for p in page_sets:
230    if input_name == p.Name():
231      return p
232  return None
233
234
235def _AvailablePageSetNamesString():
236  result = ""
237  for base_dir in config.base_paths:
238    for p in discover.DiscoverClasses(base_dir, base_dir, page_set.PageSet,
239                                      index_by_class_name=True).values():
240      result += p.Name() + "\n"
241  return result
242
243
244def _MatchTestName(input_test_name, exact_matches=True):
245  def _Matches(input_string, search_string):
246    if search_string.startswith(input_string):
247      return True
248    for part in search_string.split('.'):
249      if part.startswith(input_string):
250        return True
251    return False
252
253  # Exact matching.
254  if exact_matches:
255    # Don't add aliases to search dict, only allow exact matching for them.
256    if input_test_name in config.test_aliases:
257      exact_match = config.test_aliases[input_test_name]
258    else:
259      exact_match = input_test_name
260
261    for test_class in _Tests():
262      if exact_match == test_class.Name():
263        return [test_class]
264    return []
265
266  # Fuzzy matching.
267  return [test_class for test_class in _Tests()
268          if _Matches(input_test_name, test_class.Name())]
269
270
271def _GetJsonTestList(possible_browser, possible_reference_browser,
272                     test_classes, num_shards):
273  """Returns a list of all enabled tests in a JSON format expected by buildbots.
274
275  JSON format (see build/android/pylib/perf/test_runner.py):
276  { "version": <int>,
277    "steps": {
278      <string>: {
279        "device_affinity": <int>,
280        "cmd": <string>,
281        "perf_dashboard_id": <string>,
282      },
283      ...
284    }
285  }
286  """
287  output = {
288    'version': 1,
289    'steps': {
290    }
291  }
292  for test_class in test_classes:
293    if not issubclass(test_class, benchmark.Benchmark):
294      continue
295    if not decorators.IsEnabled(test_class, possible_browser):
296      continue
297
298    base_name = test_class.Name()
299    base_cmd = [sys.executable, os.path.realpath(sys.argv[0]),
300                '-v', '--output-format=buildbot', base_name]
301    perf_dashboard_id = base_name
302    # TODO(tonyg): Currently we set the device affinity to a stable hash of the
303    # test name. This somewhat evenly distributes benchmarks among the requested
304    # number of shards. However, it is far from optimal in terms of cycle time.
305    # We should add a test size decorator (e.g. small, medium, large) and let
306    # that inform sharding.
307    device_affinity = int(hashlib.sha1(base_name).hexdigest(), 16) % num_shards
308
309    output['steps'][base_name] = {
310      'cmd': ' '.join(base_cmd + [
311            '--browser=%s' % possible_browser.browser_type]),
312      'device_affinity': device_affinity,
313      'perf_dashboard_id': perf_dashboard_id,
314    }
315    if (possible_reference_browser and
316        decorators.IsEnabled(test_class, possible_reference_browser)):
317      output['steps'][base_name + '.reference'] = {
318        'cmd': ' '.join(base_cmd + [
319              '--browser=reference', '--output-trace-tag=_ref']),
320        'device_affinity': device_affinity,
321        'perf_dashboard_id': perf_dashboard_id,
322      }
323
324  return json.dumps(output, indent=2, sort_keys=True)
325
326
327def _PrintTestList(tests):
328  if not tests:
329    print >> sys.stderr, 'No tests found!'
330    return
331
332  # Align the test names to the longest one.
333  format_string = '  %%-%ds %%s' % max(len(t.Name()) for t in tests)
334
335  filtered_tests = [test_class for test_class in tests
336                    if issubclass(test_class, benchmark.Benchmark)]
337  if filtered_tests:
338    print >> sys.stderr, 'Available tests are:'
339    for test_class in sorted(filtered_tests, key=lambda t: t.Name()):
340      print >> sys.stderr, format_string % (
341          test_class.Name(), test_class.Description())
342    print >> sys.stderr
343
344  filtered_tests = [test_class for test_class in tests
345                    if issubclass(test_class, page_test.PageTest)]
346  if filtered_tests:
347    print >> sys.stderr, 'Available page tests are:'
348    for test_class in sorted(filtered_tests, key=lambda t: t.Name()):
349      print >> sys.stderr, format_string % (
350          test_class.Name(), test_class.Description())
351    print >> sys.stderr
352
353
354config = environment.Environment([util.GetBaseDir()])
355
356
357def main():
358  # Get the command name from the command line.
359  if len(sys.argv) > 1 and sys.argv[1] == '--help':
360    sys.argv[1] = 'help'
361
362  command_name = 'run'
363  for arg in sys.argv[1:]:
364    if not arg.startswith('-'):
365      command_name = arg
366      break
367
368  # Validate and interpret the command name.
369  commands = _MatchingCommands(command_name)
370  if len(commands) > 1:
371    print >> sys.stderr, ('"%s" is not a %s command. Did you mean one of these?'
372                          % (command_name, _ScriptName()))
373    for command in commands:
374      print >> sys.stderr, '  %-10s %s' % (
375          command.Name(), command.Description())
376    return 1
377  if commands:
378    command = commands[0]
379  else:
380    command = Run
381
382  # Parse and run the command.
383  parser = command.CreateParser()
384  command.AddCommandLineArgs(parser)
385  options, args = parser.parse_args()
386  if commands:
387    args = args[1:]
388  options.positional_args = args
389  command.ProcessCommandLineArgs(parser, options)
390  return command().Run(options)
391