1# Copyright 2013 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Parses the command line, discovers the appropriate tests, and runs them. 6 7Handles test configuration, but all the logic for 8actually running the test is in Test and PageRunner.""" 9 10import hashlib 11import inspect 12import json 13import os 14import sys 15 16from telemetry import benchmark 17from telemetry import decorators 18from telemetry.core import browser_finder 19from telemetry.core import browser_options 20from telemetry.core import command_line 21from telemetry.core import discover 22from telemetry.core import environment 23from telemetry.core import util 24from telemetry.page import page_set 25from telemetry.page import page_test 26from telemetry.page import profile_creator 27from telemetry.util import find_dependencies 28 29 30class Deps(find_dependencies.FindDependenciesCommand): 31 """Prints all dependencies""" 32 33 def Run(self, args): 34 main_module = sys.modules['__main__'] 35 args.positional_args.append(os.path.realpath(main_module.__file__)) 36 return super(Deps, self).Run(args) 37 38 39class Help(command_line.OptparseCommand): 40 """Display help information about a command""" 41 42 usage = '[command]' 43 44 def Run(self, args): 45 if len(args.positional_args) == 1: 46 commands = _MatchingCommands(args.positional_args[0]) 47 if len(commands) == 1: 48 command = commands[0] 49 parser = command.CreateParser() 50 command.AddCommandLineArgs(parser) 51 parser.print_help() 52 return 0 53 54 print >> sys.stderr, ('usage: %s [command] [<options>]' % _ScriptName()) 55 print >> sys.stderr, 'Available commands are:' 56 for command in _Commands(): 57 print >> sys.stderr, ' %-10s %s' % ( 58 command.Name(), command.Description()) 59 print >> sys.stderr, ('"%s help <command>" to see usage information ' 60 'for a specific command.' % _ScriptName()) 61 return 0 62 63 64class List(command_line.OptparseCommand): 65 """Lists the available tests""" 66 67 usage = '[test_name] [<options>]' 68 69 @classmethod 70 def CreateParser(cls): 71 options = browser_options.BrowserFinderOptions() 72 parser = options.CreateParser('%%prog %s %s' % (cls.Name(), cls.usage)) 73 return parser 74 75 @classmethod 76 def AddCommandLineArgs(cls, parser): 77 parser.add_option('-j', '--json-output-file', type='string') 78 parser.add_option('-n', '--num-shards', type='int', default=1) 79 80 @classmethod 81 def ProcessCommandLineArgs(cls, parser, args): 82 if not args.positional_args: 83 args.tests = _Tests() 84 elif len(args.positional_args) == 1: 85 args.tests = _MatchTestName(args.positional_args[0], exact_matches=False) 86 else: 87 parser.error('Must provide at most one test name.') 88 89 def Run(self, args): 90 if args.json_output_file: 91 possible_browser = browser_finder.FindBrowser(args) 92 if args.browser_type in ( 93 'exact', 'release', 'release_x64', 'debug', 'debug_x64', 'canary'): 94 args.browser_type = 'reference' 95 possible_reference_browser = browser_finder.FindBrowser(args) 96 else: 97 possible_reference_browser = None 98 with open(args.json_output_file, 'w') as f: 99 f.write(_GetJsonTestList(possible_browser, possible_reference_browser, 100 args.tests, args.num_shards)) 101 else: 102 _PrintTestList(args.tests) 103 return 0 104 105 106class Run(command_line.OptparseCommand): 107 """Run one or more tests (default)""" 108 109 usage = 'test_name [page_set] [<options>]' 110 111 @classmethod 112 def CreateParser(cls): 113 options = browser_options.BrowserFinderOptions() 114 parser = options.CreateParser('%%prog %s %s' % (cls.Name(), cls.usage)) 115 return parser 116 117 @classmethod 118 def AddCommandLineArgs(cls, parser): 119 benchmark.AddCommandLineArgs(parser) 120 121 # Allow tests to add their own command line options. 122 matching_tests = [] 123 for arg in sys.argv[1:]: 124 matching_tests += _MatchTestName(arg) 125 126 if matching_tests: 127 # TODO(dtu): After move to argparse, add command-line args for all tests 128 # to subparser. Using subparsers will avoid duplicate arguments. 129 matching_test = matching_tests.pop() 130 matching_test.AddCommandLineArgs(parser) 131 # The test's options override the defaults! 132 matching_test.SetArgumentDefaults(parser) 133 134 @classmethod 135 def ProcessCommandLineArgs(cls, parser, args): 136 if not args.positional_args: 137 _PrintTestList(_Tests()) 138 sys.exit(-1) 139 140 input_test_name = args.positional_args[0] 141 matching_tests = _MatchTestName(input_test_name) 142 if not matching_tests: 143 print >> sys.stderr, 'No test named "%s".' % input_test_name 144 print >> sys.stderr 145 _PrintTestList(_Tests()) 146 sys.exit(-1) 147 148 if len(matching_tests) > 1: 149 print >> sys.stderr, 'Multiple tests named "%s".' % input_test_name 150 print >> sys.stderr, 'Did you mean one of these?' 151 print >> sys.stderr 152 _PrintTestList(matching_tests) 153 sys.exit(-1) 154 155 test_class = matching_tests.pop() 156 if issubclass(test_class, page_test.PageTest): 157 if len(args.positional_args) < 2: 158 parser.error('Need to specify a page set for "%s".' % test_class.Name()) 159 if len(args.positional_args) > 2: 160 parser.error('Too many arguments.') 161 page_set_name = args.positional_args[1] 162 page_set_class = _MatchPageSetName(page_set_name) 163 if page_set_class is None: 164 parser.error("Page set %s not found. Available sets:\n%s" % 165 (page_set_name, _AvailablePageSetNamesString())) 166 167 class TestWrapper(benchmark.Benchmark): 168 test = test_class 169 170 @classmethod 171 def CreatePageSet(cls, options): 172 return page_set_class() 173 174 test_class = TestWrapper 175 else: 176 if len(args.positional_args) > 1: 177 parser.error('Too many arguments.') 178 179 assert issubclass(test_class, benchmark.Benchmark), ( 180 'Trying to run a non-Benchmark?!') 181 182 benchmark.ProcessCommandLineArgs(parser, args) 183 test_class.ProcessCommandLineArgs(parser, args) 184 185 cls._test = test_class 186 187 def Run(self, args): 188 return min(255, self._test().Run(args)) 189 190 191def _ScriptName(): 192 return os.path.basename(sys.argv[0]) 193 194 195def _Commands(): 196 """Generates a list of all classes in this file that subclass Command.""" 197 for _, cls in inspect.getmembers(sys.modules[__name__]): 198 if not inspect.isclass(cls): 199 continue 200 if not issubclass(cls, command_line.Command): 201 continue 202 yield cls 203 204def _MatchingCommands(string): 205 return [command for command in _Commands() 206 if command.Name().startswith(string)] 207 208@decorators.Cache 209def _Tests(): 210 tests = [] 211 for base_dir in config.base_paths: 212 tests += discover.DiscoverClasses(base_dir, base_dir, benchmark.Benchmark, 213 index_by_class_name=True).values() 214 page_tests = discover.DiscoverClasses(base_dir, base_dir, 215 page_test.PageTest, 216 index_by_class_name=True).values() 217 tests += [test_class for test_class in page_tests 218 if not issubclass(test_class, profile_creator.ProfileCreator)] 219 return tests 220 221 222# TODO(ariblue): Use discover.py's abstracted _MatchName class (in pending CL 223# 432543003) and eliminate _MatchPageSetName and _MatchTestName. 224def _MatchPageSetName(input_name): 225 page_sets = [] 226 for base_dir in config.base_paths: 227 page_sets += discover.DiscoverClasses(base_dir, base_dir, page_set.PageSet, 228 index_by_class_name=True).values() 229 for p in page_sets: 230 if input_name == p.Name(): 231 return p 232 return None 233 234 235def _AvailablePageSetNamesString(): 236 result = "" 237 for base_dir in config.base_paths: 238 for p in discover.DiscoverClasses(base_dir, base_dir, page_set.PageSet, 239 index_by_class_name=True).values(): 240 result += p.Name() + "\n" 241 return result 242 243 244def _MatchTestName(input_test_name, exact_matches=True): 245 def _Matches(input_string, search_string): 246 if search_string.startswith(input_string): 247 return True 248 for part in search_string.split('.'): 249 if part.startswith(input_string): 250 return True 251 return False 252 253 # Exact matching. 254 if exact_matches: 255 # Don't add aliases to search dict, only allow exact matching for them. 256 if input_test_name in config.test_aliases: 257 exact_match = config.test_aliases[input_test_name] 258 else: 259 exact_match = input_test_name 260 261 for test_class in _Tests(): 262 if exact_match == test_class.Name(): 263 return [test_class] 264 return [] 265 266 # Fuzzy matching. 267 return [test_class for test_class in _Tests() 268 if _Matches(input_test_name, test_class.Name())] 269 270 271def _GetJsonTestList(possible_browser, possible_reference_browser, 272 test_classes, num_shards): 273 """Returns a list of all enabled tests in a JSON format expected by buildbots. 274 275 JSON format (see build/android/pylib/perf/test_runner.py): 276 { "version": <int>, 277 "steps": { 278 <string>: { 279 "device_affinity": <int>, 280 "cmd": <string>, 281 "perf_dashboard_id": <string>, 282 }, 283 ... 284 } 285 } 286 """ 287 output = { 288 'version': 1, 289 'steps': { 290 } 291 } 292 for test_class in test_classes: 293 if not issubclass(test_class, benchmark.Benchmark): 294 continue 295 if not decorators.IsEnabled(test_class, possible_browser): 296 continue 297 298 base_name = test_class.Name() 299 base_cmd = [sys.executable, os.path.realpath(sys.argv[0]), 300 '-v', '--output-format=buildbot', base_name] 301 perf_dashboard_id = base_name 302 # TODO(tonyg): Currently we set the device affinity to a stable hash of the 303 # test name. This somewhat evenly distributes benchmarks among the requested 304 # number of shards. However, it is far from optimal in terms of cycle time. 305 # We should add a test size decorator (e.g. small, medium, large) and let 306 # that inform sharding. 307 device_affinity = int(hashlib.sha1(base_name).hexdigest(), 16) % num_shards 308 309 output['steps'][base_name] = { 310 'cmd': ' '.join(base_cmd + [ 311 '--browser=%s' % possible_browser.browser_type]), 312 'device_affinity': device_affinity, 313 'perf_dashboard_id': perf_dashboard_id, 314 } 315 if (possible_reference_browser and 316 decorators.IsEnabled(test_class, possible_reference_browser)): 317 output['steps'][base_name + '.reference'] = { 318 'cmd': ' '.join(base_cmd + [ 319 '--browser=reference', '--output-trace-tag=_ref']), 320 'device_affinity': device_affinity, 321 'perf_dashboard_id': perf_dashboard_id, 322 } 323 324 return json.dumps(output, indent=2, sort_keys=True) 325 326 327def _PrintTestList(tests): 328 if not tests: 329 print >> sys.stderr, 'No tests found!' 330 return 331 332 # Align the test names to the longest one. 333 format_string = ' %%-%ds %%s' % max(len(t.Name()) for t in tests) 334 335 filtered_tests = [test_class for test_class in tests 336 if issubclass(test_class, benchmark.Benchmark)] 337 if filtered_tests: 338 print >> sys.stderr, 'Available tests are:' 339 for test_class in sorted(filtered_tests, key=lambda t: t.Name()): 340 print >> sys.stderr, format_string % ( 341 test_class.Name(), test_class.Description()) 342 print >> sys.stderr 343 344 filtered_tests = [test_class for test_class in tests 345 if issubclass(test_class, page_test.PageTest)] 346 if filtered_tests: 347 print >> sys.stderr, 'Available page tests are:' 348 for test_class in sorted(filtered_tests, key=lambda t: t.Name()): 349 print >> sys.stderr, format_string % ( 350 test_class.Name(), test_class.Description()) 351 print >> sys.stderr 352 353 354config = environment.Environment([util.GetBaseDir()]) 355 356 357def main(): 358 # Get the command name from the command line. 359 if len(sys.argv) > 1 and sys.argv[1] == '--help': 360 sys.argv[1] = 'help' 361 362 command_name = 'run' 363 for arg in sys.argv[1:]: 364 if not arg.startswith('-'): 365 command_name = arg 366 break 367 368 # Validate and interpret the command name. 369 commands = _MatchingCommands(command_name) 370 if len(commands) > 1: 371 print >> sys.stderr, ('"%s" is not a %s command. Did you mean one of these?' 372 % (command_name, _ScriptName())) 373 for command in commands: 374 print >> sys.stderr, ' %-10s %s' % ( 375 command.Name(), command.Description()) 376 return 1 377 if commands: 378 command = commands[0] 379 else: 380 command = Run 381 382 # Parse and run the command. 383 parser = command.CreateParser() 384 command.AddCommandLineArgs(parser) 385 options, args = parser.parse_args() 386 if commands: 387 args = args[1:] 388 options.positional_args = args 389 command.ProcessCommandLineArgs(parser, options) 390 return command().Run(options) 391