1#!/usr/bin/env python
2#
3# Copyright 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Runs all types of tests from one unified interface.
8
9TODO(gkanwar):
10* Add options to run Monkey tests.
11"""
12
13import collections
14import optparse
15import os
16import shutil
17import sys
18
19from pylib import constants
20from pylib import ports
21from pylib.base import base_test_result
22from pylib.base import test_dispatcher
23from pylib.gtest import gtest_config
24from pylib.gtest import setup as gtest_setup
25from pylib.gtest import test_options as gtest_test_options
26from pylib.host_driven import setup as host_driven_setup
27from pylib.instrumentation import setup as instrumentation_setup
28from pylib.instrumentation import test_options as instrumentation_test_options
29from pylib.monkey import setup as monkey_setup
30from pylib.monkey import test_options as monkey_test_options
31from pylib.uiautomator import setup as uiautomator_setup
32from pylib.uiautomator import test_options as uiautomator_test_options
33from pylib.utils import report_results
34from pylib.utils import run_tests_helper
35
36
37_SDK_OUT_DIR = os.path.join(constants.DIR_SOURCE_ROOT, 'out')
38
39
40def AddBuildTypeOption(option_parser):
41  """Adds the build type option to |option_parser|."""
42  default_build_type = 'Debug'
43  if 'BUILDTYPE' in os.environ:
44    default_build_type = os.environ['BUILDTYPE']
45  option_parser.add_option('--debug', action='store_const', const='Debug',
46                           dest='build_type', default=default_build_type,
47                           help=('If set, run test suites under out/Debug. '
48                                 'Default is env var BUILDTYPE or Debug.'))
49  option_parser.add_option('--release', action='store_const',
50                           const='Release', dest='build_type',
51                           help=('If set, run test suites under out/Release.'
52                                 ' Default is env var BUILDTYPE or Debug.'))
53
54
55def AddCommonOptions(option_parser):
56  """Adds all common options to |option_parser|."""
57
58  AddBuildTypeOption(option_parser)
59
60  option_parser.add_option('-c', dest='cleanup_test_files',
61                           help='Cleanup test files on the device after run',
62                           action='store_true')
63  option_parser.add_option('--num_retries', dest='num_retries', type='int',
64                           default=2,
65                           help=('Number of retries for a test before '
66                                 'giving up.'))
67  option_parser.add_option('-v',
68                           '--verbose',
69                           dest='verbose_count',
70                           default=0,
71                           action='count',
72                           help='Verbose level (multiple times for more)')
73  option_parser.add_option('--tool',
74                           dest='tool',
75                           help=('Run the test under a tool '
76                                 '(use --tool help to list them)'))
77  option_parser.add_option('--flakiness-dashboard-server',
78                           dest='flakiness_dashboard_server',
79                           help=('Address of the server that is hosting the '
80                                 'Chrome for Android flakiness dashboard.'))
81  option_parser.add_option('--skip-deps-push', dest='push_deps',
82                           action='store_false', default=True,
83                           help=('Do not push dependencies to the device. '
84                                 'Use this at own risk for speeding up test '
85                                 'execution on local machine.'))
86  option_parser.add_option('-d', '--device', dest='test_device',
87                           help=('Target device for the test suite '
88                                 'to run on.'))
89
90
91def ProcessCommonOptions(options):
92  """Processes and handles all common options."""
93  run_tests_helper.SetLogLevel(options.verbose_count)
94
95
96def AddGTestOptions(option_parser):
97  """Adds gtest options to |option_parser|."""
98
99  option_parser.usage = '%prog gtest [options]'
100  option_parser.command_list = []
101  option_parser.example = '%prog gtest -s base_unittests'
102
103  # TODO(gkanwar): Make this option required
104  option_parser.add_option('-s', '--suite', dest='suite_name',
105                           help=('Executable name of the test suite to run '
106                                 '(use -s help to list them).'))
107  option_parser.add_option('-f', '--gtest_filter', dest='test_filter',
108                           help='googletest-style filter string.')
109  option_parser.add_option('-a', '--test_arguments', dest='test_arguments',
110                           help='Additional arguments to pass to the test.')
111  option_parser.add_option('-t', dest='timeout',
112                           help='Timeout to wait for each test',
113                           type='int',
114                           default=60)
115  # TODO(gkanwar): Move these to Common Options once we have the plumbing
116  # in our other test types to handle these commands
117  AddCommonOptions(option_parser)
118
119
120def ProcessGTestOptions(options):
121  """Intercept test suite help to list test suites.
122
123  Args:
124    options: Command line options.
125  """
126  if options.suite_name == 'help':
127    print 'Available test suites are:'
128    for test_suite in (gtest_config.STABLE_TEST_SUITES +
129                       gtest_config.EXPERIMENTAL_TEST_SUITES):
130      print test_suite
131    sys.exit(0)
132
133  # Convert to a list, assuming all test suites if nothing was specified.
134  # TODO(gkanwar): Require having a test suite
135  if options.suite_name:
136    options.suite_name = [options.suite_name]
137  else:
138    options.suite_name = [s for s in gtest_config.STABLE_TEST_SUITES]
139
140
141def AddJavaTestOptions(option_parser):
142  """Adds the Java test options to |option_parser|."""
143
144  option_parser.add_option('-f', '--test_filter', dest='test_filter',
145                           help=('Test filter (if not fully qualified, '
146                                 'will run all matches).'))
147  option_parser.add_option(
148      '-A', '--annotation', dest='annotation_str',
149      help=('Comma-separated list of annotations. Run only tests with any of '
150            'the given annotations. An annotation can be either a key or a '
151            'key-values pair. A test that has no annotation is considered '
152            '"SmallTest".'))
153  option_parser.add_option(
154      '-E', '--exclude-annotation', dest='exclude_annotation_str',
155      help=('Comma-separated list of annotations. Exclude tests with these '
156            'annotations.'))
157  option_parser.add_option('--screenshot', dest='screenshot_failures',
158                           action='store_true',
159                           help='Capture screenshots of test failures')
160  option_parser.add_option('--save-perf-json', action='store_true',
161                           help='Saves the JSON file for each UI Perf test.')
162  option_parser.add_option('--official-build', action='store_true',
163                           help='Run official build tests.')
164  option_parser.add_option('--keep_test_server_ports',
165                           action='store_true',
166                           help=('Indicates the test server ports must be '
167                                 'kept. When this is run via a sharder '
168                                 'the test server ports should be kept and '
169                                 'should not be reset.'))
170  option_parser.add_option('--test_data', action='append', default=[],
171                           help=('Each instance defines a directory of test '
172                                 'data that should be copied to the target(s) '
173                                 'before running the tests. The argument '
174                                 'should be of the form <target>:<source>, '
175                                 '<target> is relative to the device data'
176                                 'directory, and <source> is relative to the '
177                                 'chromium build directory.'))
178
179
180def ProcessJavaTestOptions(options, error_func):
181  """Processes options/arguments and populates |options| with defaults."""
182
183  if options.annotation_str:
184    options.annotations = options.annotation_str.split(',')
185  elif options.test_filter:
186    options.annotations = []
187  else:
188    options.annotations = ['Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
189                           'EnormousTest']
190
191  if options.exclude_annotation_str:
192    options.exclude_annotations = options.exclude_annotation_str.split(',')
193  else:
194    options.exclude_annotations = []
195
196  if not options.keep_test_server_ports:
197    if not ports.ResetTestServerPortAllocation():
198      raise Exception('Failed to reset test server port.')
199
200
201def AddInstrumentationTestOptions(option_parser):
202  """Adds Instrumentation test options to |option_parser|."""
203
204  option_parser.usage = '%prog instrumentation [options]'
205  option_parser.command_list = []
206  option_parser.example = ('%prog instrumentation '
207                           '--test-apk=ChromiumTestShellTest')
208
209  AddJavaTestOptions(option_parser)
210  AddCommonOptions(option_parser)
211
212  option_parser.add_option('-j', '--java_only', action='store_true',
213                           default=False, help='Run only the Java tests.')
214  option_parser.add_option('-p', '--python_only', action='store_true',
215                           default=False,
216                           help='Run only the host-driven tests.')
217  option_parser.add_option('--python_test_root',
218                           help='Root of the host-driven tests.')
219  option_parser.add_option('-w', '--wait_debugger', dest='wait_for_debugger',
220                           action='store_true',
221                           help='Wait for debugger.')
222  option_parser.add_option(
223      '--test-apk', dest='test_apk',
224      help=('The name of the apk containing the tests '
225            '(without the .apk extension; e.g. "ContentShellTest"). '
226            'Alternatively, this can be a full path to the apk.'))
227
228
229def ProcessInstrumentationOptions(options, error_func):
230  """Processes options/arguments and populate |options| with defaults.
231
232  Args:
233    options: optparse.Options object.
234    error_func: Function to call with the error message in case of an error.
235
236  Returns:
237    An InstrumentationOptions named tuple which contains all options relevant to
238    instrumentation tests.
239  """
240
241  ProcessJavaTestOptions(options, error_func)
242
243  if options.java_only and options.python_only:
244    error_func('Options java_only (-j) and python_only (-p) '
245               'are mutually exclusive.')
246  options.run_java_tests = True
247  options.run_python_tests = True
248  if options.java_only:
249    options.run_python_tests = False
250  elif options.python_only:
251    options.run_java_tests = False
252
253  if not options.python_test_root:
254    options.run_python_tests = False
255
256  if not options.test_apk:
257    error_func('--test-apk must be specified.')
258
259  if os.path.exists(options.test_apk):
260    # The APK is fully qualified, assume the JAR lives along side.
261    options.test_apk_path = options.test_apk
262    options.test_apk_jar_path = (os.path.splitext(options.test_apk_path)[0] +
263                                 '.jar')
264  else:
265    options.test_apk_path = os.path.join(_SDK_OUT_DIR,
266                                         options.build_type,
267                                         constants.SDK_BUILD_APKS_DIR,
268                                         '%s.apk' % options.test_apk)
269    options.test_apk_jar_path = os.path.join(
270        _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_TEST_JAVALIB_DIR,
271        '%s.jar' %  options.test_apk)
272
273  return instrumentation_test_options.InstrumentationOptions(
274      options.build_type,
275      options.tool,
276      options.cleanup_test_files,
277      options.push_deps,
278      options.annotations,
279      options.exclude_annotations,
280      options.test_filter,
281      options.test_data,
282      options.save_perf_json,
283      options.screenshot_failures,
284      options.wait_for_debugger,
285      options.test_apk,
286      options.test_apk_path,
287      options.test_apk_jar_path)
288
289
290def AddUIAutomatorTestOptions(option_parser):
291  """Adds UI Automator test options to |option_parser|."""
292
293  option_parser.usage = '%prog uiautomator [options]'
294  option_parser.command_list = []
295  option_parser.example = (
296      '%prog uiautomator --test-jar=chromium_testshell_uiautomator_tests'
297      ' --package-name=org.chromium.chrome.testshell')
298  option_parser.add_option(
299      '--package-name',
300      help='The package name used by the apk containing the application.')
301  option_parser.add_option(
302      '--test-jar', dest='test_jar',
303      help=('The name of the dexed jar containing the tests (without the '
304            '.dex.jar extension). Alternatively, this can be a full path '
305            'to the jar.'))
306
307  AddJavaTestOptions(option_parser)
308  AddCommonOptions(option_parser)
309
310
311def ProcessUIAutomatorOptions(options, error_func):
312  """Processes UIAutomator options/arguments.
313
314  Args:
315    options: optparse.Options object.
316    error_func: Function to call with the error message in case of an error.
317
318  Returns:
319    A UIAutomatorOptions named tuple which contains all options relevant to
320    uiautomator tests.
321  """
322
323  ProcessJavaTestOptions(options, error_func)
324
325  if not options.package_name:
326    error_func('--package-name must be specified.')
327
328  if not options.test_jar:
329    error_func('--test-jar must be specified.')
330
331  if os.path.exists(options.test_jar):
332    # The dexed JAR is fully qualified, assume the info JAR lives along side.
333    options.uiautomator_jar = options.test_jar
334  else:
335    options.uiautomator_jar = os.path.join(
336        _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_JAVALIB_DIR,
337        '%s.dex.jar' % options.test_jar)
338  options.uiautomator_info_jar = (
339      options.uiautomator_jar[:options.uiautomator_jar.find('.dex.jar')] +
340      '_java.jar')
341
342  return uiautomator_test_options.UIAutomatorOptions(
343      options.build_type,
344      options.tool,
345      options.cleanup_test_files,
346      options.push_deps,
347      options.annotations,
348      options.exclude_annotations,
349      options.test_filter,
350      options.test_data,
351      options.save_perf_json,
352      options.screenshot_failures,
353      options.uiautomator_jar,
354      options.uiautomator_info_jar,
355      options.package_name)
356
357
358def AddMonkeyTestOptions(option_parser):
359  """Adds monkey test options to |option_parser|."""
360
361  option_parser.usage = '%prog monkey [options]'
362  option_parser.command_list = []
363  option_parser.example = (
364      '%prog monkey --package-name=org.chromium.content_shell_apk'
365      ' --activity-name=.ContentShellActivity')
366
367  option_parser.add_option('--package-name', help='Allowed package.')
368  option_parser.add_option(
369      '--activity-name', help='Name of the activity to start.')
370  option_parser.add_option(
371      '--event-count', default=10000, type='int',
372      help='Number of events to generate [default: %default].')
373  option_parser.add_option(
374      '--category', default='',
375      help='A list of allowed categories.')
376  option_parser.add_option(
377      '--throttle', default=100, type='int',
378      help='Delay between events (ms) [default: %default]. ')
379  option_parser.add_option(
380      '--seed', type='int',
381      help=('Seed value for pseudo-random generator. Same seed value generates '
382            'the same sequence of events. Seed is randomized by default.'))
383  option_parser.add_option(
384      '--extra-args', default='',
385      help=('String of other args to pass to the command verbatim '
386            '[default: "%default"].'))
387
388  AddCommonOptions(option_parser)
389
390
391def ProcessMonkeyTestOptions(options, error_func):
392  """Processes all monkey test options.
393
394  Args:
395    options: optparse.Options object.
396    error_func: Function to call with the error message in case of an error.
397
398  Returns:
399    A MonkeyOptions named tuple which contains all options relevant to
400    monkey tests.
401  """
402  if not options.package_name:
403    error_func('Package name is required.')
404
405  category = options.category
406  if category:
407    category = options.category.split(',')
408
409  return monkey_test_options.MonkeyOptions(
410      options.build_type,
411      options.verbose_count,
412      options.package_name,
413      options.activity_name,
414      options.event_count,
415      category,
416      options.throttle,
417      options.seed,
418      options.extra_args)
419
420
421def _RunGTests(options, error_func):
422  """Subcommand of RunTestsCommands which runs gtests."""
423  ProcessGTestOptions(options)
424
425  exit_code = 0
426  for suite_name in options.suite_name:
427    # TODO(gkanwar): Move this into ProcessGTestOptions once we require -s for
428    # the gtest command.
429    gtest_options = gtest_test_options.GTestOptions(
430        options.build_type,
431        options.tool,
432        options.cleanup_test_files,
433        options.push_deps,
434        options.test_filter,
435        options.test_arguments,
436        options.timeout,
437        suite_name)
438    runner_factory, tests = gtest_setup.Setup(gtest_options)
439
440    results, test_exit_code = test_dispatcher.RunTests(
441        tests, runner_factory, False, options.test_device,
442        shard=True,
443        build_type=options.build_type,
444        test_timeout=None,
445        num_retries=options.num_retries)
446
447    if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
448      exit_code = test_exit_code
449
450    report_results.LogFull(
451        results=results,
452        test_type='Unit test',
453        test_package=suite_name,
454        build_type=options.build_type,
455        flakiness_server=options.flakiness_dashboard_server)
456
457  if os.path.isdir(constants.ISOLATE_DEPS_DIR):
458    shutil.rmtree(constants.ISOLATE_DEPS_DIR)
459
460  return exit_code
461
462
463def _RunInstrumentationTests(options, error_func):
464  """Subcommand of RunTestsCommands which runs instrumentation tests."""
465  instrumentation_options = ProcessInstrumentationOptions(options, error_func)
466
467  results = base_test_result.TestRunResults()
468  exit_code = 0
469
470  if options.run_java_tests:
471    runner_factory, tests = instrumentation_setup.Setup(instrumentation_options)
472
473    test_results, exit_code = test_dispatcher.RunTests(
474        tests, runner_factory, options.wait_for_debugger,
475        options.test_device,
476        shard=True,
477        build_type=options.build_type,
478        test_timeout=None,
479        num_retries=options.num_retries)
480
481    results.AddTestRunResults(test_results)
482
483  if options.run_python_tests:
484    runner_factory, tests = host_driven_setup.InstrumentationSetup(
485        options.python_test_root, options.official_build,
486        instrumentation_options)
487
488    if tests:
489      test_results, test_exit_code = test_dispatcher.RunTests(
490          tests, runner_factory, False,
491          options.test_device,
492          shard=True,
493          build_type=options.build_type,
494          test_timeout=None,
495          num_retries=options.num_retries)
496
497      results.AddTestRunResults(test_results)
498
499      # Only allow exit code escalation
500      if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
501        exit_code = test_exit_code
502
503  report_results.LogFull(
504      results=results,
505      test_type='Instrumentation',
506      test_package=os.path.basename(options.test_apk),
507      annotation=options.annotations,
508      build_type=options.build_type,
509      flakiness_server=options.flakiness_dashboard_server)
510
511  return exit_code
512
513
514def _RunUIAutomatorTests(options, error_func):
515  """Subcommand of RunTestsCommands which runs uiautomator tests."""
516  uiautomator_options = ProcessUIAutomatorOptions(options, error_func)
517
518  runner_factory, tests = uiautomator_setup.Setup(uiautomator_options)
519
520  results, exit_code = test_dispatcher.RunTests(
521      tests, runner_factory, False, options.test_device,
522      shard=True,
523      build_type=options.build_type,
524      test_timeout=None,
525      num_retries=options.num_retries)
526
527  report_results.LogFull(
528      results=results,
529      test_type='UIAutomator',
530      test_package=os.path.basename(options.test_jar),
531      annotation=options.annotations,
532      build_type=options.build_type,
533      flakiness_server=options.flakiness_dashboard_server)
534
535  return exit_code
536
537
538def _RunMonkeyTests(options, error_func):
539  """Subcommand of RunTestsCommands which runs monkey tests."""
540  monkey_options = ProcessMonkeyTestOptions(options, error_func)
541
542  runner_factory, tests = monkey_setup.Setup(monkey_options)
543
544  results, exit_code = test_dispatcher.RunTests(
545      tests, runner_factory, False, None, shard=False, test_timeout=None)
546
547  report_results.LogFull(
548      results=results,
549      test_type='Monkey',
550      test_package='Monkey',
551      build_type=options.build_type)
552
553  return exit_code
554
555
556
557def RunTestsCommand(command, options, args, option_parser):
558  """Checks test type and dispatches to the appropriate function.
559
560  Args:
561    command: String indicating the command that was received to trigger
562        this function.
563    options: optparse options dictionary.
564    args: List of extra args from optparse.
565    option_parser: optparse.OptionParser object.
566
567  Returns:
568    Integer indicated exit code.
569
570  Raises:
571    Exception: Unknown command name passed in, or an exception from an
572        individual test runner.
573  """
574
575  # Check for extra arguments
576  if len(args) > 2:
577    option_parser.error('Unrecognized arguments: %s' % (' '.join(args[2:])))
578    return constants.ERROR_EXIT_CODE
579
580  ProcessCommonOptions(options)
581
582  if command == 'gtest':
583    return _RunGTests(options, option_parser.error)
584  elif command == 'instrumentation':
585    return _RunInstrumentationTests(options, option_parser.error)
586  elif command == 'uiautomator':
587    return _RunUIAutomatorTests(options, option_parser.error)
588  elif command == 'monkey':
589    return _RunMonkeyTests(options, option_parser.error)
590  else:
591    raise Exception('Unknown test type.')
592
593
594def HelpCommand(command, options, args, option_parser):
595  """Display help for a certain command, or overall help.
596
597  Args:
598    command: String indicating the command that was received to trigger
599        this function.
600    options: optparse options dictionary.
601    args: List of extra args from optparse.
602    option_parser: optparse.OptionParser object.
603
604  Returns:
605    Integer indicated exit code.
606  """
607  # If we don't have any args, display overall help
608  if len(args) < 3:
609    option_parser.print_help()
610    return 0
611  # If we have too many args, print an error
612  if len(args) > 3:
613    option_parser.error('Unrecognized arguments: %s' % (' '.join(args[3:])))
614    return constants.ERROR_EXIT_CODE
615
616  command = args[2]
617
618  if command not in VALID_COMMANDS:
619    option_parser.error('Unrecognized command.')
620
621  # Treat the help command as a special case. We don't care about showing a
622  # specific help page for itself.
623  if command == 'help':
624    option_parser.print_help()
625    return 0
626
627  VALID_COMMANDS[command].add_options_func(option_parser)
628  option_parser.usage = '%prog ' + command + ' [options]'
629  option_parser.command_list = None
630  option_parser.print_help()
631
632  return 0
633
634
635# Define a named tuple for the values in the VALID_COMMANDS dictionary so the
636# syntax is a bit prettier. The tuple is two functions: (add options, run
637# command).
638CommandFunctionTuple = collections.namedtuple(
639    'CommandFunctionTuple', ['add_options_func', 'run_command_func'])
640VALID_COMMANDS = {
641    'gtest': CommandFunctionTuple(AddGTestOptions, RunTestsCommand),
642    'instrumentation': CommandFunctionTuple(
643        AddInstrumentationTestOptions, RunTestsCommand),
644    'uiautomator': CommandFunctionTuple(
645        AddUIAutomatorTestOptions, RunTestsCommand),
646    'monkey': CommandFunctionTuple(
647        AddMonkeyTestOptions, RunTestsCommand),
648    'help': CommandFunctionTuple(lambda option_parser: None, HelpCommand)
649    }
650
651
652class CommandOptionParser(optparse.OptionParser):
653  """Wrapper class for OptionParser to help with listing commands."""
654
655  def __init__(self, *args, **kwargs):
656    self.command_list = kwargs.pop('command_list', [])
657    self.example = kwargs.pop('example', '')
658    optparse.OptionParser.__init__(self, *args, **kwargs)
659
660  #override
661  def get_usage(self):
662    normal_usage = optparse.OptionParser.get_usage(self)
663    command_list = self.get_command_list()
664    example = self.get_example()
665    return self.expand_prog_name(normal_usage + example + command_list)
666
667  #override
668  def get_command_list(self):
669    if self.command_list:
670      return '\nCommands:\n  %s\n' % '\n  '.join(sorted(self.command_list))
671    return ''
672
673  def get_example(self):
674    if self.example:
675      return '\nExample:\n  %s\n' % self.example
676    return ''
677
678
679def main(argv):
680  option_parser = CommandOptionParser(
681      usage='Usage: %prog <command> [options]',
682      command_list=VALID_COMMANDS.keys())
683
684  if len(argv) < 2 or argv[1] not in VALID_COMMANDS:
685    option_parser.error('Invalid command.')
686  command = argv[1]
687  VALID_COMMANDS[command].add_options_func(option_parser)
688  options, args = option_parser.parse_args(argv)
689  return VALID_COMMANDS[command].run_command_func(
690      command, options, args, option_parser)
691
692
693if __name__ == '__main__':
694  sys.exit(main(sys.argv))
695