1eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#!/usr/bin/env python 2eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch# 3eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch# Copyright 2013 The Chromium Authors. All rights reserved. 4eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch# Use of this source code is governed by a BSD-style license that can be 5eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch# found in the LICENSE file. 6eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)"""Runs all types of tests from one unified interface.""" 8eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 9eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochimport collections 103551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)import logging 11eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochimport optparse 12eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochimport os 13ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochimport shutil 14f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)import signal 15eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochimport sys 16f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)import threading 17f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)import traceback 18eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 193551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)from pylib import android_commands 20eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochfrom pylib import constants 21f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)from pylib import forwarder 22eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochfrom pylib import ports 23eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochfrom pylib.base import base_test_result 24ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochfrom pylib.base import test_dispatcher 25ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochfrom pylib.gtest import gtest_config 26a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles)from pylib.gtest import setup as gtest_setup 27a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles)from pylib.gtest import test_options as gtest_test_options 284e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)from pylib.linker import setup as linker_setup 293240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdochfrom pylib.host_driven import setup as host_driven_setup 30ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochfrom pylib.instrumentation import setup as instrumentation_setup 31a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles)from pylib.instrumentation import test_options as instrumentation_test_options 32bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdochfrom pylib.monkey import setup as monkey_setup 33bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdochfrom pylib.monkey import test_options as monkey_test_options 343551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)from pylib.perf import setup as perf_setup 353551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)from pylib.perf import test_options as perf_test_options 363551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)from pylib.perf import test_runner as perf_test_runner 37ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochfrom pylib.uiautomator import setup as uiautomator_setup 38a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles)from pylib.uiautomator import test_options as uiautomator_test_options 393551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)from pylib.utils import command_option_parser 40ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochfrom pylib.utils import report_results 41f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)from pylib.utils import reraiser_thread 42ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochfrom pylib.utils import run_tests_helper 43eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 44eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 45eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef AddCommonOptions(option_parser): 46eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch """Adds all common options to |option_parser|.""" 47eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 48d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) group = optparse.OptionGroup(option_parser, 'Common Options') 49d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) default_build_type = os.environ.get('BUILDTYPE', 'Debug') 50d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) group.add_option('--debug', action='store_const', const='Debug', 51d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) dest='build_type', default=default_build_type, 52d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) help=('If set, run test suites under out/Debug. ' 53d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 'Default is env var BUILDTYPE or Debug.')) 54d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) group.add_option('--release', action='store_const', 55d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) const='Release', dest='build_type', 56d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) help=('If set, run test suites under out/Release.' 57d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) ' Default is env var BUILDTYPE or Debug.')) 58d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) group.add_option('-c', dest='cleanup_test_files', 59d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) help='Cleanup test files on the device after run', 60d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) action='store_true') 61d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) group.add_option('--num_retries', dest='num_retries', type='int', 62d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) default=2, 63d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) help=('Number of retries for a test before ' 64d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 'giving up.')) 65d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) group.add_option('-v', 66d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) '--verbose', 67d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) dest='verbose_count', 68d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) default=0, 69d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) action='count', 70d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) help='Verbose level (multiple times for more)') 71d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) group.add_option('--tool', 72d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) dest='tool', 73d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) help=('Run the test under a tool ' 74d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) '(use --tool help to list them)')) 75d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) group.add_option('--flakiness-dashboard-server', 76d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) dest='flakiness_dashboard_server', 77d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) help=('Address of the server that is hosting the ' 78d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 'Chrome for Android flakiness dashboard.')) 79d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) group.add_option('--skip-deps-push', dest='push_deps', 80d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) action='store_false', default=True, 81d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) help=('Do not push dependencies to the device. ' 82d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 'Use this at own risk for speeding up test ' 83d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 'execution on local machine.')) 84d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) group.add_option('-d', '--device', dest='test_device', 85d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) help=('Target device for the test suite ' 86d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 'to run on.')) 87d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) option_parser.add_option_group(group) 88eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 89eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 90eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef ProcessCommonOptions(options): 91eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch """Processes and handles all common options.""" 92eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch run_tests_helper.SetLogLevel(options.verbose_count) 933551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) constants.SetBuildType(options.build_type) 94eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 95eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 967dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochdef AddGTestOptions(option_parser): 977dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """Adds gtest options to |option_parser|.""" 987dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 997dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch option_parser.usage = '%prog gtest [options]' 100d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) option_parser.commands_dict = {} 1017dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch option_parser.example = '%prog gtest -s base_unittests' 1027dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 103ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # TODO(gkanwar): Make this option required 104ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch option_parser.add_option('-s', '--suite', dest='suite_name', 1057dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch help=('Executable name of the test suite to run ' 1067dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch '(use -s help to list them).')) 107f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) option_parser.add_option('-f', '--gtest_filter', '--gtest-filter', 108f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) dest='test_filter', 109fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch help='googletest-style filter string.') 110f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) option_parser.add_option('--gtest_also_run_disabled_tests', 111f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) '--gtest-also-run-disabled-tests', 112d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) dest='run_disabled', action='store_true', 113d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) help='Also run disabled tests if applicable.') 114d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) option_parser.add_option('-a', '--test-arguments', dest='test_arguments', 115d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) default='', 116fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch help='Additional arguments to pass to the test.') 117fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch option_parser.add_option('-t', dest='timeout', 118fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch help='Timeout to wait for each test', 119fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch type='int', 120fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch default=60) 121eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch # TODO(gkanwar): Move these to Common Options once we have the plumbing 122eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch # in our other test types to handle these commands 123eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch AddCommonOptions(option_parser) 124eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 125eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 1264e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)def AddLinkerTestOptions(option_parser): 1274e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) option_parser.usage = '%prog linker' 1284e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) option_parser.commands_dict = {} 1294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) option_parser.example = '%prog linker' 1304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) option_parser.add_option('-f', '--gtest-filter', dest='test_filter', 1324e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) help='googletest-style filter string.') 1334e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) AddCommonOptions(option_parser) 1344e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1354e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 136ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef ProcessGTestOptions(options): 137ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch """Intercept test suite help to list test suites. 138ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 139ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch Args: 140ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch options: Command line options. 141ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch """ 142ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if options.suite_name == 'help': 143ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch print 'Available test suites are:' 144fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch for test_suite in (gtest_config.STABLE_TEST_SUITES + 145fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch gtest_config.EXPERIMENTAL_TEST_SUITES): 146fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch print test_suite 147a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) sys.exit(0) 148ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 149ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # Convert to a list, assuming all test suites if nothing was specified. 150ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # TODO(gkanwar): Require having a test suite 151ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if options.suite_name: 152ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch options.suite_name = [options.suite_name] 153ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch else: 154fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch options.suite_name = [s for s in gtest_config.STABLE_TEST_SUITES] 155ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 156ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 157eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef AddJavaTestOptions(option_parser): 158eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch """Adds the Java test options to |option_parser|.""" 159eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 160d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) option_parser.add_option('-f', '--test-filter', dest='test_filter', 161eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch help=('Test filter (if not fully qualified, ' 162eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'will run all matches).')) 163eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.add_option( 164eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch '-A', '--annotation', dest='annotation_str', 165eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch help=('Comma-separated list of annotations. Run only tests with any of ' 166eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'the given annotations. An annotation can be either a key or a ' 167eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'key-values pair. A test that has no annotation is considered ' 168eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch '"SmallTest".')) 169eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.add_option( 170eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch '-E', '--exclude-annotation', dest='exclude_annotation_str', 171eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch help=('Comma-separated list of annotations. Exclude tests with these ' 172eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'annotations.')) 173eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.add_option('--screenshot', dest='screenshot_failures', 174eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch action='store_true', 175eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch help='Capture screenshots of test failures') 176eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.add_option('--save-perf-json', action='store_true', 177eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch help='Saves the JSON file for each UI Perf test.') 1783240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch option_parser.add_option('--official-build', action='store_true', 1793240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch help='Run official build tests.') 180eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.add_option('--test_data', action='append', default=[], 181eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch help=('Each instance defines a directory of test ' 182eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'data that should be copied to the target(s) ' 183eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'before running the tests. The argument ' 184eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'should be of the form <target>:<source>, ' 185eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch '<target> is relative to the device data' 186eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'directory, and <source> is relative to the ' 187eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'chromium build directory.')) 188eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 189eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 190eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef ProcessJavaTestOptions(options, error_func): 191eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch """Processes options/arguments and populates |options| with defaults.""" 192eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 193eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if options.annotation_str: 194eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch options.annotations = options.annotation_str.split(',') 195eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch elif options.test_filter: 196eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch options.annotations = [] 197eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch else: 198ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch options.annotations = ['Smoke', 'SmallTest', 'MediumTest', 'LargeTest', 199ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 'EnormousTest'] 200eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 201eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if options.exclude_annotation_str: 202eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch options.exclude_annotations = options.exclude_annotation_str.split(',') 203eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch else: 204eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch options.exclude_annotations = [] 205eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 206eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 207eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef AddInstrumentationTestOptions(option_parser): 208eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch """Adds Instrumentation test options to |option_parser|.""" 209eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 210eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.usage = '%prog instrumentation [options]' 211d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) option_parser.commands_dict = {} 212558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch option_parser.example = ('%prog instrumentation ' 213eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch '--test-apk=ChromiumTestShellTest') 214eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 215eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch AddJavaTestOptions(option_parser) 216eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch AddCommonOptions(option_parser) 217eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 218d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) option_parser.add_option('-j', '--java-only', action='store_true', 2193240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch default=False, help='Run only the Java tests.') 220d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) option_parser.add_option('-p', '--python-only', action='store_true', 2213240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch default=False, 2223240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch help='Run only the host-driven tests.') 2233551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) option_parser.add_option('--host-driven-root', 2243240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch help='Root of the host-driven tests.') 225eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.add_option('-w', '--wait_debugger', dest='wait_for_debugger', 226eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch action='store_true', 227eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch help='Wait for debugger.') 228eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.add_option( 229eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch '--test-apk', dest='test_apk', 230eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch help=('The name of the apk containing the tests ' 23168043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) '(without the .apk extension; e.g. "ContentShellTest").')) 2323551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) option_parser.add_option('--coverage-dir', 2333551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) help=('Directory in which to place all generated ' 2343551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 'EMMA coverage files.')) 235eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 236eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 237eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef ProcessInstrumentationOptions(options, error_func): 238a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) """Processes options/arguments and populate |options| with defaults. 239a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 240a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) Args: 241a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options: optparse.Options object. 242a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) error_func: Function to call with the error message in case of an error. 243a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 244a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) Returns: 245a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) An InstrumentationOptions named tuple which contains all options relevant to 246a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) instrumentation tests. 247a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) """ 248eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 249eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch ProcessJavaTestOptions(options, error_func) 250eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 2513240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch if options.java_only and options.python_only: 2523240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch error_func('Options java_only (-j) and python_only (-p) ' 2533240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 'are mutually exclusive.') 2543240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch options.run_java_tests = True 2553240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch options.run_python_tests = True 2563240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch if options.java_only: 2573240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch options.run_python_tests = False 2583240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch elif options.python_only: 2593240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch options.run_java_tests = False 2603240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 2613551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if not options.host_driven_root: 2623240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch options.run_python_tests = False 2633240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 264eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if not options.test_apk: 265eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch error_func('--test-apk must be specified.') 266eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 26768043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) 26868043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) options.test_apk_path = os.path.join(constants.GetOutDirectory(), 26968043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) constants.SDK_BUILD_APKS_DIR, 27068043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) '%s.apk' % options.test_apk) 27168043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) options.test_apk_jar_path = os.path.join( 27268043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) constants.GetOutDirectory(), 27368043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) constants.SDK_BUILD_TEST_JAVALIB_DIR, 27468043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) '%s.jar' % options.test_apk) 275eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 276a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) return instrumentation_test_options.InstrumentationOptions( 277a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.tool, 278a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.cleanup_test_files, 279a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.push_deps, 280a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.annotations, 281a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.exclude_annotations, 282a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.test_filter, 283a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.test_data, 284a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.save_perf_json, 285a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.screenshot_failures, 286a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.wait_for_debugger, 2873551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) options.coverage_dir, 288a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.test_apk, 289a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.test_apk_path, 290a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.test_apk_jar_path) 291a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 292eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 293eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef AddUIAutomatorTestOptions(option_parser): 294eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch """Adds UI Automator test options to |option_parser|.""" 295eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 296eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.usage = '%prog uiautomator [options]' 297d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) option_parser.commands_dict = {} 298eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.example = ( 299eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch '%prog uiautomator --test-jar=chromium_testshell_uiautomator_tests' 3004e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) ' --package=chromium_test_shell') 301eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.add_option( 3024e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--package', 3034e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) help=('Package under test. Possible values: %s' % 3044e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) constants.PACKAGE_INFO.keys())) 305eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.add_option( 306eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch '--test-jar', dest='test_jar', 307eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch help=('The name of the dexed jar containing the tests (without the ' 308eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch '.dex.jar extension). Alternatively, this can be a full path ' 309eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'to the jar.')) 310eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 311eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch AddJavaTestOptions(option_parser) 312eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch AddCommonOptions(option_parser) 313eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 314eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 315eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef ProcessUIAutomatorOptions(options, error_func): 316a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) """Processes UIAutomator options/arguments. 317a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 318a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) Args: 319a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options: optparse.Options object. 320a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) error_func: Function to call with the error message in case of an error. 321a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 322a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) Returns: 323a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) A UIAutomatorOptions named tuple which contains all options relevant to 324bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch uiautomator tests. 325a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) """ 326eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 327eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch ProcessJavaTestOptions(options, error_func) 328eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 3294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if not options.package: 3304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) error_func('--package is required.') 3314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 3324e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if options.package not in constants.PACKAGE_INFO: 3334e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) error_func('Invalid package.') 334eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 335eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if not options.test_jar: 336eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch error_func('--test-jar must be specified.') 337eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 338eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if os.path.exists(options.test_jar): 339eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch # The dexed JAR is fully qualified, assume the info JAR lives along side. 340eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch options.uiautomator_jar = options.test_jar 341eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch else: 342eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch options.uiautomator_jar = os.path.join( 34368043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) constants.GetOutDirectory(), 34468043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) constants.SDK_BUILD_JAVALIB_DIR, 345eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch '%s.dex.jar' % options.test_jar) 346eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch options.uiautomator_info_jar = ( 347eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch options.uiautomator_jar[:options.uiautomator_jar.find('.dex.jar')] + 348eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch '_java.jar') 349eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 350a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) return uiautomator_test_options.UIAutomatorOptions( 351a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.tool, 352a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.cleanup_test_files, 353a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.push_deps, 354a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.annotations, 355a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.exclude_annotations, 356a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.test_filter, 357a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.test_data, 358a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.save_perf_json, 359a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.screenshot_failures, 360a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.uiautomator_jar, 361a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.uiautomator_info_jar, 3624e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) options.package) 363a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 364eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 365bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdochdef AddMonkeyTestOptions(option_parser): 366bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch """Adds monkey test options to |option_parser|.""" 367ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch 368ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch option_parser.usage = '%prog monkey [options]' 369d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) option_parser.commands_dict = {} 370ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch option_parser.example = ( 3714e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '%prog monkey --package=chromium_test_shell') 372ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch 373bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch option_parser.add_option( 3744e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--package', 3754e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) help=('Package under test. Possible values: %s' % 3764e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) constants.PACKAGE_INFO.keys())) 377bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch option_parser.add_option( 378bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch '--event-count', default=10000, type='int', 379bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch help='Number of events to generate [default: %default].') 380bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch option_parser.add_option( 381bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch '--category', default='', 382ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch help='A list of allowed categories.') 383bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch option_parser.add_option( 384bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch '--throttle', default=100, type='int', 385bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch help='Delay between events (ms) [default: %default]. ') 386bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch option_parser.add_option( 387bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch '--seed', type='int', 388bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch help=('Seed value for pseudo-random generator. Same seed value generates ' 389bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 'the same sequence of events. Seed is randomized by default.')) 390bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch option_parser.add_option( 391bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch '--extra-args', default='', 392bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch help=('String of other args to pass to the command verbatim ' 393bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch '[default: "%default"].')) 394bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 395bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch AddCommonOptions(option_parser) 396bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 397bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 398bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdochdef ProcessMonkeyTestOptions(options, error_func): 399bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch """Processes all monkey test options. 400bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 401bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch Args: 402bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch options: optparse.Options object. 403bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch error_func: Function to call with the error message in case of an error. 404bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 405bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch Returns: 406bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch A MonkeyOptions named tuple which contains all options relevant to 407bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch monkey tests. 408bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch """ 4094e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if not options.package: 4104e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) error_func('--package is required.') 4114e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 4124e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if options.package not in constants.PACKAGE_INFO: 4134e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) error_func('Invalid package.') 414bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 415bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch category = options.category 416bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch if category: 417bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch category = options.category.split(',') 418bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 419bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch return monkey_test_options.MonkeyOptions( 420bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch options.verbose_count, 4214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) options.package, 422bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch options.event_count, 423bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch category, 424bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch options.throttle, 425bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch options.seed, 426bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch options.extra_args) 427bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 428bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 4293551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)def AddPerfTestOptions(option_parser): 4303551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) """Adds perf test options to |option_parser|.""" 4313551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 4323551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) option_parser.usage = '%prog perf [options]' 433d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) option_parser.commands_dict = {} 434f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) option_parser.example = ('%prog perf ' 435f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) '[--single-step -- command args] or ' 436f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) '[--steps perf_steps.json] or ' 437f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) '[--print-step step]') 4383551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 43958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) option_parser.add_option( 440f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) '--single-step', 441f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) action='store_true', 442f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) help='Execute the given command with retries, but only print the result ' 443f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 'for the "most successful" round.') 444f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) option_parser.add_option( 44558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) '--steps', 446f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) help='JSON file containing the list of commands to run.') 44758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) option_parser.add_option( 44858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) '--flaky-steps', 44958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) help=('A JSON file containing steps that are flaky ' 45058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 'and will have its exit code ignored.')) 45158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) option_parser.add_option( 45258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) '--print-step', 45358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) help='The name of a previously executed perf step to print.') 45458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) option_parser.add_option( 45558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) '--no-timeout', action='store_true', 45658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) help=('Do not impose a timeout. Each perf step is responsible for ' 45758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 'implementing the timeout logic.')) 4584e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) option_parser.add_option( 4594e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '-f', '--test-filter', 4604e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) help=('Test filter (will match against the names listed in --steps).')) 4614e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) option_parser.add_option( 4624e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--dry-run', 4634e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) action='store_true', 4644e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) help='Just print the steps without executing.') 4653551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) AddCommonOptions(option_parser) 4663551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 4673551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 468f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)def ProcessPerfTestOptions(options, args, error_func): 4693551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) """Processes all perf test options. 4703551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 4713551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) Args: 4723551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) options: optparse.Options object. 4733551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) error_func: Function to call with the error message in case of an error. 4743551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 4753551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) Returns: 4763551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) A PerfOptions named tuple which contains all options relevant to 4773551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) perf tests. 4783551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) """ 479f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) # Only one of steps, print_step or single_step must be provided. 480f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) count = len(filter(None, 481f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) [options.steps, options.print_step, options.single_step])) 482f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if count != 1: 483f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) error_func('Please specify one of: --steps, --print-step, --single-step.') 484f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) single_step = None 485f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if options.single_step: 486f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) single_step = ' '.join(args[2:]) 4873551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return perf_test_options.PerfOptions( 48858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) options.steps, options.flaky_steps, options.print_step, 489f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) options.no_timeout, options.test_filter, options.dry_run, 490f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) single_step) 4913551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 4923551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 4933551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)def _RunGTests(options, error_func, devices): 494ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch """Subcommand of RunTestsCommands which runs gtests.""" 495a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) ProcessGTestOptions(options) 496ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 497ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch exit_code = 0 498ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch for suite_name in options.suite_name: 499a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) # TODO(gkanwar): Move this into ProcessGTestOptions once we require -s for 500a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) # the gtest command. 501a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) gtest_options = gtest_test_options.GTestOptions( 502a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.tool, 503a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.cleanup_test_files, 504a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.push_deps, 505a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.test_filter, 506d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) options.run_disabled, 507a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.test_arguments, 508a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.timeout, 509a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) suite_name) 5103551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) runner_factory, tests = gtest_setup.Setup(gtest_options, devices) 511ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 512ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch results, test_exit_code = test_dispatcher.RunTests( 5133551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) tests, runner_factory, devices, shard=True, test_timeout=None, 514ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch num_retries=options.num_retries) 515ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 516ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: 517ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch exit_code = test_exit_code 518ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 519ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch report_results.LogFull( 520ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch results=results, 521ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch test_type='Unit test', 522ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch test_package=suite_name, 523ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch flakiness_server=options.flakiness_dashboard_server) 524ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 525ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if os.path.isdir(constants.ISOLATE_DEPS_DIR): 526ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch shutil.rmtree(constants.ISOLATE_DEPS_DIR) 527ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 528ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return exit_code 529ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 530ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 5314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)def _RunLinkerTests(options, error_func, devices): 5324e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) """Subcommand of RunTestsCommands which runs linker tests.""" 5334e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) runner_factory, tests = linker_setup.Setup(options, devices) 5344e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 5354e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) results, exit_code = test_dispatcher.RunTests( 5364e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) tests, runner_factory, devices, shard=True, test_timeout=60, 5374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) num_retries=options.num_retries) 5384e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 5394e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) report_results.LogFull( 5404e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) results=results, 5414e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_type='Linker test', 5424e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_package='ContentLinkerTest') 5434e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 5444e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return exit_code 5454e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 5464e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 5473551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)def _RunInstrumentationTests(options, error_func, devices): 548ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch """Subcommand of RunTestsCommands which runs instrumentation tests.""" 549a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) instrumentation_options = ProcessInstrumentationOptions(options, error_func) 550ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 5513551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if len(devices) > 1 and options.wait_for_debugger: 5523551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) logging.warning('Debugger can not be sharded, using first available device') 5533551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) devices = devices[:1] 5543551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 555ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch results = base_test_result.TestRunResults() 556ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch exit_code = 0 557ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 558ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if options.run_java_tests: 559a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) runner_factory, tests = instrumentation_setup.Setup(instrumentation_options) 560ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 561ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch test_results, exit_code = test_dispatcher.RunTests( 5623551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) tests, runner_factory, devices, shard=True, test_timeout=None, 563ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch num_retries=options.num_retries) 564ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 565ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch results.AddTestRunResults(test_results) 566ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 567ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if options.run_python_tests: 5683240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch runner_factory, tests = host_driven_setup.InstrumentationSetup( 5693551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) options.host_driven_root, options.official_build, 5703240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch instrumentation_options) 571ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 5723240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch if tests: 5733240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch test_results, test_exit_code = test_dispatcher.RunTests( 5743551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) tests, runner_factory, devices, shard=True, test_timeout=None, 5753240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch num_retries=options.num_retries) 576ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 5773240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch results.AddTestRunResults(test_results) 5783240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 5793240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch # Only allow exit code escalation 5803240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: 5813240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch exit_code = test_exit_code 582ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 583ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch report_results.LogFull( 584ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch results=results, 585ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch test_type='Instrumentation', 586ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch test_package=os.path.basename(options.test_apk), 587ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch annotation=options.annotations, 588ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch flakiness_server=options.flakiness_dashboard_server) 589ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 590ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return exit_code 591ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 592ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 5933551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)def _RunUIAutomatorTests(options, error_func, devices): 594ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch """Subcommand of RunTestsCommands which runs uiautomator tests.""" 595a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) uiautomator_options = ProcessUIAutomatorOptions(options, error_func) 596ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 5973240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch runner_factory, tests = uiautomator_setup.Setup(uiautomator_options) 598ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 5993240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch results, exit_code = test_dispatcher.RunTests( 6003551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) tests, runner_factory, devices, shard=True, test_timeout=None, 6013240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch num_retries=options.num_retries) 602ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 603ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch report_results.LogFull( 604ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch results=results, 605ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch test_type='UIAutomator', 606ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch test_package=os.path.basename(options.test_jar), 607ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch annotation=options.annotations, 608ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch flakiness_server=options.flakiness_dashboard_server) 609ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 610ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return exit_code 611ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 612ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 6133551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)def _RunMonkeyTests(options, error_func, devices): 614bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch """Subcommand of RunTestsCommands which runs monkey tests.""" 615bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch monkey_options = ProcessMonkeyTestOptions(options, error_func) 616bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 617bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch runner_factory, tests = monkey_setup.Setup(monkey_options) 618bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 619bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch results, exit_code = test_dispatcher.RunTests( 62058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) tests, runner_factory, devices, shard=False, test_timeout=None, 62158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) num_retries=options.num_retries) 622bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 623bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch report_results.LogFull( 624bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch results=results, 625bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch test_type='Monkey', 6263551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) test_package='Monkey') 627bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 628bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch return exit_code 629bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 630bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 631f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)def _RunPerfTests(options, args, error_func, devices): 6323551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) """Subcommand of RunTestsCommands which runs perf tests.""" 633f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) perf_options = ProcessPerfTestOptions(options, args, error_func) 634f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) # Just print the results from a single previously executed step. 6353551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if perf_options.print_step: 6363551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return perf_test_runner.PrintTestOutput(perf_options.print_step) 6373551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 6383551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) runner_factory, tests = perf_setup.Setup(perf_options) 6393551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 6403551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) results, _ = test_dispatcher.RunTests( 64158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) tests, runner_factory, devices, shard=True, test_timeout=None, 64258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) num_retries=options.num_retries) 6433551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 6443551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) report_results.LogFull( 6453551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) results=results, 6463551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) test_type='Perf', 6473551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) test_package='Perf') 648f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 649f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if perf_options.single_step: 650f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return perf_test_runner.PrintTestOutput('single_step') 651f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 6523551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) # Always return 0 on the sharding stage. Individual tests exit_code 6533551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) # will be returned on the print_step stage. 6543551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return 0 6553551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 6563551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 6573551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)def _GetAttachedDevices(test_device=None): 6583551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) """Get all attached devices. 6593551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 6603551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) Args: 6613551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) test_device: Name of a specific device to use. 6623551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 6633551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) Returns: 6643551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) A list of attached devices. 6653551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) """ 6663551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) attached_devices = [] 6673551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 6683551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) attached_devices = android_commands.GetAttachedDevices() 6693551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if test_device: 6703551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) assert test_device in attached_devices, ( 6713551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 'Did not find device %s among attached device. Attached devices: %s' 6723551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) % (test_device, ', '.join(attached_devices))) 6733551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) attached_devices = [test_device] 6743551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 6753551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) assert attached_devices, 'No devices attached.' 6763551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 6773551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return sorted(attached_devices) 6783551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 679bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 680eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef RunTestsCommand(command, options, args, option_parser): 681eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch """Checks test type and dispatches to the appropriate function. 682eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 683eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch Args: 684eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch command: String indicating the command that was received to trigger 685eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch this function. 686eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch options: optparse options dictionary. 687eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch args: List of extra args from optparse. 688eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser: optparse.OptionParser object. 689eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 690eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch Returns: 691eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch Integer indicated exit code. 6927dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 6937dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Raises: 6947dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Exception: Unknown command name passed in, or an exception from an 6957dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch individual test runner. 696eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch """ 697eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 6987dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # Check for extra arguments 699f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if len(args) > 2 and command != 'perf': 7007dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch option_parser.error('Unrecognized arguments: %s' % (' '.join(args[2:]))) 7017dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch return constants.ERROR_EXIT_CODE 702f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if command == 'perf': 703f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if ((options.single_step and len(args) <= 2) or 704f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) (not options.single_step and len(args) > 2)): 705f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) option_parser.error('Unrecognized arguments: %s' % (' '.join(args))) 706f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return constants.ERROR_EXIT_CODE 7077dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 708eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch ProcessCommonOptions(options) 709eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 7103551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) devices = _GetAttachedDevices(options.test_device) 7113551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 712f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) forwarder.Forwarder.RemoveHostLog() 713f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if not ports.ResetTestServerPortAllocation(): 714f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) raise Exception('Failed to reset test server port.') 715f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 716eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if command == 'gtest': 7173551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return _RunGTests(options, option_parser.error, devices) 7184e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) elif command == 'linker': 7194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return _RunLinkerTests(options, option_parser.error, devices) 720eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch elif command == 'instrumentation': 7213551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return _RunInstrumentationTests(options, option_parser.error, devices) 722eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch elif command == 'uiautomator': 7233551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return _RunUIAutomatorTests(options, option_parser.error, devices) 724bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch elif command == 'monkey': 7253551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return _RunMonkeyTests(options, option_parser.error, devices) 7263551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) elif command == 'perf': 727f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return _RunPerfTests(options, args, option_parser.error, devices) 728eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch else: 729ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch raise Exception('Unknown test type.') 730eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 731eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 732eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef HelpCommand(command, options, args, option_parser): 733eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch """Display help for a certain command, or overall help. 734eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 735eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch Args: 736eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch command: String indicating the command that was received to trigger 737eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch this function. 738eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch options: optparse options dictionary. 739eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch args: List of extra args from optparse. 740eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser: optparse.OptionParser object. 741eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 742eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch Returns: 743eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch Integer indicated exit code. 744eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch """ 745eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch # If we don't have any args, display overall help 746eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if len(args) < 3: 747eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.print_help() 748eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch return 0 7497dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # If we have too many args, print an error 7507dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if len(args) > 3: 7517dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch option_parser.error('Unrecognized arguments: %s' % (' '.join(args[3:]))) 7527dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch return constants.ERROR_EXIT_CODE 753eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 754eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch command = args[2] 755eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 756eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if command not in VALID_COMMANDS: 757eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.error('Unrecognized command.') 758eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 759eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch # Treat the help command as a special case. We don't care about showing a 760eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch # specific help page for itself. 761eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if command == 'help': 762eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.print_help() 763eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch return 0 764eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 765eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch VALID_COMMANDS[command].add_options_func(option_parser) 766eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.usage = '%prog ' + command + ' [options]' 767d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) option_parser.commands_dict = {} 768eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch option_parser.print_help() 769eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 770eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch return 0 771eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 772eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 773eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch# Define a named tuple for the values in the VALID_COMMANDS dictionary so the 774eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch# syntax is a bit prettier. The tuple is two functions: (add options, run 775eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch# command). 776eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen MurdochCommandFunctionTuple = collections.namedtuple( 777eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'CommandFunctionTuple', ['add_options_func', 'run_command_func']) 778eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen MurdochVALID_COMMANDS = { 779eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'gtest': CommandFunctionTuple(AddGTestOptions, RunTestsCommand), 780eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'instrumentation': CommandFunctionTuple( 781eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch AddInstrumentationTestOptions, RunTestsCommand), 782eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'uiautomator': CommandFunctionTuple( 783eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch AddUIAutomatorTestOptions, RunTestsCommand), 784bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 'monkey': CommandFunctionTuple( 785bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch AddMonkeyTestOptions, RunTestsCommand), 7863551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 'perf': CommandFunctionTuple( 7873551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) AddPerfTestOptions, RunTestsCommand), 7884e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'linker': CommandFunctionTuple( 7894e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) AddLinkerTestOptions, RunTestsCommand), 790eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'help': CommandFunctionTuple(lambda option_parser: None, HelpCommand) 791eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch } 792eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 793eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 794f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)def DumpThreadStacks(signal, frame): 795f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) for thread in threading.enumerate(): 796f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) reraiser_thread.LogThreadStack(thread) 797f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 798f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 799eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef main(argv): 800f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) signal.signal(signal.SIGUSR1, DumpThreadStacks) 8013551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) option_parser = command_option_parser.CommandOptionParser( 8023551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) commands_dict=VALID_COMMANDS) 8033551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return command_option_parser.ParseAndExecute(option_parser) 804eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 805eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 806eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochif __name__ == '__main__': 807eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch sys.exit(main(sys.argv)) 808