1# Copyright 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import fnmatch 6import logging 7import os 8import sys 9 10from telemetry.core import util 11from telemetry.core import platform as platform_module 12from telemetry import decorators 13from telemetry.internal.browser import browser_finder 14from telemetry.internal.browser import browser_finder_exceptions 15from telemetry.internal.browser import browser_options 16from telemetry.internal.platform import android_device 17from telemetry.internal.util import binary_manager 18from telemetry.internal.util import command_line 19from telemetry.internal.util import ps_util 20from telemetry.testing import browser_test_case 21from telemetry.testing import options_for_unittests 22 23from py_utils import cloud_storage 24from py_utils import xvfb 25 26import typ 27 28 29class RunTestsCommand(command_line.OptparseCommand): 30 """Run unit tests""" 31 32 usage = '[test_name ...] [<options>]' 33 xvfb_process = None 34 35 def __init__(self): 36 super(RunTestsCommand, self).__init__() 37 self.stream = sys.stdout 38 39 @classmethod 40 def CreateParser(cls): 41 options = browser_options.BrowserFinderOptions() 42 options.browser_type = 'any' 43 parser = options.CreateParser('%%prog %s' % cls.usage) 44 return parser 45 46 @classmethod 47 def AddCommandLineArgs(cls, parser, _): 48 parser.add_option('--start-xvfb', action='store_true', 49 default=False, help='Start Xvfb display if needed.') 50 parser.add_option('--disable-cloud-storage-io', action='store_true', 51 default=False, help=('Disable cloud storage IO when ' 52 'tests are run in parallel.')) 53 parser.add_option('--repeat-count', type='int', default=1, 54 help='Repeats each a provided number of times.') 55 parser.add_option('--no-browser', action='store_true', default=False, 56 help='Don\'t require an actual browser to run the tests.') 57 parser.add_option('-d', '--also-run-disabled-tests', 58 dest='run_disabled_tests', 59 action='store_true', default=False, 60 help='Ignore @Disabled and @Enabled restrictions.') 61 parser.add_option('--exact-test-filter', action='store_true', default=False, 62 help='Treat test filter as exact matches (default is ' 63 'substring matches).') 64 parser.add_option('--client-config', dest='client_configs', 65 action='append', default=[]) 66 parser.add_option('--disable-logging-config', action='store_true', 67 default=False, help='Configure logging (default on)') 68 parser.add_option('--skip', metavar='glob', default=[], 69 action='append', help=( 70 'Globs of test names to skip (defaults to ' 71 '%(default)s).')) 72 typ.ArgumentParser.add_option_group(parser, 73 "Options for running the tests", 74 running=True, 75 skip=['-d', '-v', '--verbose']) 76 typ.ArgumentParser.add_option_group(parser, 77 "Options for reporting the results", 78 reporting=True) 79 80 @classmethod 81 def ProcessCommandLineArgs(cls, parser, args, _): 82 # We retry failures by default unless we're running a list of tests 83 # explicitly. 84 if not args.retry_limit and not args.positional_args: 85 args.retry_limit = 3 86 87 if args.no_browser: 88 return 89 90 if args.start_xvfb and xvfb.ShouldStartXvfb(): 91 cls.xvfb_process = xvfb.StartXvfb() 92 # Work around Mesa issues on Linux. See 93 # https://github.com/catapult-project/catapult/issues/3074 94 args.browser_options.AppendExtraBrowserArgs('--disable-gpu') 95 96 try: 97 possible_browser = browser_finder.FindBrowser(args) 98 except browser_finder_exceptions.BrowserFinderException, ex: 99 parser.error(ex) 100 101 if not possible_browser: 102 parser.error('No browser found of type %s. Cannot run tests.\n' 103 'Re-run with --browser=list to see ' 104 'available browser types.' % args.browser_type) 105 106 @classmethod 107 def main(cls, args=None, stream=None): # pylint: disable=arguments-differ 108 # We override the superclass so that we can hook in the 'stream' arg. 109 parser = cls.CreateParser() 110 cls.AddCommandLineArgs(parser, None) 111 options, positional_args = parser.parse_args(args) 112 options.positional_args = positional_args 113 114 try: 115 # Must initialize the DependencyManager before calling 116 # browser_finder.FindBrowser(args) 117 binary_manager.InitDependencyManager(options.client_configs) 118 cls.ProcessCommandLineArgs(parser, options, None) 119 120 obj = cls() 121 if stream is not None: 122 obj.stream = stream 123 return obj.Run(options) 124 finally: 125 if cls.xvfb_process: 126 cls.xvfb_process.kill() 127 128 def Run(self, args): 129 runner = typ.Runner() 130 if self.stream: 131 runner.host.stdout = self.stream 132 133 if args.no_browser: 134 possible_browser = None 135 platform = platform_module.GetHostPlatform() 136 else: 137 possible_browser = browser_finder.FindBrowser(args) 138 platform = possible_browser.platform 139 140 fetch_reference_chrome_binary = False 141 # Fetch all binaries needed by telemetry before we run the benchmark. 142 if possible_browser and possible_browser.browser_type == 'reference': 143 fetch_reference_chrome_binary = True 144 binary_manager.FetchBinaryDependencies( 145 platform, args.client_configs, fetch_reference_chrome_binary) 146 147 # Telemetry seems to overload the system if we run one test per core, 148 # so we scale things back a fair amount. Many of the telemetry tests 149 # are long-running, so there's a limit to how much parallelism we 150 # can effectively use for now anyway. 151 # 152 # It should be possible to handle multiple devices if we adjust the 153 # browser_finder code properly, but for now we only handle one on ChromeOS. 154 if platform.GetOSName() == 'chromeos': 155 runner.args.jobs = 1 156 elif platform.GetOSName() == 'android': 157 android_devs = android_device.FindAllAvailableDevices(args) 158 runner.args.jobs = len(android_devs) 159 if runner.args.jobs == 0: 160 raise RuntimeError("No Android device found") 161 print 'Running tests with %d Android device(s).' % runner.args.jobs 162 elif platform.GetOSVersionName() == 'xp': 163 # For an undiagnosed reason, XP falls over with more parallelism. 164 # See crbug.com/388256 165 runner.args.jobs = max(int(args.jobs) // 4, 1) 166 else: 167 runner.args.jobs = max(int(args.jobs) // 2, 1) 168 169 runner.args.skip = args.skip 170 runner.args.metadata = args.metadata 171 runner.args.passthrough = args.passthrough 172 runner.args.path = args.path 173 runner.args.retry_limit = args.retry_limit 174 runner.args.test_results_server = args.test_results_server 175 runner.args.test_type = args.test_type 176 runner.args.top_level_dir = args.top_level_dir 177 runner.args.write_full_results_to = args.write_full_results_to 178 runner.args.write_trace_to = args.write_trace_to 179 runner.args.list_only = args.list_only 180 runner.args.shard_index = args.shard_index 181 runner.args.total_shards = args.total_shards 182 183 runner.args.path.append(util.GetUnittestDataDir()) 184 185 # Always print out these info for the ease of debugging. 186 runner.args.timing = True 187 runner.args.verbose = 3 188 189 runner.classifier = GetClassifier(args, possible_browser) 190 runner.context = args 191 runner.setup_fn = _SetUpProcess 192 runner.teardown_fn = _TearDownProcess 193 runner.win_multiprocessing = typ.WinMultiprocessing.importable 194 try: 195 ret, _, _ = runner.run() 196 except KeyboardInterrupt: 197 print >> sys.stderr, "interrupted, exiting" 198 ret = 130 199 return ret 200 201 202def _SkipMatch(name, skipGlobs): 203 return any(fnmatch.fnmatch(name, glob) for glob in skipGlobs) 204 205 206def GetClassifier(args, possible_browser): 207 208 def ClassifyTestWithoutBrowser(test_set, test): 209 name = test.id() 210 if _SkipMatch(name, args.skip): 211 test_set.tests_to_skip.append( 212 typ.TestInput(name, 'skipped because matched --skip')) 213 return 214 if (not args.positional_args 215 or _MatchesSelectedTest(name, args.positional_args, 216 args.exact_test_filter)): 217 # TODO(telemetry-team): Make sure that all telemetry unittest that invokes 218 # actual browser are subclasses of browser_test_case.BrowserTestCase 219 # (crbug.com/537428) 220 if issubclass(test.__class__, browser_test_case.BrowserTestCase): 221 test_set.tests_to_skip.append(typ.TestInput( 222 name, msg='Skip the test because it requires a browser.')) 223 else: 224 test_set.parallel_tests.append(typ.TestInput(name)) 225 226 def ClassifyTestWithBrowser(test_set, test): 227 name = test.id() 228 if _SkipMatch(name, args.skip): 229 test_set.tests_to_skip.append( 230 typ.TestInput(name, 'skipped because matched --skip')) 231 return 232 if (not args.positional_args 233 or _MatchesSelectedTest(name, args.positional_args, 234 args.exact_test_filter)): 235 assert hasattr(test, '_testMethodName') 236 method = getattr( 237 test, test._testMethodName) # pylint: disable=protected-access 238 should_skip, reason = decorators.ShouldSkip(method, possible_browser) 239 if should_skip and not args.run_disabled_tests: 240 test_set.tests_to_skip.append(typ.TestInput(name, msg=reason)) 241 elif decorators.ShouldBeIsolated(method, possible_browser): 242 test_set.isolated_tests.append(typ.TestInput(name)) 243 else: 244 test_set.parallel_tests.append(typ.TestInput(name)) 245 246 if possible_browser: 247 return ClassifyTestWithBrowser 248 else: 249 return ClassifyTestWithoutBrowser 250 251 252def _MatchesSelectedTest(name, selected_tests, selected_tests_are_exact): 253 if not selected_tests: 254 return False 255 if selected_tests_are_exact: 256 return any(name in selected_tests) 257 else: 258 return any(test in name for test in selected_tests) 259 260 261def _SetUpProcess(child, context): # pylint: disable=unused-argument 262 ps_util.EnableListingStrayProcessesUponExitHook() 263 # Make sure that we don't invokes cloud storage I/Os when we run the tests in 264 # parallel. 265 # TODO(nednguyen): always do this once telemetry tests in Chromium is updated 266 # to prefetch files. 267 # (https://github.com/catapult-project/catapult/issues/2192) 268 args = context 269 if args.disable_cloud_storage_io: 270 os.environ[cloud_storage.DISABLE_CLOUD_STORAGE_IO] = '1' 271 if binary_manager.NeedsInit(): 272 # Typ doesn't keep the DependencyManager initialization in the child 273 # processes. 274 binary_manager.InitDependencyManager(context.client_configs) 275 # We need to reset the handlers in case some other parts of telemetry already 276 # set it to make this work. 277 if not args.disable_logging_config: 278 logging.getLogger().handlers = [] 279 logging.basicConfig( 280 level=logging.INFO, 281 format='(%(levelname)s) %(asctime)s pid=%(process)d' 282 ' %(module)s.%(funcName)s:%(lineno)d' 283 ' %(message)s') 284 if args.remote_platform_options.device == 'android': 285 android_devices = android_device.FindAllAvailableDevices(args) 286 if not android_devices: 287 raise RuntimeError("No Android device found") 288 android_devices.sort(key=lambda device: device.name) 289 args.remote_platform_options.device = ( 290 android_devices[child.worker_num-1].guid) 291 options_for_unittests.Push(args) 292 293 294def _TearDownProcess(child, context): # pylint: disable=unused-argument 295 # It's safe to call teardown_browser even if we did not start any browser 296 # in any of the tests. 297 browser_test_case.teardown_browser() 298 options_for_unittests.Pop() 299 300 301if __name__ == '__main__': 302 ret_code = RunTestsCommand.main() 303 sys.exit(ret_code) 304