1#!/usr/bin/env python 2# Copyright (c) 2013 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import optparse 7import os 8import subprocess 9import sys 10import time 11 12import build_projects 13import build_version 14import buildbot_common 15import parse_dsc 16 17from build_paths import OUT_DIR, SRC_DIR, SDK_SRC_DIR, SCRIPT_DIR 18 19sys.path.append(os.path.join(SDK_SRC_DIR, 'tools')) 20import getos 21platform = getos.GetPlatform() 22 23# TODO(binji): ugly hack -- can I get the browser in a cleaner way? 24sys.path.append(os.path.join(SRC_DIR, 'chrome', 'test', 'nacl_test_injection')) 25import find_chrome 26browser_path = find_chrome.FindChrome(SRC_DIR, ['Debug', 'Release']) 27 28 29pepper_ver = str(int(build_version.ChromeMajorVersion())) 30pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver) 31 32browser_tester_py = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'tools', 33 'browser_tester', 'browser_tester.py') 34 35 36ALL_CONFIGS = ['Debug', 'Release'] 37ALL_TOOLCHAINS = ['newlib', 'glibc', 'pnacl', 'win', 'linux', 'mac'] 38 39# Values you can filter by: 40# name: The name of the test. (e.g. "pi_generator") 41# config: See ALL_CONFIGS above. 42# toolchain: See ALL_TOOLCHAINS above. 43# platform: mac/win/linux. 44# 45# All keys must be matched, but any value that matches in a sequence is 46# considered a match for that key. For example: 47# 48# {'name': ('pi_generator', 'input_event'), 'toolchain': ('newlib', 'pnacl')} 49# 50# Will match 8 tests: 51# pi_generator.newlib_debug_test 52# pi_generator.newlib_release_test 53# input_event.newlib_debug_test 54# input_event.newlib_release_test 55# pi_generator.glibc_debug_test 56# pi_generator.glibc_release_test 57# input_event.glibc_debug_test 58# input_event.glibc_release_test 59DISABLED_TESTS = [ 60 # TODO(binji): Disable 3D examples on linux/win/mac. See 61 # http://crbug.com/262379. 62 {'name': 'graphics_3d', 'platform': ('win', 'linux', 'mac')}, 63 {'name': 'video_decode', 'platform': ('win', 'linux', 'mac')}, 64 # media_stream_audio uses audio input devices which are not supported. 65 {'name': 'media_stream_audio', 'platform': ('win', 'linux', 'mac')}, 66 # media_stream_video uses 3D and webcam which are not supported. 67 {'name': 'media_stream_video', 'platform': ('win', 'linux', 'mac')}, 68 # TODO(binji): These tests timeout on the trybots because the NEXEs take 69 # more than 40 seconds to load (!). See http://crbug.com/280753 70 {'name': 'nacl_io_test', 'platform': 'win', 'toolchain': 'glibc'}, 71 # We don't test "getting_started/part1" because it would complicate the 72 # example. 73 # TODO(binji): figure out a way to inject the testing code without 74 # modifying the example; maybe an extension? 75 {'name': 'part1'}, 76] 77 78def ValidateToolchains(toolchains): 79 invalid_toolchains = set(toolchains) - set(ALL_TOOLCHAINS) 80 if invalid_toolchains: 81 buildbot_common.ErrorExit('Invalid toolchain(s): %s' % ( 82 ', '.join(invalid_toolchains))) 83 84 85def GetServingDirForProject(desc): 86 dest = desc['DEST'] 87 path = os.path.join(pepperdir, *dest.split('/')) 88 return os.path.join(path, desc['NAME']) 89 90 91def GetRepoServingDirForProject(desc): 92 # This differs from GetServingDirForProject, because it returns the location 93 # within the Chrome repository of the project, not the "pepperdir". 94 return os.path.dirname(desc['FILEPATH']) 95 96 97def GetExecutableDirForProject(desc, toolchain, config): 98 return os.path.join(GetServingDirForProject(desc), toolchain, config) 99 100 101def GetBrowserTesterCommand(desc, toolchain, config): 102 if browser_path is None: 103 buildbot_common.ErrorExit('Failed to find chrome browser using FindChrome.') 104 105 args = [ 106 sys.executable, 107 browser_tester_py, 108 '--browser_path', browser_path, 109 '--timeout', '30.0', # seconds 110 # Prevent the infobar that shows up when requesting filesystem quota. 111 '--browser_flag', '--unlimited-storage', 112 '--enable_sockets', 113 # Prevent installing a new copy of PNaCl. 114 '--browser_flag', '--disable-component-update', 115 ] 116 117 args.extend(['--serving_dir', GetServingDirForProject(desc)]) 118 # Fall back on the example directory in the Chromium repo, to find test.js. 119 args.extend(['--serving_dir', GetRepoServingDirForProject(desc)]) 120 # If it is not found there, fall back on the dummy one (in this directory.) 121 args.extend(['--serving_dir', SCRIPT_DIR]) 122 123 if toolchain == platform: 124 exe_dir = GetExecutableDirForProject(desc, toolchain, config) 125 ppapi_plugin = os.path.join(exe_dir, desc['NAME']) 126 if platform == 'win': 127 ppapi_plugin += '.dll' 128 else: 129 ppapi_plugin += '.so' 130 args.extend(['--ppapi_plugin', ppapi_plugin]) 131 132 ppapi_plugin_mimetype = 'application/x-ppapi-%s' % config.lower() 133 args.extend(['--ppapi_plugin_mimetype', ppapi_plugin_mimetype]) 134 135 if toolchain == 'pnacl': 136 args.extend(['--browser_flag', '--enable-pnacl']) 137 138 url = 'index.html' 139 url += '?tc=%s&config=%s&test=true' % (toolchain, config) 140 args.extend(['--url', url]) 141 return args 142 143 144def GetBrowserTesterEnv(): 145 # browser_tester imports tools/valgrind/memcheck_analyze, which imports 146 # tools/valgrind/common. Well, it tries to, anyway, but instead imports 147 # common from PYTHONPATH first (which on the buildbots, is a 148 # common/__init__.py file...). 149 # 150 # Clear the PYTHONPATH so it imports the correct file. 151 env = dict(os.environ) 152 env['PYTHONPATH'] = '' 153 return env 154 155 156def RunTestOnce(desc, toolchain, config): 157 args = GetBrowserTesterCommand(desc, toolchain, config) 158 env = GetBrowserTesterEnv() 159 start_time = time.time() 160 try: 161 subprocess.check_call(args, env=env) 162 result = True 163 except subprocess.CalledProcessError: 164 result = False 165 elapsed = (time.time() - start_time) * 1000 166 return result, elapsed 167 168 169def RunTestNTimes(desc, toolchain, config, times): 170 total_elapsed = 0 171 for _ in xrange(times): 172 result, elapsed = RunTestOnce(desc, toolchain, config) 173 total_elapsed += elapsed 174 if result: 175 # Success, stop retrying. 176 break 177 return result, total_elapsed 178 179 180def RunTestWithGtestOutput(desc, toolchain, config, retry_on_failure_times): 181 test_name = GetTestName(desc, toolchain, config) 182 WriteGtestHeader(test_name) 183 result, elapsed = RunTestNTimes(desc, toolchain, config, 184 retry_on_failure_times) 185 WriteGtestFooter(result, test_name, elapsed) 186 return result 187 188 189def WriteGtestHeader(test_name): 190 print '\n[ RUN ] %s' % test_name 191 sys.stdout.flush() 192 sys.stderr.flush() 193 194 195def WriteGtestFooter(success, test_name, elapsed): 196 sys.stdout.flush() 197 sys.stderr.flush() 198 if success: 199 message = '[ OK ]' 200 else: 201 message = '[ FAILED ]' 202 print '%s %s (%d ms)' % (message, test_name, elapsed) 203 204 205def GetTestName(desc, toolchain, config): 206 return '%s.%s_%s_test' % (desc['NAME'], toolchain, config.lower()) 207 208 209def IsTestDisabled(desc, toolchain, config): 210 def AsList(value): 211 if type(value) not in (list, tuple): 212 return [value] 213 return value 214 215 def TestMatchesDisabled(test_values, disabled_test): 216 for key in test_values: 217 if key in disabled_test: 218 if test_values[key] not in AsList(disabled_test[key]): 219 return False 220 return True 221 222 test_values = { 223 'name': desc['NAME'], 224 'toolchain': toolchain, 225 'config': config, 226 'platform': platform 227 } 228 229 for disabled_test in DISABLED_TESTS: 230 if TestMatchesDisabled(test_values, disabled_test): 231 return True 232 return False 233 234 235def WriteHorizontalBar(): 236 print '-' * 80 237 238 239def WriteBanner(message): 240 WriteHorizontalBar() 241 print message 242 WriteHorizontalBar() 243 244 245def RunAllTestsInTree(tree, toolchains, configs, retry_on_failure_times): 246 tests_run = 0 247 total_tests = 0 248 failed = [] 249 disabled = [] 250 251 for _, desc in parse_dsc.GenerateProjects(tree): 252 desc_configs = desc.get('CONFIGS', ALL_CONFIGS) 253 valid_toolchains = set(toolchains) & set(desc['TOOLS']) 254 valid_configs = set(configs) & set(desc_configs) 255 for toolchain in sorted(valid_toolchains): 256 for config in sorted(valid_configs): 257 test_name = GetTestName(desc, toolchain, config) 258 total_tests += 1 259 if IsTestDisabled(desc, toolchain, config): 260 disabled.append(test_name) 261 continue 262 263 tests_run += 1 264 success = RunTestWithGtestOutput(desc, toolchain, config, 265 retry_on_failure_times) 266 if not success: 267 failed.append(test_name) 268 269 if failed: 270 WriteBanner('FAILED TESTS') 271 for test in failed: 272 print ' %s failed.' % test 273 274 if disabled: 275 WriteBanner('DISABLED TESTS') 276 for test in disabled: 277 print ' %s disabled.' % test 278 279 WriteHorizontalBar() 280 print 'Tests run: %d/%d (%d disabled).' % ( 281 tests_run, total_tests, len(disabled)) 282 print 'Tests succeeded: %d/%d.' % (tests_run - len(failed), tests_run) 283 284 success = len(failed) != 0 285 return success 286 287 288def BuildAllTestsInTree(tree, toolchains, configs): 289 for branch, desc in parse_dsc.GenerateProjects(tree): 290 desc_configs = desc.get('CONFIGS', ALL_CONFIGS) 291 valid_toolchains = set(toolchains) & set(desc['TOOLS']) 292 valid_configs = set(configs) & set(desc_configs) 293 for toolchain in sorted(valid_toolchains): 294 for config in sorted(valid_configs): 295 name = '%s/%s' % (branch, desc['NAME']) 296 build_projects.BuildProjectsBranch(pepperdir, name, deps=False, 297 clean=False, config=config, 298 args=['TOOLCHAIN=%s' % toolchain]) 299 300 301def GetProjectTree(include): 302 # Everything in src is a library, and cannot be run. 303 exclude = {'DEST': 'src'} 304 try: 305 return parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=include, 306 exclude=exclude) 307 except parse_dsc.ValidationError as e: 308 buildbot_common.ErrorExit(str(e)) 309 310 311def main(args): 312 parser = optparse.OptionParser() 313 parser.add_option('-c', '--config', 314 help='Choose configuration to run (Debug or Release). Runs both ' 315 'by default', action='append') 316 parser.add_option('-x', '--experimental', 317 help='Run experimental projects', action='store_true') 318 parser.add_option('-t', '--toolchain', 319 help='Run using toolchain. Can be passed more than once.', 320 action='append', default=[]) 321 parser.add_option('-d', '--dest', 322 help='Select which destinations (project types) are valid.', 323 action='append') 324 parser.add_option('-b', '--build', 325 help='Build each project before testing.', action='store_true') 326 parser.add_option('--retry-times', 327 help='Number of types to retry on failure (Default: %default)', 328 type='int', default=1) 329 330 options, args = parser.parse_args(args[1:]) 331 332 if not options.toolchain: 333 options.toolchain = ['newlib', 'glibc', 'pnacl', 'host'] 334 335 if 'host' in options.toolchain: 336 options.toolchain.remove('host') 337 options.toolchain.append(platform) 338 print 'Adding platform: ' + platform 339 340 ValidateToolchains(options.toolchain) 341 342 include = {} 343 if options.toolchain: 344 include['TOOLS'] = options.toolchain 345 print 'Filter by toolchain: ' + str(options.toolchain) 346 if not options.experimental: 347 include['EXPERIMENTAL'] = False 348 if options.dest: 349 include['DEST'] = options.dest 350 print 'Filter by type: ' + str(options.dest) 351 if args: 352 include['NAME'] = args 353 print 'Filter by name: ' + str(args) 354 if not options.config: 355 options.config = ALL_CONFIGS 356 357 project_tree = GetProjectTree(include) 358 if options.build: 359 BuildAllTestsInTree(project_tree, options.toolchain, options.config) 360 361 return RunAllTestsInTree(project_tree, options.toolchain, options.config, 362 options.retry_times) 363 364 365if __name__ == '__main__': 366 script_name = os.path.basename(sys.argv[0]) 367 try: 368 sys.exit(main(sys.argv)) 369 except parse_dsc.ValidationError as e: 370 buildbot_common.ErrorExit('%s: %s' % (script_name, e)) 371 except KeyboardInterrupt: 372 buildbot_common.ErrorExit('%s: interrupted' % script_name) 373