1ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch#!/usr/bin/env python 2ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# Copyright (c) 2013 The Chromium Authors. All rights reserved. 3ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# Use of this source code is governed by a BSD-style license that can be 4ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# found in the LICENSE file. 5ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 6ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochimport optparse 7ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochimport os 8ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochimport subprocess 9ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochimport sys 10ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochimport time 11ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import build_projects 13ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochimport build_version 145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import buildbot_common 15ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochimport parse_dsc 16ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 173551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)from build_paths import OUT_DIR, SRC_DIR, SDK_SRC_DIR, SCRIPT_DIR 18ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 19ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochsys.path.append(os.path.join(SDK_SRC_DIR, 'tools')) 20ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochimport getos 21ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochplatform = getos.GetPlatform() 22ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 23ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# TODO(binji): ugly hack -- can I get the browser in a cleaner way? 24ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochsys.path.append(os.path.join(SRC_DIR, 'chrome', 'test', 'nacl_test_injection')) 25ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochimport find_chrome 26ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochbrowser_path = find_chrome.FindChrome(SRC_DIR, ['Debug', 'Release']) 27ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 28ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 29ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochpepper_ver = str(int(build_version.ChromeMajorVersion())) 30ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochpepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver) 31ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 32ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochbrowser_tester_py = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'tools', 33ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 'browser_tester', 'browser_tester.py') 34ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 35ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 36ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben MurdochALL_CONFIGS = ['Debug', 'Release'] 37ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben MurdochALL_TOOLCHAINS = ['newlib', 'glibc', 'pnacl', 'win', 'linux', 'mac'] 38ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 39ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# Values you can filter by: 40ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# name: The name of the test. (e.g. "pi_generator") 41ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# config: See ALL_CONFIGS above. 42ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# toolchain: See ALL_TOOLCHAINS above. 43ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# platform: mac/win/linux. 44ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# 45ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# All keys must be matched, but any value that matches in a sequence is 46ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# considered a match for that key. For example: 47ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# 48ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# {'name': ('pi_generator', 'input_event'), 'toolchain': ('newlib', 'pnacl')} 49ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# 50ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# Will match 8 tests: 51ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# pi_generator.newlib_debug_test 52ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# pi_generator.newlib_release_test 53ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# input_event.newlib_debug_test 54ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# input_event.newlib_release_test 55ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# pi_generator.glibc_debug_test 56ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# pi_generator.glibc_release_test 57ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# input_event.glibc_debug_test 58ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# input_event.glibc_release_test 59ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben MurdochDISABLED_TESTS = [ 60a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) # TODO(binji): Disable 3D examples on linux/win/mac. See 61ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # http://crbug.com/262379. 62a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) {'name': 'graphics_3d', 'platform': ('win', 'linux', 'mac')}, 63116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch {'name': 'video_decode', 'platform': ('win', 'linux', 'mac')}, 64effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch # media_stream_audio uses audio input devices which are not supported. 65effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch {'name': 'media_stream_audio', 'platform': ('win', 'linux', 'mac')}, 665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # media_stream_video uses 3D and webcam which are not supported. 675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) {'name': 'media_stream_video', 'platform': ('win', 'linux', 'mac')}, 68424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) # TODO(binji): These tests timeout on the trybots because the NEXEs take 69424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) # more than 40 seconds to load (!). See http://crbug.com/280753 70424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) {'name': 'nacl_io_test', 'platform': 'win', 'toolchain': 'glibc'}, 7158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) # We don't test "getting_started/part1" because it would complicate the 7258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) # example. 7358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) # TODO(binji): figure out a way to inject the testing code without 7458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) # modifying the example; maybe an extension? 7558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) {'name': 'part1'}, 76ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch] 77ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 78ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef ValidateToolchains(toolchains): 79ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch invalid_toolchains = set(toolchains) - set(ALL_TOOLCHAINS) 80ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if invalid_toolchains: 81ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch buildbot_common.ErrorExit('Invalid toolchain(s): %s' % ( 82ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch ', '.join(invalid_toolchains))) 83ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 84ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 85ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef GetServingDirForProject(desc): 86ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch dest = desc['DEST'] 87ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch path = os.path.join(pepperdir, *dest.split('/')) 88ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return os.path.join(path, desc['NAME']) 89ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 90ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 913551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)def GetRepoServingDirForProject(desc): 923551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) # This differs from GetServingDirForProject, because it returns the location 933551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) # within the Chrome repository of the project, not the "pepperdir". 943551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return os.path.dirname(desc['FILEPATH']) 953551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 963551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 97ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef GetExecutableDirForProject(desc, toolchain, config): 98ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return os.path.join(GetServingDirForProject(desc), toolchain, config) 99ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 100ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 101ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef GetBrowserTesterCommand(desc, toolchain, config): 102424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) if browser_path is None: 103424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) buildbot_common.ErrorExit('Failed to find chrome browser using FindChrome.') 104424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) 105ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch args = [ 106ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch sys.executable, 107ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch browser_tester_py, 108ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch '--browser_path', browser_path, 109ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch '--timeout', '30.0', # seconds 110ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # Prevent the infobar that shows up when requesting filesystem quota. 111ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch '--browser_flag', '--unlimited-storage', 1123551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) '--enable_sockets', 113f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) # Prevent installing a new copy of PNaCl. 114f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) '--browser_flag', '--disable-component-update', 115ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch ] 116ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 117ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch args.extend(['--serving_dir', GetServingDirForProject(desc)]) 1183551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) # Fall back on the example directory in the Chromium repo, to find test.js. 1193551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) args.extend(['--serving_dir', GetRepoServingDirForProject(desc)]) 1203551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) # If it is not found there, fall back on the dummy one (in this directory.) 1213551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) args.extend(['--serving_dir', SCRIPT_DIR]) 122ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 123ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if toolchain == platform: 1243551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) exe_dir = GetExecutableDirForProject(desc, toolchain, config) 125ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch ppapi_plugin = os.path.join(exe_dir, desc['NAME']) 126ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if platform == 'win': 127ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch ppapi_plugin += '.dll' 128ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch else: 129ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch ppapi_plugin += '.so' 130ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch args.extend(['--ppapi_plugin', ppapi_plugin]) 131ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 132424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) ppapi_plugin_mimetype = 'application/x-ppapi-%s' % config.lower() 133424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) args.extend(['--ppapi_plugin_mimetype', ppapi_plugin_mimetype]) 134424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) 1353240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch if toolchain == 'pnacl': 1363240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch args.extend(['--browser_flag', '--enable-pnacl']) 1373240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch 138ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch url = 'index.html' 139ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch url += '?tc=%s&config=%s&test=true' % (toolchain, config) 140ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch args.extend(['--url', url]) 141ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return args 142ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 143ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 144ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef GetBrowserTesterEnv(): 145ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # browser_tester imports tools/valgrind/memcheck_analyze, which imports 146ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # tools/valgrind/common. Well, it tries to, anyway, but instead imports 147ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # common from PYTHONPATH first (which on the buildbots, is a 148ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # common/__init__.py file...). 149ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # 150ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # Clear the PYTHONPATH so it imports the correct file. 151ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch env = dict(os.environ) 152ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch env['PYTHONPATH'] = '' 153ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return env 154ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 155ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 156ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef RunTestOnce(desc, toolchain, config): 157ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch args = GetBrowserTesterCommand(desc, toolchain, config) 158ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch env = GetBrowserTesterEnv() 159ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch start_time = time.time() 160ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch try: 161ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch subprocess.check_call(args, env=env) 162ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch result = True 163ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch except subprocess.CalledProcessError: 164ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch result = False 165ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch elapsed = (time.time() - start_time) * 1000 166ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return result, elapsed 167ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 168ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 169ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef RunTestNTimes(desc, toolchain, config, times): 170ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch total_elapsed = 0 171ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch for _ in xrange(times): 172ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch result, elapsed = RunTestOnce(desc, toolchain, config) 173ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch total_elapsed += elapsed 174ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if result: 175ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # Success, stop retrying. 176ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch break 177ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return result, total_elapsed 178ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 179ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 180ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef RunTestWithGtestOutput(desc, toolchain, config, retry_on_failure_times): 181ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch test_name = GetTestName(desc, toolchain, config) 182ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch WriteGtestHeader(test_name) 183ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch result, elapsed = RunTestNTimes(desc, toolchain, config, 184ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch retry_on_failure_times) 185ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch WriteGtestFooter(result, test_name, elapsed) 186ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return result 187ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 188ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 189ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef WriteGtestHeader(test_name): 190ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch print '\n[ RUN ] %s' % test_name 191ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch sys.stdout.flush() 192ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch sys.stderr.flush() 193ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 194ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 195ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef WriteGtestFooter(success, test_name, elapsed): 196ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch sys.stdout.flush() 197ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch sys.stderr.flush() 198ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if success: 199ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch message = '[ OK ]' 200ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch else: 201ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch message = '[ FAILED ]' 202ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch print '%s %s (%d ms)' % (message, test_name, elapsed) 203ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 204ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 205ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef GetTestName(desc, toolchain, config): 206ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return '%s.%s_%s_test' % (desc['NAME'], toolchain, config.lower()) 207ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 208ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 209ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef IsTestDisabled(desc, toolchain, config): 210ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch def AsList(value): 2113551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if type(value) not in (list, tuple): 2123551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return [value] 213ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return value 214ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 2153551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) def TestMatchesDisabled(test_values, disabled_test): 2163551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) for key in test_values: 2173551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if key in disabled_test: 2183551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if test_values[key] not in AsList(disabled_test[key]): 2193551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return False 2203551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return True 2213551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 222ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch test_values = { 223ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 'name': desc['NAME'], 224ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 'toolchain': toolchain, 225ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 'config': config, 226ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 'platform': platform 227ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch } 228ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 229ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch for disabled_test in DISABLED_TESTS: 2303551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if TestMatchesDisabled(test_values, disabled_test): 2313551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return True 232ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return False 233ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 234ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 235ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef WriteHorizontalBar(): 236ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch print '-' * 80 237ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 238ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 239ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef WriteBanner(message): 240ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch WriteHorizontalBar() 241ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch print message 242ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch WriteHorizontalBar() 243ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 244ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 245ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef RunAllTestsInTree(tree, toolchains, configs, retry_on_failure_times): 246ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch tests_run = 0 247ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch total_tests = 0 248ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch failed = [] 249ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch disabled = [] 250ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 251ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch for _, desc in parse_dsc.GenerateProjects(tree): 252ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch desc_configs = desc.get('CONFIGS', ALL_CONFIGS) 253ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch valid_toolchains = set(toolchains) & set(desc['TOOLS']) 254ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch valid_configs = set(configs) & set(desc_configs) 255ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch for toolchain in sorted(valid_toolchains): 256ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch for config in sorted(valid_configs): 257ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch test_name = GetTestName(desc, toolchain, config) 258ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch total_tests += 1 259ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if IsTestDisabled(desc, toolchain, config): 260ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch disabled.append(test_name) 261ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch continue 262ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 263ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch tests_run += 1 264ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch success = RunTestWithGtestOutput(desc, toolchain, config, 265ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch retry_on_failure_times) 266ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if not success: 267ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch failed.append(test_name) 268ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 269ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if failed: 270ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch WriteBanner('FAILED TESTS') 271ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch for test in failed: 272ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch print ' %s failed.' % test 273ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 274ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if disabled: 275ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch WriteBanner('DISABLED TESTS') 276ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch for test in disabled: 277ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch print ' %s disabled.' % test 278ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 279ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch WriteHorizontalBar() 280ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch print 'Tests run: %d/%d (%d disabled).' % ( 281ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch tests_run, total_tests, len(disabled)) 282ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch print 'Tests succeeded: %d/%d.' % (tests_run - len(failed), tests_run) 283ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 284ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch success = len(failed) != 0 285ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return success 286ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 287ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 2885d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)def BuildAllTestsInTree(tree, toolchains, configs): 2895d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for branch, desc in parse_dsc.GenerateProjects(tree): 2905d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) desc_configs = desc.get('CONFIGS', ALL_CONFIGS) 2915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) valid_toolchains = set(toolchains) & set(desc['TOOLS']) 2925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) valid_configs = set(configs) & set(desc_configs) 2935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for toolchain in sorted(valid_toolchains): 2945d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for config in sorted(valid_configs): 2955d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) name = '%s/%s' % (branch, desc['NAME']) 2965d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) build_projects.BuildProjectsBranch(pepperdir, name, deps=False, 2975d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) clean=False, config=config, 2985d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) args=['TOOLCHAIN=%s' % toolchain]) 2995d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 3005d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 301ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef GetProjectTree(include): 302ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch # Everything in src is a library, and cannot be run. 303ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch exclude = {'DEST': 'src'} 304ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch try: 305ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=include, 306ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch exclude=exclude) 307ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch except parse_dsc.ValidationError as e: 308ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch buildbot_common.ErrorExit(str(e)) 309ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 310ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 311ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochdef main(args): 312ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch parser = optparse.OptionParser() 3133551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) parser.add_option('-c', '--config', 314ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch help='Choose configuration to run (Debug or Release). Runs both ' 315ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 'by default', action='append') 316ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch parser.add_option('-x', '--experimental', 317ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch help='Run experimental projects', action='store_true') 318ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch parser.add_option('-t', '--toolchain', 319ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch help='Run using toolchain. Can be passed more than once.', 320ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch action='append', default=[]) 321ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch parser.add_option('-d', '--dest', 322ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch help='Select which destinations (project types) are valid.', 323ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch action='append') 3245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parser.add_option('-b', '--build', 3255d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) help='Build each project before testing.', action='store_true') 326ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch parser.add_option('--retry-times', 327ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch help='Number of types to retry on failure (Default: %default)', 3283551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) type='int', default=1) 329ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 330ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch options, args = parser.parse_args(args[1:]) 331ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 332ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if not options.toolchain: 333ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch options.toolchain = ['newlib', 'glibc', 'pnacl', 'host'] 334ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 335ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if 'host' in options.toolchain: 336ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch options.toolchain.remove('host') 337ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch options.toolchain.append(platform) 338ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch print 'Adding platform: ' + platform 339ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 340ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch ValidateToolchains(options.toolchain) 341ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 342ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch include = {} 343ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if options.toolchain: 344ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch include['TOOLS'] = options.toolchain 345ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch print 'Filter by toolchain: ' + str(options.toolchain) 346ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if not options.experimental: 347ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch include['EXPERIMENTAL'] = False 348ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if options.dest: 349ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch include['DEST'] = options.dest 350ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch print 'Filter by type: ' + str(options.dest) 351424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) if args: 352424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) include['NAME'] = args 353424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) print 'Filter by name: ' + str(args) 354ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if not options.config: 355ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch options.config = ALL_CONFIGS 356ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 357ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch project_tree = GetProjectTree(include) 3585d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if options.build: 3595d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) BuildAllTestsInTree(project_tree, options.toolchain, options.config) 3605d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 361ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return RunAllTestsInTree(project_tree, options.toolchain, options.config, 362ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch options.retry_times) 363ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 364ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 365ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochif __name__ == '__main__': 366ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch script_name = os.path.basename(sys.argv[0]) 367ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch try: 368ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch sys.exit(main(sys.argv)) 369ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch except parse_dsc.ValidationError as e: 370ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch buildbot_common.ErrorExit('%s: %s' % (script_name, e)) 371ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch except KeyboardInterrupt: 372ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch buildbot_common.ErrorExit('%s: interrupted' % script_name) 373