1b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#!/usr/bin/env python 2b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# 3b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# Copyright 2012 the V8 project authors. All rights reserved. 4b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# Redistribution and use in source and binary forms, with or without 5b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# modification, are permitted provided that the following conditions are 6b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# met: 7b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# 8b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# * Redistributions of source code must retain the above copyright 9b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# notice, this list of conditions and the following disclaimer. 10b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# * Redistributions in binary form must reproduce the above 11b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# copyright notice, this list of conditions and the following 12b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# disclaimer in the documentation and/or other materials provided 13b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# with the distribution. 14b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# * Neither the name of Google Inc. nor the names of its 15b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# contributors may be used to endorse or promote products derived 16b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# from this software without specific prior written permission. 17b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# 18b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 30b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 31b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom collections import OrderedDict 32b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport itertools 33b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport multiprocessing 34b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport optparse 35b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport os 36b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom os.path import join 37b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport platform 38b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport random 39b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport shlex 40b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport subprocess 41b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport sys 42b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport time 43b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 44b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom testrunner.local import execution 45b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom testrunner.local import progress 46b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom testrunner.local import testsuite 47b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom testrunner.local import utils 48b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom testrunner.local import verbose 49b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom testrunner.network import network_execution 50b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom testrunner.objects import context 51b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 52b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 53b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochARCH_GUESS = utils.DefaultArch() 54b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochDEFAULT_TESTS = ["mjsunit", "fuzz-natives", "base-unittests", 55b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "cctest", "compiler-unittests", "heap-unittests", 56b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "libplatform-unittests", "message", "preparser"] 57b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 58b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# Map of test name synonyms to lists of test suites. Should be ordered by 59b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# expected runtimes (suites with slow test cases first). These groups are 60b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# invoked in seperate steps on the bots. 61b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochTEST_MAP = { 62b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "default": [ 63b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "mjsunit", 64b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "fuzz-natives", 65b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "cctest", 66b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "message", 67b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "preparser", 68b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ], 69b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "optimize_for_size": [ 70b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "mjsunit", 71b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "cctest", 72b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "webkit", 73b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ], 74b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "unittests": [ 75b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "compiler-unittests", 76b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "heap-unittests", 77b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "base-unittests", 78b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "libplatform-unittests", 79b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ], 80b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 81b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 82b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochTIMEOUT_DEFAULT = 60 83b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochTIMEOUT_SCALEFACTOR = {"debug" : 4, 84b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "release" : 1 } 85b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 86b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# Use this to run several variants of the tests. 87b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochVARIANT_FLAGS = { 88b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "default": [], 89b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "stress": ["--stress-opt", "--always-opt"], 90b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "turbofan": ["--turbo-filter=*", "--always-opt"], 91b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "nocrankshaft": ["--nocrankshaft"]} 92b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 93b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochVARIANTS = ["default", "stress", "turbofan", "nocrankshaft"] 94b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 95b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochMODE_FLAGS = { 96b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "debug" : ["--nohard-abort", "--nodead-code-elimination", 97b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "--nofold-constants", "--enable-slow-asserts", 98b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "--debug-code", "--verify-heap"], 99b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "release" : ["--nohard-abort", "--nodead-code-elimination", 100b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "--nofold-constants"]} 101b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 102b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochGC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", 103b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "--concurrent-recompilation-queue-length=64", 104b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "--concurrent-recompilation-delay=500", 105b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "--concurrent-recompilation"] 106b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 107b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochSUPPORTED_ARCHS = ["android_arm", 108b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "android_arm64", 109b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "android_ia32", 110b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "arm", 111b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "ia32", 112b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "x87", 113b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "mips", 114b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "mipsel", 115b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "mips64el", 116b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "nacl_ia32", 117b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "nacl_x64", 118b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "x64", 119b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "x32", 120b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "arm64"] 121b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# Double the timeout for these: 122b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochSLOW_ARCHS = ["android_arm", 123b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "android_arm64", 124b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "android_ia32", 125b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "arm", 126b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "mips", 127b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "mipsel", 128b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "mips64el", 129b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "nacl_ia32", 130b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "nacl_x64", 131b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "x87", 132b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "arm64"] 133b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 134b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 135b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochdef BuildOptions(): 136b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result = optparse.OptionParser() 137b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--arch", 138b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help=("The architecture to run tests for, " 139b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "'auto' or 'native' for auto-detect"), 140b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default="ia32,x64,arm") 141b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--arch-and-mode", 142b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Architecture and mode in the format 'arch.mode'", 143b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=None) 144b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--asan", 145b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Regard test expectations for ASAN", 146b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 147b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--buildbot", 148b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Adapt to path structure used on buildbots", 149b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 150b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--cat", help="Print the source of the tests", 151b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 152b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--flaky-tests", 153b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Regard tests marked as flaky (run|skip|dontcare)", 154b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default="dontcare") 155b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--slow-tests", 156b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Regard slow tests (run|skip|dontcare)", 157b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default="dontcare") 158b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--pass-fail-tests", 159b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Regard pass|fail tests (run|skip|dontcare)", 160b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default="dontcare") 161b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--gc-stress", 162b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Switch on GC stress mode", 163b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 164b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--command-prefix", 165b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Prepended to each shell command used to run a test", 166b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default="") 167b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--download-data", help="Download missing test suite data", 168b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 169b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--extra-flags", 170b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Additional flags to pass to each test command", 171b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default="") 172b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--isolates", help="Whether to test isolates", 173b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 174b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("-j", help="The number of parallel tasks to run", 175b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=0, type="int") 176b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("-m", "--mode", 177b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="The test modes in which to run (comma-separated)", 178b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default="release,debug") 179b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--no-i18n", "--noi18n", 180b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Skip internationalization tests", 181b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 182b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--no-network", "--nonetwork", 183b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Don't distribute tests on the network", 184b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=(utils.GuessOS() != "linux"), 185b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch dest="no_network", action="store_true") 186b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--no-presubmit", "--nopresubmit", 187b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help='Skip presubmit checks', 188b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, dest="no_presubmit", action="store_true") 189b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--no-snap", "--nosnap", 190b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help='Test a build compiled without snapshot.', 191b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, dest="no_snap", action="store_true") 192b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--no-sorting", "--nosorting", 193b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Don't sort tests according to duration of last run.", 194b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, dest="no_sorting", action="store_true") 195b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--no-stress", "--nostress", 196b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Don't run crankshaft --always-opt --stress-op test", 197b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, dest="no_stress", action="store_true") 198b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--no-variants", "--novariants", 199b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Don't run any testing variants", 200b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, dest="no_variants", action="store_true") 201b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--variants", 202b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Comma-separated list of testing variants") 203b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--outdir", help="Base directory with compile output", 204b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default="out") 205b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--predictable", 206b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Compare output of several reruns of each test", 207b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 208b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("-p", "--progress", 209b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help=("The style of progress indicator" 210b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch " (verbose, dots, color, mono)"), 211b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch choices=progress.PROGRESS_INDICATORS.keys(), default="mono") 212b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--quickcheck", default=False, action="store_true", 213b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help=("Quick check mode (skip slow/flaky tests)")) 214b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--report", help="Print a summary of the tests to be run", 215b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 216b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--json-test-results", 217b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Path to a file for storing json results.") 218b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--rerun-failures-count", 219b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help=("Number of times to rerun each failing test case. " 220b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "Very slow tests will be rerun only once."), 221b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=0, type="int") 222b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--rerun-failures-max", 223b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Maximum number of failing test cases to rerun.", 224b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=100, type="int") 225b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--shard-count", 226b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Split testsuites into this number of shards", 227b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=1, type="int") 228b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--shard-run", 229b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Run this shard from the split up tests.", 230b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=1, type="int") 231b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") 232b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--shell-dir", help="Directory containing executables", 233b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default="") 234b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--dont-skip-slow-simulator-tests", 235b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Don't skip more slow tests when using a simulator.", 236b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true", 237b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch dest="dont_skip_simulator_slow_tests") 238b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--stress-only", 239b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Only run tests with --always-opt --stress-opt", 240b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 241b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--time", help="Print timing information after running", 242b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 243b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("-t", "--timeout", help="Timeout in seconds", 244b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default= -1, type="int") 245b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--tsan", 246b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Regard test expectations for TSAN", 247b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 248b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("-v", "--verbose", help="Verbose output", 249b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 250b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--valgrind", help="Run tests through valgrind", 251b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 252b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--warn-unused", help="Report unused rules", 253b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default=False, action="store_true") 254b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--junitout", help="File name of the JUnit output") 255b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--junittestsuite", 256b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="The testsuite name in the JUnit output file", 257b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch default="v8tests") 258b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.add_option("--random-seed", default=0, dest="random_seed", 259b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch help="Default seed for initializing random generator") 260b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return result 261b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 262b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 263b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochdef ProcessOptions(options): 264b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch global VARIANT_FLAGS 265b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch global VARIANTS 266b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 267b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Architecture and mode related stuff. 268b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.arch_and_mode: 269b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.arch_and_mode = [arch_and_mode.split(".") 270b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for arch_and_mode in options.arch_and_mode.split(",")] 271b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode]) 272b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode]) 273b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.mode = options.mode.split(",") 274b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for mode in options.mode: 275b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not mode.lower() in ["debug", "release", "optdebug"]: 276b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print "Unknown mode %s" % mode 277b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return False 278b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.arch in ["auto", "native"]: 279b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.arch = ARCH_GUESS 280b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.arch = options.arch.split(",") 281b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for arch in options.arch: 282b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not arch in SUPPORTED_ARCHS: 283b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print "Unknown architecture %s" % arch 284b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return False 285b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 286b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Store the final configuration in arch_and_mode list. Don't overwrite 287b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # predefined arch_and_mode since it is more expressive than arch and mode. 288b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not options.arch_and_mode: 289b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.arch_and_mode = itertools.product(options.arch, options.mode) 290b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 291b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Special processing of other options, sorted alphabetically. 292b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 293b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.buildbot: 294b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Buildbots run presubmit tests as a separate step. 295b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.no_presubmit = True 296b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.no_network = True 297b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.command_prefix: 298b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print("Specifying --command-prefix disables network distribution, " 299b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "running tests locally.") 300b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.no_network = True 301b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.command_prefix = shlex.split(options.command_prefix) 302b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.extra_flags = shlex.split(options.extra_flags) 303b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 304b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.gc_stress: 305b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.extra_flags += GC_STRESS_FLAGS 306b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 307b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.asan: 308b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.extra_flags.append("--invoke-weak-callbacks") 309b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 310b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.tsan: 311b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch VARIANTS = ["default"] 312b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 313b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.j == 0: 314b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.j = multiprocessing.cpu_count() 315b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 316b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while options.random_seed == 0: 317b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647) 318b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 319b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def excl(*args): 320b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch """Returns true if zero or one of multiple arguments are true.""" 321b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return reduce(lambda x, y: x + y, args) <= 1 322b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 323b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not excl(options.no_stress, options.stress_only, options.no_variants, 324b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool(options.variants)): 325b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print("Use only one of --no-stress, --stress-only, --no-variants, " 326b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "or --variants.") 327b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return False 328b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.quickcheck: 329b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch VARIANTS = ["default", "stress"] 330b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.flaky_tests = "skip" 331b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.slow_tests = "skip" 332b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.pass_fail_tests = "skip" 333b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.no_stress: 334b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch VARIANTS = ["default", "nocrankshaft"] 335b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.no_variants: 336b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch VARIANTS = ["default"] 337b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.stress_only: 338b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch VARIANTS = ["stress"] 339b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.variants: 340b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch VARIANTS = options.variants.split(",") 341b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not set(VARIANTS).issubset(VARIANT_FLAGS.keys()): 342b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print "All variants must be in %s" % str(VARIANT_FLAGS.keys()) 343b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return False 344b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.predictable: 345b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch VARIANTS = ["default"] 346b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.extra_flags.append("--predictable") 347b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.extra_flags.append("--verify_predictable") 348b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.extra_flags.append("--no-inline-new") 349b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 350b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not options.shell_dir: 351b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.shell: 352b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print "Warning: --shell is deprecated, use --shell-dir instead." 353b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.shell_dir = os.path.dirname(options.shell) 354b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.valgrind: 355b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch run_valgrind = os.path.join("tools", "run-valgrind.py") 356b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # This is OK for distributed running, so we don't need to set no_network. 357b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.command_prefix = (["python", "-u", run_valgrind] + 358b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.command_prefix) 359b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def CheckTestMode(name, option): 360b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not option in ["run", "skip", "dontcare"]: 361b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print "Unknown %s mode %s" % (name, option) 362b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return False 363b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return True 364b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not CheckTestMode("flaky test", options.flaky_tests): 365b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return False 366b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not CheckTestMode("slow test", options.slow_tests): 367b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return False 368b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not CheckTestMode("pass|fail test", options.pass_fail_tests): 369b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return False 370b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not options.no_i18n: 371b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DEFAULT_TESTS.append("intl") 372b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return True 373b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 374b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 375b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochdef ShardTests(tests, shard_count, shard_run): 376b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if shard_count < 2: 377b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return tests 378b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if shard_run < 1 or shard_run > shard_count: 379b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print "shard-run not a valid number, should be in [1:shard-count]" 380b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print "defaulting back to running all tests" 381b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return tests 382b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch count = 0 383b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch shard = [] 384b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for test in tests: 385b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if count % shard_count == shard_run - 1: 386b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch shard.append(test) 387b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch count += 1 388b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return shard 389b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 390b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 391b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochdef Main(): 392b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch parser = BuildOptions() 393b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch (options, args) = parser.parse_args() 394b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not ProcessOptions(options): 395b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch parser.print_help() 396b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return 1 397b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 398b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch exit_code = 0 399b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), "..")) 400b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not options.no_presubmit: 401b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print ">>> running presubmit tests" 402b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch exit_code = subprocess.call( 403b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch [sys.executable, join(workspace, "tools", "presubmit.py")]) 404b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 405b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch suite_paths = utils.GetSuitePaths(join(workspace, "test")) 406b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 407b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Expand arguments with grouped tests. The args should reflect the list of 408b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # suites as otherwise filters would break. 409b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def ExpandTestGroups(name): 410b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if name in TEST_MAP: 411b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return [suite for suite in TEST_MAP[arg]] 412b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch else: 413b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return [name] 414b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch args = reduce(lambda x, y: x + y, 415b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch [ExpandTestGroups(arg) for arg in args], 416b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch []) 417b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 418b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if len(args) == 0: 419b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch suite_paths = [ s for s in DEFAULT_TESTS if s in suite_paths ] 420b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch else: 421b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch args_suites = OrderedDict() # Used as set 422b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for arg in args: 423b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch args_suites[arg.split(os.path.sep)[0]] = True 424b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch suite_paths = [ s for s in args_suites if s in suite_paths ] 425b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 426b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch suites = [] 427b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for root in suite_paths: 428b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch suite = testsuite.TestSuite.LoadTestSuite( 429b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch os.path.join(workspace, "test", root)) 430b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if suite: 431b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch suites.append(suite) 432b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 433b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.download_data: 434b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for s in suites: 435b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch s.DownloadData() 436b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 437b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for (arch, mode) in options.arch_and_mode: 438b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch try: 439b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch code = Execute(arch, mode, args, options, suites, workspace) 440b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch except KeyboardInterrupt: 441b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return 2 442b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch exit_code = exit_code or code 443b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return exit_code 444b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 445b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 446b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochdef Execute(arch, mode, args, options, suites, workspace): 447b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print(">>> Running tests for %s.%s" % (arch, mode)) 448b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 449b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch shell_dir = options.shell_dir 450b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not shell_dir: 451b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.buildbot: 452b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch shell_dir = os.path.join(workspace, options.outdir, mode) 453b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch mode = mode.lower() 454b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch else: 455b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch shell_dir = os.path.join(workspace, options.outdir, 456b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "%s.%s" % (arch, mode)) 457b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch shell_dir = os.path.relpath(shell_dir) 458b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 459b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if mode == "optdebug": 460b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch mode = "debug" # "optdebug" is just an alias. 461b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 462b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Populate context object. 463b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch mode_flags = MODE_FLAGS[mode] 464b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch timeout = options.timeout 465b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if timeout == -1: 466b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Simulators are slow, therefore allow a longer default timeout. 467b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if arch in SLOW_ARCHS: 468b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch timeout = 2 * TIMEOUT_DEFAULT; 469b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch else: 470b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch timeout = TIMEOUT_DEFAULT; 471b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 472b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch timeout *= TIMEOUT_SCALEFACTOR[mode] 473b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 474b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.predictable: 475b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Predictable mode is slower. 476b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch timeout *= 2 477b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 478b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ctx = context.Context(arch, mode, shell_dir, 479b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch mode_flags, options.verbose, 480b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch timeout, options.isolates, 481b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.command_prefix, 482b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.extra_flags, 483b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.no_i18n, 484b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.random_seed, 485b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.no_sorting, 486b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.rerun_failures_count, 487b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.rerun_failures_max, 488b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.predictable) 489b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 490b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # TODO(all): Combine "simulator" and "simulator_run". 491b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch simulator_run = not options.dont_skip_simulator_slow_tests and \ 492b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch arch in ['arm64', 'arm', 'mips'] and ARCH_GUESS and arch != ARCH_GUESS 493b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Find available test suites and read test cases from them. 494b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch variables = { 495b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "arch": arch, 496b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "asan": options.asan, 497b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "deopt_fuzzer": False, 498b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "gc_stress": options.gc_stress, 499b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "isolates": options.isolates, 500b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "mode": mode, 501b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "no_i18n": options.no_i18n, 502b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "no_snap": options.no_snap, 503b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "simulator_run": simulator_run, 504b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "simulator": utils.UseSimulator(arch), 505b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "system": utils.GuessOS(), 506b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "tsan": options.tsan, 507b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 508b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch all_tests = [] 509b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch num_tests = 0 510b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test_id = 0 511b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for s in suites: 512b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch s.ReadStatusFile(variables) 513b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch s.ReadTestCases(ctx) 514b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if len(args) > 0: 515b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch s.FilterTestCasesByArgs(args) 516b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch all_tests += s.tests 517b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests, 518b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch options.slow_tests, options.pass_fail_tests) 519b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.cat: 520b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch verbose.PrintTestSource(s.tests) 521b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch continue 522b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS] 523b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch s.tests = [ t.CopyAddingFlags(v) 524b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for t in s.tests 525b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for v in s.VariantFlags(t, variant_flags) ] 526b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch s.tests = ShardTests(s.tests, options.shard_count, options.shard_run) 527b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch num_tests += len(s.tests) 528b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for t in s.tests: 529b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch t.id = test_id 530b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test_id += 1 531b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 532b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.cat: 533b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return 0 # We're done here. 534b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 535b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.report: 536b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch verbose.PrintReport(all_tests) 537b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 538b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if num_tests == 0: 539b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print "No tests to run." 540b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return 0 541b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 542b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Run the tests, either locally or distributed on the network. 543b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch start_time = time.time() 544b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() 545b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.junitout: 546b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch progress_indicator = progress.JUnitTestProgressIndicator( 547b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch progress_indicator, options.junitout, options.junittestsuite) 548b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.json_test_results: 549b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch progress_indicator = progress.JsonTestProgressIndicator( 550b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch progress_indicator, options.json_test_results, arch, mode) 551b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 552b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch run_networked = not options.no_network 553b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not run_networked: 554b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print("Network distribution disabled, running tests locally.") 555b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch elif utils.GuessOS() != "linux": 556b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print("Network distribution is only supported on Linux, sorry!") 557b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch run_networked = False 558b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch peers = [] 559b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if run_networked: 560b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch peers = network_execution.GetPeers() 561b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not peers: 562b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print("No connection to distribution server; running tests locally.") 563b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch run_networked = False 564b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch elif len(peers) == 1: 565b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print("No other peers on the network; running tests locally.") 566b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch run_networked = False 567b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch elif num_tests <= 100: 568b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch print("Less than 100 tests, running them locally.") 569b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch run_networked = False 570b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 571b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if run_networked: 572b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch runner = network_execution.NetworkedRunner(suites, progress_indicator, 573b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ctx, peers, workspace) 574b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch else: 575b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch runner = execution.Runner(suites, progress_indicator, ctx) 576b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 577b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch exit_code = runner.Run(options.j) 578b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch overall_duration = time.time() - start_time 579b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 580b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if options.time: 581b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch verbose.PrintTestDurations(suites, overall_duration) 582b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return exit_code 583b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 584b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 585b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochif __name__ == "__main__": 586b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch sys.exit(Main()) 587