1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31from collections import OrderedDict
32import itertools
33import multiprocessing
34import optparse
35import os
36from os.path import join
37import platform
38import random
39import shlex
40import subprocess
41import sys
42import time
43
44from testrunner.local import execution
45from testrunner.local import progress
46from testrunner.local import testsuite
47from testrunner.local import utils
48from testrunner.local import verbose
49from testrunner.network import network_execution
50from testrunner.objects import context
51
52
53ARCH_GUESS = utils.DefaultArch()
54DEFAULT_TESTS = ["mjsunit", "fuzz-natives", "base-unittests",
55                 "cctest", "compiler-unittests", "heap-unittests",
56                 "libplatform-unittests", "message", "preparser"]
57
58# Map of test name synonyms to lists of test suites. Should be ordered by
59# expected runtimes (suites with slow test cases first). These groups are
60# invoked in seperate steps on the bots.
61TEST_MAP = {
62  "default": [
63    "mjsunit",
64    "fuzz-natives",
65    "cctest",
66    "message",
67    "preparser",
68  ],
69  "optimize_for_size": [
70    "mjsunit",
71    "cctest",
72    "webkit",
73  ],
74  "unittests": [
75    "compiler-unittests",
76    "heap-unittests",
77    "base-unittests",
78    "libplatform-unittests",
79  ],
80}
81
82TIMEOUT_DEFAULT = 60
83TIMEOUT_SCALEFACTOR = {"debug"   : 4,
84                       "release" : 1 }
85
86# Use this to run several variants of the tests.
87VARIANT_FLAGS = {
88    "default": [],
89    "stress": ["--stress-opt", "--always-opt"],
90    "turbofan": ["--turbo-filter=*", "--always-opt"],
91    "nocrankshaft": ["--nocrankshaft"]}
92
93VARIANTS = ["default", "stress", "turbofan", "nocrankshaft"]
94
95MODE_FLAGS = {
96    "debug"   : ["--nohard-abort", "--nodead-code-elimination",
97                 "--nofold-constants", "--enable-slow-asserts",
98                 "--debug-code", "--verify-heap"],
99    "release" : ["--nohard-abort", "--nodead-code-elimination",
100                 "--nofold-constants"]}
101
102GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction",
103                   "--concurrent-recompilation-queue-length=64",
104                   "--concurrent-recompilation-delay=500",
105                   "--concurrent-recompilation"]
106
107SUPPORTED_ARCHS = ["android_arm",
108                   "android_arm64",
109                   "android_ia32",
110                   "arm",
111                   "ia32",
112                   "x87",
113                   "mips",
114                   "mipsel",
115                   "mips64el",
116                   "nacl_ia32",
117                   "nacl_x64",
118                   "x64",
119                   "x32",
120                   "arm64"]
121# Double the timeout for these:
122SLOW_ARCHS = ["android_arm",
123              "android_arm64",
124              "android_ia32",
125              "arm",
126              "mips",
127              "mipsel",
128              "mips64el",
129              "nacl_ia32",
130              "nacl_x64",
131              "x87",
132              "arm64"]
133
134
135def BuildOptions():
136  result = optparse.OptionParser()
137  result.add_option("--arch",
138                    help=("The architecture to run tests for, "
139                          "'auto' or 'native' for auto-detect"),
140                    default="ia32,x64,arm")
141  result.add_option("--arch-and-mode",
142                    help="Architecture and mode in the format 'arch.mode'",
143                    default=None)
144  result.add_option("--asan",
145                    help="Regard test expectations for ASAN",
146                    default=False, action="store_true")
147  result.add_option("--buildbot",
148                    help="Adapt to path structure used on buildbots",
149                    default=False, action="store_true")
150  result.add_option("--cat", help="Print the source of the tests",
151                    default=False, action="store_true")
152  result.add_option("--flaky-tests",
153                    help="Regard tests marked as flaky (run|skip|dontcare)",
154                    default="dontcare")
155  result.add_option("--slow-tests",
156                    help="Regard slow tests (run|skip|dontcare)",
157                    default="dontcare")
158  result.add_option("--pass-fail-tests",
159                    help="Regard pass|fail tests (run|skip|dontcare)",
160                    default="dontcare")
161  result.add_option("--gc-stress",
162                    help="Switch on GC stress mode",
163                    default=False, action="store_true")
164  result.add_option("--command-prefix",
165                    help="Prepended to each shell command used to run a test",
166                    default="")
167  result.add_option("--download-data", help="Download missing test suite data",
168                    default=False, action="store_true")
169  result.add_option("--extra-flags",
170                    help="Additional flags to pass to each test command",
171                    default="")
172  result.add_option("--isolates", help="Whether to test isolates",
173                    default=False, action="store_true")
174  result.add_option("-j", help="The number of parallel tasks to run",
175                    default=0, type="int")
176  result.add_option("-m", "--mode",
177                    help="The test modes in which to run (comma-separated)",
178                    default="release,debug")
179  result.add_option("--no-i18n", "--noi18n",
180                    help="Skip internationalization tests",
181                    default=False, action="store_true")
182  result.add_option("--no-network", "--nonetwork",
183                    help="Don't distribute tests on the network",
184                    default=(utils.GuessOS() != "linux"),
185                    dest="no_network", action="store_true")
186  result.add_option("--no-presubmit", "--nopresubmit",
187                    help='Skip presubmit checks',
188                    default=False, dest="no_presubmit", action="store_true")
189  result.add_option("--no-snap", "--nosnap",
190                    help='Test a build compiled without snapshot.',
191                    default=False, dest="no_snap", action="store_true")
192  result.add_option("--no-sorting", "--nosorting",
193                    help="Don't sort tests according to duration of last run.",
194                    default=False, dest="no_sorting", action="store_true")
195  result.add_option("--no-stress", "--nostress",
196                    help="Don't run crankshaft --always-opt --stress-op test",
197                    default=False, dest="no_stress", action="store_true")
198  result.add_option("--no-variants", "--novariants",
199                    help="Don't run any testing variants",
200                    default=False, dest="no_variants", action="store_true")
201  result.add_option("--variants",
202                    help="Comma-separated list of testing variants")
203  result.add_option("--outdir", help="Base directory with compile output",
204                    default="out")
205  result.add_option("--predictable",
206                    help="Compare output of several reruns of each test",
207                    default=False, action="store_true")
208  result.add_option("-p", "--progress",
209                    help=("The style of progress indicator"
210                          " (verbose, dots, color, mono)"),
211                    choices=progress.PROGRESS_INDICATORS.keys(), default="mono")
212  result.add_option("--quickcheck", default=False, action="store_true",
213                    help=("Quick check mode (skip slow/flaky tests)"))
214  result.add_option("--report", help="Print a summary of the tests to be run",
215                    default=False, action="store_true")
216  result.add_option("--json-test-results",
217                    help="Path to a file for storing json results.")
218  result.add_option("--rerun-failures-count",
219                    help=("Number of times to rerun each failing test case. "
220                          "Very slow tests will be rerun only once."),
221                    default=0, type="int")
222  result.add_option("--rerun-failures-max",
223                    help="Maximum number of failing test cases to rerun.",
224                    default=100, type="int")
225  result.add_option("--shard-count",
226                    help="Split testsuites into this number of shards",
227                    default=1, type="int")
228  result.add_option("--shard-run",
229                    help="Run this shard from the split up tests.",
230                    default=1, type="int")
231  result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="")
232  result.add_option("--shell-dir", help="Directory containing executables",
233                    default="")
234  result.add_option("--dont-skip-slow-simulator-tests",
235                    help="Don't skip more slow tests when using a simulator.",
236                    default=False, action="store_true",
237                    dest="dont_skip_simulator_slow_tests")
238  result.add_option("--stress-only",
239                    help="Only run tests with --always-opt --stress-opt",
240                    default=False, action="store_true")
241  result.add_option("--time", help="Print timing information after running",
242                    default=False, action="store_true")
243  result.add_option("-t", "--timeout", help="Timeout in seconds",
244                    default= -1, type="int")
245  result.add_option("--tsan",
246                    help="Regard test expectations for TSAN",
247                    default=False, action="store_true")
248  result.add_option("-v", "--verbose", help="Verbose output",
249                    default=False, action="store_true")
250  result.add_option("--valgrind", help="Run tests through valgrind",
251                    default=False, action="store_true")
252  result.add_option("--warn-unused", help="Report unused rules",
253                    default=False, action="store_true")
254  result.add_option("--junitout", help="File name of the JUnit output")
255  result.add_option("--junittestsuite",
256                    help="The testsuite name in the JUnit output file",
257                    default="v8tests")
258  result.add_option("--random-seed", default=0, dest="random_seed",
259                    help="Default seed for initializing random generator")
260  return result
261
262
263def ProcessOptions(options):
264  global VARIANT_FLAGS
265  global VARIANTS
266
267  # Architecture and mode related stuff.
268  if options.arch_and_mode:
269    options.arch_and_mode = [arch_and_mode.split(".")
270        for arch_and_mode in options.arch_and_mode.split(",")]
271    options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode])
272    options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode])
273  options.mode = options.mode.split(",")
274  for mode in options.mode:
275    if not mode.lower() in ["debug", "release", "optdebug"]:
276      print "Unknown mode %s" % mode
277      return False
278  if options.arch in ["auto", "native"]:
279    options.arch = ARCH_GUESS
280  options.arch = options.arch.split(",")
281  for arch in options.arch:
282    if not arch in SUPPORTED_ARCHS:
283      print "Unknown architecture %s" % arch
284      return False
285
286  # Store the final configuration in arch_and_mode list. Don't overwrite
287  # predefined arch_and_mode since it is more expressive than arch and mode.
288  if not options.arch_and_mode:
289    options.arch_and_mode = itertools.product(options.arch, options.mode)
290
291  # Special processing of other options, sorted alphabetically.
292
293  if options.buildbot:
294    # Buildbots run presubmit tests as a separate step.
295    options.no_presubmit = True
296    options.no_network = True
297  if options.command_prefix:
298    print("Specifying --command-prefix disables network distribution, "
299          "running tests locally.")
300    options.no_network = True
301  options.command_prefix = shlex.split(options.command_prefix)
302  options.extra_flags = shlex.split(options.extra_flags)
303
304  if options.gc_stress:
305    options.extra_flags += GC_STRESS_FLAGS
306
307  if options.asan:
308    options.extra_flags.append("--invoke-weak-callbacks")
309
310  if options.tsan:
311    VARIANTS = ["default"]
312
313  if options.j == 0:
314    options.j = multiprocessing.cpu_count()
315
316  while options.random_seed == 0:
317    options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647)
318
319  def excl(*args):
320    """Returns true if zero or one of multiple arguments are true."""
321    return reduce(lambda x, y: x + y, args) <= 1
322
323  if not excl(options.no_stress, options.stress_only, options.no_variants,
324              bool(options.variants)):
325    print("Use only one of --no-stress, --stress-only, --no-variants, "
326          "or --variants.")
327    return False
328  if options.quickcheck:
329    VARIANTS = ["default", "stress"]
330    options.flaky_tests = "skip"
331    options.slow_tests = "skip"
332    options.pass_fail_tests = "skip"
333  if options.no_stress:
334    VARIANTS = ["default", "nocrankshaft"]
335  if options.no_variants:
336    VARIANTS = ["default"]
337  if options.stress_only:
338    VARIANTS = ["stress"]
339  if options.variants:
340    VARIANTS = options.variants.split(",")
341    if not set(VARIANTS).issubset(VARIANT_FLAGS.keys()):
342      print "All variants must be in %s" % str(VARIANT_FLAGS.keys())
343      return False
344  if options.predictable:
345    VARIANTS = ["default"]
346    options.extra_flags.append("--predictable")
347    options.extra_flags.append("--verify_predictable")
348    options.extra_flags.append("--no-inline-new")
349
350  if not options.shell_dir:
351    if options.shell:
352      print "Warning: --shell is deprecated, use --shell-dir instead."
353      options.shell_dir = os.path.dirname(options.shell)
354  if options.valgrind:
355    run_valgrind = os.path.join("tools", "run-valgrind.py")
356    # This is OK for distributed running, so we don't need to set no_network.
357    options.command_prefix = (["python", "-u", run_valgrind] +
358                              options.command_prefix)
359  def CheckTestMode(name, option):
360    if not option in ["run", "skip", "dontcare"]:
361      print "Unknown %s mode %s" % (name, option)
362      return False
363    return True
364  if not CheckTestMode("flaky test", options.flaky_tests):
365    return False
366  if not CheckTestMode("slow test", options.slow_tests):
367    return False
368  if not CheckTestMode("pass|fail test", options.pass_fail_tests):
369    return False
370  if not options.no_i18n:
371    DEFAULT_TESTS.append("intl")
372  return True
373
374
375def ShardTests(tests, shard_count, shard_run):
376  if shard_count < 2:
377    return tests
378  if shard_run < 1 or shard_run > shard_count:
379    print "shard-run not a valid number, should be in [1:shard-count]"
380    print "defaulting back to running all tests"
381    return tests
382  count = 0
383  shard = []
384  for test in tests:
385    if count % shard_count == shard_run - 1:
386      shard.append(test)
387    count += 1
388  return shard
389
390
391def Main():
392  parser = BuildOptions()
393  (options, args) = parser.parse_args()
394  if not ProcessOptions(options):
395    parser.print_help()
396    return 1
397
398  exit_code = 0
399  workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), ".."))
400  if not options.no_presubmit:
401    print ">>> running presubmit tests"
402    exit_code = subprocess.call(
403        [sys.executable, join(workspace, "tools", "presubmit.py")])
404
405  suite_paths = utils.GetSuitePaths(join(workspace, "test"))
406
407  # Expand arguments with grouped tests. The args should reflect the list of
408  # suites as otherwise filters would break.
409  def ExpandTestGroups(name):
410    if name in TEST_MAP:
411      return [suite for suite in TEST_MAP[arg]]
412    else:
413      return [name]
414  args = reduce(lambda x, y: x + y,
415         [ExpandTestGroups(arg) for arg in args],
416         [])
417
418  if len(args) == 0:
419    suite_paths = [ s for s in DEFAULT_TESTS if s in suite_paths ]
420  else:
421    args_suites = OrderedDict() # Used as set
422    for arg in args:
423      args_suites[arg.split(os.path.sep)[0]] = True
424    suite_paths = [ s for s in args_suites if s in suite_paths ]
425
426  suites = []
427  for root in suite_paths:
428    suite = testsuite.TestSuite.LoadTestSuite(
429        os.path.join(workspace, "test", root))
430    if suite:
431      suites.append(suite)
432
433  if options.download_data:
434    for s in suites:
435      s.DownloadData()
436
437  for (arch, mode) in options.arch_and_mode:
438    try:
439      code = Execute(arch, mode, args, options, suites, workspace)
440    except KeyboardInterrupt:
441      return 2
442    exit_code = exit_code or code
443  return exit_code
444
445
446def Execute(arch, mode, args, options, suites, workspace):
447  print(">>> Running tests for %s.%s" % (arch, mode))
448
449  shell_dir = options.shell_dir
450  if not shell_dir:
451    if options.buildbot:
452      shell_dir = os.path.join(workspace, options.outdir, mode)
453      mode = mode.lower()
454    else:
455      shell_dir = os.path.join(workspace, options.outdir,
456                               "%s.%s" % (arch, mode))
457  shell_dir = os.path.relpath(shell_dir)
458
459  if mode == "optdebug":
460    mode = "debug"  # "optdebug" is just an alias.
461
462  # Populate context object.
463  mode_flags = MODE_FLAGS[mode]
464  timeout = options.timeout
465  if timeout == -1:
466    # Simulators are slow, therefore allow a longer default timeout.
467    if arch in SLOW_ARCHS:
468      timeout = 2 * TIMEOUT_DEFAULT;
469    else:
470      timeout = TIMEOUT_DEFAULT;
471
472  timeout *= TIMEOUT_SCALEFACTOR[mode]
473
474  if options.predictable:
475    # Predictable mode is slower.
476    timeout *= 2
477
478  ctx = context.Context(arch, mode, shell_dir,
479                        mode_flags, options.verbose,
480                        timeout, options.isolates,
481                        options.command_prefix,
482                        options.extra_flags,
483                        options.no_i18n,
484                        options.random_seed,
485                        options.no_sorting,
486                        options.rerun_failures_count,
487                        options.rerun_failures_max,
488                        options.predictable)
489
490  # TODO(all): Combine "simulator" and "simulator_run".
491  simulator_run = not options.dont_skip_simulator_slow_tests and \
492      arch in ['arm64', 'arm', 'mips'] and ARCH_GUESS and arch != ARCH_GUESS
493  # Find available test suites and read test cases from them.
494  variables = {
495    "arch": arch,
496    "asan": options.asan,
497    "deopt_fuzzer": False,
498    "gc_stress": options.gc_stress,
499    "isolates": options.isolates,
500    "mode": mode,
501    "no_i18n": options.no_i18n,
502    "no_snap": options.no_snap,
503    "simulator_run": simulator_run,
504    "simulator": utils.UseSimulator(arch),
505    "system": utils.GuessOS(),
506    "tsan": options.tsan,
507  }
508  all_tests = []
509  num_tests = 0
510  test_id = 0
511  for s in suites:
512    s.ReadStatusFile(variables)
513    s.ReadTestCases(ctx)
514    if len(args) > 0:
515      s.FilterTestCasesByArgs(args)
516    all_tests += s.tests
517    s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests,
518                              options.slow_tests, options.pass_fail_tests)
519    if options.cat:
520      verbose.PrintTestSource(s.tests)
521      continue
522    variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS]
523    s.tests = [ t.CopyAddingFlags(v)
524                for t in s.tests
525                for v in s.VariantFlags(t, variant_flags) ]
526    s.tests = ShardTests(s.tests, options.shard_count, options.shard_run)
527    num_tests += len(s.tests)
528    for t in s.tests:
529      t.id = test_id
530      test_id += 1
531
532  if options.cat:
533    return 0  # We're done here.
534
535  if options.report:
536    verbose.PrintReport(all_tests)
537
538  if num_tests == 0:
539    print "No tests to run."
540    return 0
541
542  # Run the tests, either locally or distributed on the network.
543  start_time = time.time()
544  progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
545  if options.junitout:
546    progress_indicator = progress.JUnitTestProgressIndicator(
547        progress_indicator, options.junitout, options.junittestsuite)
548  if options.json_test_results:
549    progress_indicator = progress.JsonTestProgressIndicator(
550        progress_indicator, options.json_test_results, arch, mode)
551
552  run_networked = not options.no_network
553  if not run_networked:
554    print("Network distribution disabled, running tests locally.")
555  elif utils.GuessOS() != "linux":
556    print("Network distribution is only supported on Linux, sorry!")
557    run_networked = False
558  peers = []
559  if run_networked:
560    peers = network_execution.GetPeers()
561    if not peers:
562      print("No connection to distribution server; running tests locally.")
563      run_networked = False
564    elif len(peers) == 1:
565      print("No other peers on the network; running tests locally.")
566      run_networked = False
567    elif num_tests <= 100:
568      print("Less than 100 tests, running them locally.")
569      run_networked = False
570
571  if run_networked:
572    runner = network_execution.NetworkedRunner(suites, progress_indicator,
573                                               ctx, peers, workspace)
574  else:
575    runner = execution.Runner(suites, progress_indicator, ctx)
576
577  exit_code = runner.Run(options.j)
578  overall_duration = time.time() - start_time
579
580  if options.time:
581    verbose.PrintTestDurations(suites, overall_duration)
582  return exit_code
583
584
585if __name__ == "__main__":
586  sys.exit(Main())
587