1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31import json
32import math
33import multiprocessing
34import optparse
35import os
36from os.path import join
37import random
38import shlex
39import subprocess
40import sys
41import time
42
43from testrunner.local import execution
44from testrunner.local import progress
45from testrunner.local import testsuite
46from testrunner.local import utils
47from testrunner.local import verbose
48from testrunner.objects import context
49
50
51ARCH_GUESS = utils.DefaultArch()
52DEFAULT_TESTS = ["mjsunit", "webkit"]
53TIMEOUT_DEFAULT = 60
54TIMEOUT_SCALEFACTOR = {"debug"   : 4,
55                       "release" : 1 }
56
57MODE_FLAGS = {
58    "debug"   : ["--nobreak-on-abort", "--nodead-code-elimination",
59                 "--nofold-constants", "--enable-slow-asserts",
60                 "--debug-code", "--verify-heap",
61                 "--noparallel-recompilation"],
62    "release" : ["--nobreak-on-abort", "--nodead-code-elimination",
63                 "--nofold-constants", "--noparallel-recompilation"]}
64
65SUPPORTED_ARCHS = ["android_arm",
66                   "android_ia32",
67                   "arm",
68                   "ia32",
69                   "mipsel",
70                   "nacl_ia32",
71                   "nacl_x64",
72                   "x64"]
73# Double the timeout for these:
74SLOW_ARCHS = ["android_arm",
75              "android_ia32",
76              "arm",
77              "mipsel",
78              "nacl_ia32",
79              "nacl_x64"]
80MAX_DEOPT = 1000000000
81DISTRIBUTION_MODES = ["smooth", "random"]
82
83
84class RandomDistribution:
85  def __init__(self, seed=None):
86    seed = seed or random.randint(1, sys.maxint)
87    print "Using random distribution with seed %d" % seed
88    self._random = random.Random(seed)
89
90  def Distribute(self, n, m):
91    if n > m:
92      n = m
93    return self._random.sample(xrange(1, m + 1), n)
94
95
96class SmoothDistribution:
97  """Distribute n numbers into the interval [1:m].
98  F1: Factor of the first derivation of the distribution function.
99  F2: Factor of the second derivation of the distribution function.
100  With F1 and F2 set to 0, the distribution will be equal.
101  """
102  def __init__(self, factor1=2.0, factor2=0.2):
103    self._factor1 = factor1
104    self._factor2 = factor2
105
106  def Distribute(self, n, m):
107    if n > m:
108      n = m
109    if n <= 1:
110      return [ 1 ]
111
112    result = []
113    x = 0.0
114    dx = 1.0
115    ddx = self._factor1
116    dddx = self._factor2
117    for i in range(0, n):
118      result += [ x ]
119      x += dx
120      dx += ddx
121      ddx += dddx
122
123    # Project the distribution into the interval [0:M].
124    result = [ x * m / result[-1] for x in result ]
125
126    # Equalize by n. The closer n is to m, the more equal will be the
127    # distribution.
128    for (i, x) in enumerate(result):
129      # The value of x if it was equally distributed.
130      equal_x = i / float(n - 1) * float(m - 1) + 1
131
132      # Difference factor between actual and equal distribution.
133      diff = 1 - (x / equal_x)
134
135      # Equalize x dependent on the number of values to distribute.
136      result[i] = int(x + (i + 1) * diff)
137    return result
138
139
140def Distribution(options):
141  if options.distribution_mode == "random":
142    return RandomDistribution(options.seed)
143  if options.distribution_mode == "smooth":
144    return SmoothDistribution(options.distribution_factor1,
145                              options.distribution_factor2)
146
147
148def BuildOptions():
149  result = optparse.OptionParser()
150  result.add_option("--arch",
151                    help=("The architecture to run tests for, "
152                          "'auto' or 'native' for auto-detect"),
153                    default="ia32,x64,arm")
154  result.add_option("--arch-and-mode",
155                    help="Architecture and mode in the format 'arch.mode'",
156                    default=None)
157  result.add_option("--buildbot",
158                    help="Adapt to path structure used on buildbots",
159                    default=False, action="store_true")
160  result.add_option("--command-prefix",
161                    help="Prepended to each shell command used to run a test",
162                    default="")
163  result.add_option("--coverage", help=("Exponential test coverage "
164                    "(range 0.0, 1.0) -- 0.0: one test, 1.0 all tests (slow)"),
165                    default=0.4, type="float")
166  result.add_option("--coverage-lift", help=("Lifts test coverage for tests "
167                    "with a small number of deopt points (range 0, inf)"),
168                    default=20, type="int")
169  result.add_option("--download-data", help="Download missing test suite data",
170                    default=False, action="store_true")
171  result.add_option("--distribution-factor1", help=("Factor of the first "
172                    "derivation of the distribution function"), default=2.0,
173                    type="float")
174  result.add_option("--distribution-factor2", help=("Factor of the second "
175                    "derivation of the distribution function"), default=0.7,
176                    type="float")
177  result.add_option("--distribution-mode", help=("How to select deopt points "
178                    "for a given test (smooth|random)"),
179                    default="smooth")
180  result.add_option("--dump-results-file", help=("Dump maximum number of "
181                    "deopt points per test to a file"))
182  result.add_option("--extra-flags",
183                    help="Additional flags to pass to each test command",
184                    default="")
185  result.add_option("--isolates", help="Whether to test isolates",
186                    default=False, action="store_true")
187  result.add_option("-j", help="The number of parallel tasks to run",
188                    default=0, type="int")
189  result.add_option("-m", "--mode",
190                    help="The test modes in which to run (comma-separated)",
191                    default="release,debug")
192  result.add_option("--outdir", help="Base directory with compile output",
193                    default="out")
194  result.add_option("-p", "--progress",
195                    help=("The style of progress indicator"
196                          " (verbose, dots, color, mono)"),
197                    choices=progress.PROGRESS_INDICATORS.keys(),
198                    default="mono")
199  result.add_option("--shard-count",
200                    help="Split testsuites into this number of shards",
201                    default=1, type="int")
202  result.add_option("--shard-run",
203                    help="Run this shard from the split up tests.",
204                    default=1, type="int")
205  result.add_option("--shell-dir", help="Directory containing executables",
206                    default="")
207  result.add_option("--seed", help="The seed for the random distribution",
208                    type="int")
209  result.add_option("-t", "--timeout", help="Timeout in seconds",
210                    default= -1, type="int")
211  result.add_option("-v", "--verbose", help="Verbose output",
212                    default=False, action="store_true")
213  return result
214
215
216def ProcessOptions(options):
217  global VARIANT_FLAGS
218
219  # Architecture and mode related stuff.
220  if options.arch_and_mode:
221    tokens = options.arch_and_mode.split(".")
222    options.arch = tokens[0]
223    options.mode = tokens[1]
224  options.mode = options.mode.split(",")
225  for mode in options.mode:
226    if not mode.lower() in ["debug", "release"]:
227      print "Unknown mode %s" % mode
228      return False
229  if options.arch in ["auto", "native"]:
230    options.arch = ARCH_GUESS
231  options.arch = options.arch.split(",")
232  for arch in options.arch:
233    if not arch in SUPPORTED_ARCHS:
234      print "Unknown architecture %s" % arch
235      return False
236
237  # Special processing of other options, sorted alphabetically.
238  options.command_prefix = shlex.split(options.command_prefix)
239  options.extra_flags = shlex.split(options.extra_flags)
240  if options.j == 0:
241    options.j = multiprocessing.cpu_count()
242  if not options.distribution_mode in DISTRIBUTION_MODES:
243    print "Unknown distribution mode %s" % options.distribution_mode
244    return False
245  if options.distribution_factor1 < 0.0:
246    print ("Distribution factor1 %s is out of range. Defaulting to 0.0"
247        % options.distribution_factor1)
248    options.distribution_factor1 = 0.0
249  if options.distribution_factor2 < 0.0:
250    print ("Distribution factor2 %s is out of range. Defaulting to 0.0"
251        % options.distribution_factor2)
252    options.distribution_factor2 = 0.0
253  if options.coverage < 0.0 or options.coverage > 1.0:
254    print ("Coverage %s is out of range. Defaulting to 0.4"
255        % options.coverage)
256    options.coverage = 0.4
257  if options.coverage_lift < 0:
258    print ("Coverage lift %s is out of range. Defaulting to 0"
259        % options.coverage_lift)
260    options.coverage_lift = 0
261  return True
262
263
264def ShardTests(tests, shard_count, shard_run):
265  if shard_count < 2:
266    return tests
267  if shard_run < 1 or shard_run > shard_count:
268    print "shard-run not a valid number, should be in [1:shard-count]"
269    print "defaulting back to running all tests"
270    return tests
271  count = 0
272  shard = []
273  for test in tests:
274    if count % shard_count == shard_run - 1:
275      shard.append(test)
276    count += 1
277  return shard
278
279
280def Main():
281  parser = BuildOptions()
282  (options, args) = parser.parse_args()
283  if not ProcessOptions(options):
284    parser.print_help()
285    return 1
286
287  exit_code = 0
288  workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), ".."))
289
290  suite_paths = utils.GetSuitePaths(join(workspace, "test"))
291
292  if len(args) == 0:
293    suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
294  else:
295    args_suites = set()
296    for arg in args:
297      suite = arg.split(os.path.sep)[0]
298      if not suite in args_suites:
299        args_suites.add(suite)
300    suite_paths = [ s for s in suite_paths if s in args_suites ]
301
302  suites = []
303  for root in suite_paths:
304    suite = testsuite.TestSuite.LoadTestSuite(
305        os.path.join(workspace, "test", root))
306    if suite:
307      suites.append(suite)
308
309  if options.download_data:
310    for s in suites:
311      s.DownloadData()
312
313  for mode in options.mode:
314    for arch in options.arch:
315      code = Execute(arch, mode, args, options, suites, workspace)
316      exit_code = exit_code or code
317  return exit_code
318
319
320def CalculateNTests(m, options):
321  """Calculates the number of tests from m deopt points with exponential
322  coverage.
323  The coverage is expected to be between 0.0 and 1.0.
324  The 'coverage lift' lifts the coverage for tests with smaller m values.
325  """
326  c = float(options.coverage)
327  l = float(options.coverage_lift)
328  return int(math.pow(m, (m * c + l) / (m + l)))
329
330
331def Execute(arch, mode, args, options, suites, workspace):
332  print(">>> Running tests for %s.%s" % (arch, mode))
333
334  dist = Distribution(options)
335
336  shell_dir = options.shell_dir
337  if not shell_dir:
338    if options.buildbot:
339      shell_dir = os.path.join(workspace, options.outdir, mode)
340      mode = mode.lower()
341    else:
342      shell_dir = os.path.join(workspace, options.outdir,
343                               "%s.%s" % (arch, mode))
344  shell_dir = os.path.relpath(shell_dir)
345
346  # Populate context object.
347  mode_flags = MODE_FLAGS[mode]
348  timeout = options.timeout
349  if timeout == -1:
350    # Simulators are slow, therefore allow a longer default timeout.
351    if arch in SLOW_ARCHS:
352      timeout = 2 * TIMEOUT_DEFAULT;
353    else:
354      timeout = TIMEOUT_DEFAULT;
355
356  timeout *= TIMEOUT_SCALEFACTOR[mode]
357  ctx = context.Context(arch, mode, shell_dir,
358                        mode_flags, options.verbose,
359                        timeout, options.isolates,
360                        options.command_prefix,
361                        options.extra_flags)
362
363  # Find available test suites and read test cases from them.
364  variables = {
365    "mode": mode,
366    "arch": arch,
367    "system": utils.GuessOS(),
368    "isolates": options.isolates,
369    "deopt_fuzzer": True,
370  }
371  all_tests = []
372  num_tests = 0
373  test_id = 0
374
375  # Remember test case prototypes for the fuzzing phase.
376  test_backup = dict((s, []) for s in suites)
377
378  for s in suites:
379    s.ReadStatusFile(variables)
380    s.ReadTestCases(ctx)
381    if len(args) > 0:
382      s.FilterTestCasesByArgs(args)
383    all_tests += s.tests
384    s.FilterTestCasesByStatus(False)
385    test_backup[s] = s.tests
386    analysis_flags = ["--deopt-every-n-times", "%d" % MAX_DEOPT,
387                      "--print-deopt-stress"]
388    s.tests = [ t.CopyAddingFlags(analysis_flags) for t in s.tests ]
389    num_tests += len(s.tests)
390    for t in s.tests:
391      t.id = test_id
392      test_id += 1
393
394  if num_tests == 0:
395    print "No tests to run."
396    return 0
397
398  try:
399    print(">>> Collection phase")
400    progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
401    runner = execution.Runner(suites, progress_indicator, ctx)
402
403    exit_code = runner.Run(options.j)
404    if runner.terminate:
405      return exit_code
406
407  except KeyboardInterrupt:
408    return 1
409
410  print(">>> Analysis phase")
411  num_tests = 0
412  test_id = 0
413  for s in suites:
414    test_results = {}
415    for t in s.tests:
416      for line in t.output.stdout.splitlines():
417        if line.startswith("=== Stress deopt counter: "):
418          test_results[t.path] = MAX_DEOPT - int(line.split(" ")[-1])
419    for t in s.tests:
420      if t.path not in test_results:
421        print "Missing results for %s" % t.path
422    if options.dump_results_file:
423      results_dict = dict((t.path, n) for (t, n) in test_results.iteritems())
424      with file("%s.%d.txt" % (dump_results_file, time.time()), "w") as f:
425        f.write(json.dumps(results_dict))
426
427    # Reset tests and redistribute the prototypes from the collection phase.
428    s.tests = []
429    if options.verbose:
430      print "Test distributions:"
431    for t in test_backup[s]:
432      max_deopt = test_results.get(t.path, 0)
433      if max_deopt == 0:
434        continue
435      n_deopt = CalculateNTests(max_deopt, options)
436      distribution = dist.Distribute(n_deopt, max_deopt)
437      if options.verbose:
438        print "%s %s" % (t.path, distribution)
439      for i in distribution:
440        fuzzing_flags = ["--deopt-every-n-times", "%d" % i]
441        s.tests.append(t.CopyAddingFlags(fuzzing_flags))
442    num_tests += len(s.tests)
443    for t in s.tests:
444      t.id = test_id
445      test_id += 1
446
447  if num_tests == 0:
448    print "No tests to run."
449    return 0
450
451  try:
452    print(">>> Deopt fuzzing phase (%d test cases)" % num_tests)
453    progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
454    runner = execution.Runner(suites, progress_indicator, ctx)
455
456    exit_code = runner.Run(options.j)
457    if runner.terminate:
458      return exit_code
459
460  except KeyboardInterrupt:
461    return 1
462
463  return exit_code
464
465
466if __name__ == "__main__":
467  sys.exit(Main())
468