1#!/usr/bin/env python
2#
3# Copyright 2008 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31import imp
32import optparse
33import os
34from os.path import join, dirname, abspath, basename, isdir, exists
35import platform
36import re
37import signal
38import subprocess
39import sys
40import tempfile
41import time
42import threading
43import utils
44from Queue import Queue, Empty
45
46
47VERBOSE = False
48
49
50# ---------------------------------------------
51# --- P r o g r e s s   I n d i c a t o r s ---
52# ---------------------------------------------
53
54
55class ProgressIndicator(object):
56
57  def __init__(self, cases):
58    self.cases = cases
59    self.queue = Queue(len(cases))
60    for case in cases:
61      self.queue.put_nowait(case)
62    self.succeeded = 0
63    self.remaining = len(cases)
64    self.total = len(cases)
65    self.failed = [ ]
66    self.crashed = 0
67    self.terminate = False
68    self.lock = threading.Lock()
69
70  def PrintFailureHeader(self, test):
71    if test.IsNegative():
72      negative_marker = '[negative] '
73    else:
74      negative_marker = ''
75    print "=== %(label)s %(negative)s===" % {
76      'label': test.GetLabel(),
77      'negative': negative_marker
78    }
79    print "Path: %s" % "/".join(test.path)
80
81  def Run(self, tasks):
82    self.Starting()
83    threads = []
84    # Spawn N-1 threads and then use this thread as the last one.
85    # That way -j1 avoids threading altogether which is a nice fallback
86    # in case of threading problems.
87    for i in xrange(tasks - 1):
88      thread = threading.Thread(target=self.RunSingle, args=[])
89      threads.append(thread)
90      thread.start()
91    try:
92      self.RunSingle()
93      # Wait for the remaining threads
94      for thread in threads:
95        # Use a timeout so that signals (ctrl-c) will be processed.
96        thread.join(timeout=10000000)
97    except Exception, e:
98      # If there's an exception we schedule an interruption for any
99      # remaining threads.
100      self.terminate = True
101      # ...and then reraise the exception to bail out
102      raise
103    self.Done()
104    return not self.failed
105
106  def RunSingle(self):
107    while not self.terminate:
108      try:
109        test = self.queue.get_nowait()
110      except Empty:
111        return
112      case = test.case
113      self.lock.acquire()
114      self.AboutToRun(case)
115      self.lock.release()
116      try:
117        start = time.time()
118        output = case.Run()
119        case.duration = (time.time() - start)
120      except IOError, e:
121        assert self.terminate
122        return
123      if self.terminate:
124        return
125      self.lock.acquire()
126      if output.UnexpectedOutput():
127        self.failed.append(output)
128        if output.HasCrashed():
129          self.crashed += 1
130      else:
131        self.succeeded += 1
132      self.remaining -= 1
133      self.HasRun(output)
134      self.lock.release()
135
136
137def EscapeCommand(command):
138  parts = []
139  for part in command:
140    if ' ' in part:
141      # Escape spaces.  We may need to escape more characters for this
142      # to work properly.
143      parts.append('"%s"' % part)
144    else:
145      parts.append(part)
146  return " ".join(parts)
147
148
149class SimpleProgressIndicator(ProgressIndicator):
150
151  def Starting(self):
152    print 'Running %i tests' % len(self.cases)
153
154  def Done(self):
155    print
156    for failed in self.failed:
157      self.PrintFailureHeader(failed.test)
158      if failed.output.stderr:
159        print "--- stderr ---"
160        print failed.output.stderr.strip()
161      if failed.output.stdout:
162        print "--- stdout ---"
163        print failed.output.stdout.strip()
164      print "Command: %s" % EscapeCommand(failed.command)
165      if failed.HasCrashed():
166        print "--- CRASHED ---"
167      if failed.HasTimedOut():
168        print "--- TIMEOUT ---"
169    if len(self.failed) == 0:
170      print "==="
171      print "=== All tests succeeded"
172      print "==="
173    else:
174      print
175      print "==="
176      print "=== %i tests failed" % len(self.failed)
177      if self.crashed > 0:
178        print "=== %i tests CRASHED" % self.crashed
179      print "==="
180
181
182class VerboseProgressIndicator(SimpleProgressIndicator):
183
184  def AboutToRun(self, case):
185    print 'Starting %s...' % case.GetLabel()
186    sys.stdout.flush()
187
188  def HasRun(self, output):
189    if output.UnexpectedOutput():
190      if output.HasCrashed():
191        outcome = 'CRASH'
192      else:
193        outcome = 'FAIL'
194    else:
195      outcome = 'pass'
196    print 'Done running %s: %s' % (output.test.GetLabel(), outcome)
197
198
199class DotsProgressIndicator(SimpleProgressIndicator):
200
201  def AboutToRun(self, case):
202    pass
203
204  def HasRun(self, output):
205    total = self.succeeded + len(self.failed)
206    if (total > 1) and (total % 50 == 1):
207      sys.stdout.write('\n')
208    if output.UnexpectedOutput():
209      if output.HasCrashed():
210        sys.stdout.write('C')
211        sys.stdout.flush()
212      elif output.HasTimedOut():
213        sys.stdout.write('T')
214        sys.stdout.flush()
215      else:
216        sys.stdout.write('F')
217        sys.stdout.flush()
218    else:
219      sys.stdout.write('.')
220      sys.stdout.flush()
221
222
223class CompactProgressIndicator(ProgressIndicator):
224
225  def __init__(self, cases, templates):
226    super(CompactProgressIndicator, self).__init__(cases)
227    self.templates = templates
228    self.last_status_length = 0
229    self.start_time = time.time()
230
231  def Starting(self):
232    pass
233
234  def Done(self):
235    self.PrintProgress('Done')
236
237  def AboutToRun(self, case):
238    self.PrintProgress(case.GetLabel())
239
240  def HasRun(self, output):
241    if output.UnexpectedOutput():
242      self.ClearLine(self.last_status_length)
243      self.PrintFailureHeader(output.test)
244      stdout = output.output.stdout.strip()
245      if len(stdout):
246        print self.templates['stdout'] % stdout
247      stderr = output.output.stderr.strip()
248      if len(stderr):
249        print self.templates['stderr'] % stderr
250      print "Command: %s" % EscapeCommand(output.command)
251      if output.HasCrashed():
252        print "--- CRASHED ---"
253      if output.HasTimedOut():
254        print "--- TIMEOUT ---"
255
256  def Truncate(self, str, length):
257    if length and (len(str) > (length - 3)):
258      return str[:(length-3)] + "..."
259    else:
260      return str
261
262  def PrintProgress(self, name):
263    self.ClearLine(self.last_status_length)
264    elapsed = time.time() - self.start_time
265    status = self.templates['status_line'] % {
266      'passed': self.succeeded,
267      'remaining': (((self.total - self.remaining) * 100) // self.total),
268      'failed': len(self.failed),
269      'test': name,
270      'mins': int(elapsed) / 60,
271      'secs': int(elapsed) % 60
272    }
273    status = self.Truncate(status, 78)
274    self.last_status_length = len(status)
275    print status,
276    sys.stdout.flush()
277
278
279class ColorProgressIndicator(CompactProgressIndicator):
280
281  def __init__(self, cases):
282    templates = {
283      'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s",
284      'stdout': "\033[1m%s\033[0m",
285      'stderr': "\033[31m%s\033[0m",
286    }
287    super(ColorProgressIndicator, self).__init__(cases, templates)
288
289  def ClearLine(self, last_line_length):
290    print "\033[1K\r",
291
292
293class MonochromeProgressIndicator(CompactProgressIndicator):
294
295  def __init__(self, cases):
296    templates = {
297      'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
298      'stdout': '%s',
299      'stderr': '%s',
300      'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"),
301      'max_length': 78
302    }
303    super(MonochromeProgressIndicator, self).__init__(cases, templates)
304
305  def ClearLine(self, last_line_length):
306    print ("\r" + (" " * last_line_length) + "\r"),
307
308
309PROGRESS_INDICATORS = {
310  'verbose': VerboseProgressIndicator,
311  'dots': DotsProgressIndicator,
312  'color': ColorProgressIndicator,
313  'mono': MonochromeProgressIndicator
314}
315
316
317# -------------------------
318# --- F r a m e w o r k ---
319# -------------------------
320
321
322class CommandOutput(object):
323
324  def __init__(self, exit_code, timed_out, stdout, stderr):
325    self.exit_code = exit_code
326    self.timed_out = timed_out
327    self.stdout = stdout
328    self.stderr = stderr
329    self.failed = None
330
331
332class TestCase(object):
333
334  def __init__(self, context, path, mode):
335    self.path = path
336    self.context = context
337    self.duration = None
338    self.mode = mode
339
340  def IsNegative(self):
341    return False
342
343  def TestsIsolates(self):
344    return False
345
346  def CompareTime(self, other):
347    return cmp(other.duration, self.duration)
348
349  def DidFail(self, output):
350    if output.failed is None:
351      output.failed = self.IsFailureOutput(output)
352    return output.failed
353
354  def IsFailureOutput(self, output):
355    return output.exit_code != 0
356
357  def GetSource(self):
358    return "(no source available)"
359
360  def RunCommand(self, command):
361    full_command = self.context.processor(command)
362    output = Execute(full_command,
363                     self.context,
364                     self.context.GetTimeout(self, self.mode))
365    self.Cleanup()
366    return TestOutput(self,
367                      full_command,
368                      output,
369                      self.context.store_unexpected_output)
370
371  def BeforeRun(self):
372    pass
373
374  def AfterRun(self, result):
375    pass
376
377  def GetCustomFlags(self, mode):
378    return None
379
380  def Run(self):
381    self.BeforeRun()
382    result = "exception"
383    try:
384      result = self.RunCommand(self.GetCommand())
385    finally:
386      self.AfterRun(result)
387    return result
388
389  def Cleanup(self):
390    return
391
392
393class TestOutput(object):
394
395  def __init__(self, test, command, output, store_unexpected_output):
396    self.test = test
397    self.command = command
398    self.output = output
399    self.store_unexpected_output = store_unexpected_output
400
401  def UnexpectedOutput(self):
402    if self.HasCrashed():
403      outcome = CRASH
404    elif self.HasTimedOut():
405      outcome = TIMEOUT
406    elif self.HasFailed():
407      outcome = FAIL
408    else:
409      outcome = PASS
410    return not outcome in self.test.outcomes
411
412  def HasPreciousOutput(self):
413    return self.UnexpectedOutput() and self.store_unexpected_output
414
415  def HasCrashed(self):
416    if utils.IsWindows():
417      return 0x80000000 & self.output.exit_code and not (0x3FFFFF00 & self.output.exit_code)
418    else:
419      # Timed out tests will have exit_code -signal.SIGTERM.
420      if self.output.timed_out:
421        return False
422      return self.output.exit_code < 0 and \
423             self.output.exit_code != -signal.SIGABRT
424
425  def HasTimedOut(self):
426    return self.output.timed_out;
427
428  def HasFailed(self):
429    execution_failed = self.test.DidFail(self.output)
430    if self.test.IsNegative():
431      return not execution_failed
432    else:
433      return execution_failed
434
435
436def KillProcessWithID(pid):
437  if utils.IsWindows():
438    os.popen('taskkill /T /F /PID %d' % pid)
439  else:
440    os.kill(pid, signal.SIGTERM)
441
442
443MAX_SLEEP_TIME = 0.1
444INITIAL_SLEEP_TIME = 0.0001
445SLEEP_TIME_FACTOR = 1.25
446
447SEM_INVALID_VALUE = -1
448SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h
449
450def Win32SetErrorMode(mode):
451  prev_error_mode = SEM_INVALID_VALUE
452  try:
453    import ctypes
454    prev_error_mode = ctypes.windll.kernel32.SetErrorMode(mode);
455  except ImportError:
456    pass
457  return prev_error_mode
458
459def RunProcess(context, timeout, args, **rest):
460  if context.verbose: print "#", " ".join(args)
461  popen_args = args
462  prev_error_mode = SEM_INVALID_VALUE;
463  if utils.IsWindows():
464    popen_args = '"' + subprocess.list2cmdline(args) + '"'
465    if context.suppress_dialogs:
466      # Try to change the error mode to avoid dialogs on fatal errors. Don't
467      # touch any existing error mode flags by merging the existing error mode.
468      # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx.
469      error_mode = SEM_NOGPFAULTERRORBOX;
470      prev_error_mode = Win32SetErrorMode(error_mode);
471      Win32SetErrorMode(error_mode | prev_error_mode);
472  process = subprocess.Popen(
473    shell = utils.IsWindows(),
474    args = popen_args,
475    **rest
476  )
477  if utils.IsWindows() and context.suppress_dialogs and prev_error_mode != SEM_INVALID_VALUE:
478    Win32SetErrorMode(prev_error_mode)
479  # Compute the end time - if the process crosses this limit we
480  # consider it timed out.
481  if timeout is None: end_time = None
482  else: end_time = time.time() + timeout
483  timed_out = False
484  # Repeatedly check the exit code from the process in a
485  # loop and keep track of whether or not it times out.
486  exit_code = None
487  sleep_time = INITIAL_SLEEP_TIME
488  while exit_code is None:
489    if (not end_time is None) and (time.time() >= end_time):
490      # Kill the process and wait for it to exit.
491      KillProcessWithID(process.pid)
492      exit_code = process.wait()
493      timed_out = True
494    else:
495      exit_code = process.poll()
496      time.sleep(sleep_time)
497      sleep_time = sleep_time * SLEEP_TIME_FACTOR
498      if sleep_time > MAX_SLEEP_TIME:
499        sleep_time = MAX_SLEEP_TIME
500  return (process, exit_code, timed_out)
501
502
503def PrintError(str):
504  sys.stderr.write(str)
505  sys.stderr.write('\n')
506
507
508def CheckedUnlink(name):
509  # On Windows, when run with -jN in parallel processes,
510  # OS often fails to unlink the temp file. Not sure why.
511  # Need to retry.
512  # Idea from https://bugs.webkit.org/attachment.cgi?id=75982&action=prettypatch
513  retry_count = 0
514  while retry_count < 30:
515    try:
516      os.unlink(name)
517      return
518    except OSError, e:
519      retry_count += 1;
520      time.sleep(retry_count * 0.1)
521  PrintError("os.unlink() " + str(e))
522
523def Execute(args, context, timeout=None):
524  (fd_out, outname) = tempfile.mkstemp()
525  (fd_err, errname) = tempfile.mkstemp()
526  (process, exit_code, timed_out) = RunProcess(
527    context,
528    timeout,
529    args = args,
530    stdout = fd_out,
531    stderr = fd_err,
532  )
533  os.close(fd_out)
534  os.close(fd_err)
535  output = file(outname).read()
536  errors = file(errname).read()
537  CheckedUnlink(outname)
538  CheckedUnlink(errname)
539  return CommandOutput(exit_code, timed_out, output, errors)
540
541
542def ExecuteNoCapture(args, context, timeout=None):
543  (process, exit_code, timed_out) = RunProcess(
544    context,
545    timeout,
546    args = args,
547  )
548  return CommandOutput(exit_code, False, "", "")
549
550
551def CarCdr(path):
552  if len(path) == 0:
553    return (None, [ ])
554  else:
555    return (path[0], path[1:])
556
557
558class TestConfiguration(object):
559
560  def __init__(self, context, root):
561    self.context = context
562    self.root = root
563
564  def Contains(self, path, file):
565    if len(path) > len(file):
566      return False
567    for i in xrange(len(path)):
568      if not path[i].match(file[i]):
569        return False
570    return True
571
572  def GetTestStatus(self, sections, defs):
573    pass
574
575
576class TestSuite(object):
577
578  def __init__(self, name):
579    self.name = name
580
581  def GetName(self):
582    return self.name
583
584
585# Use this to run several variants of the tests, e.g.:
586# VARIANT_FLAGS = [[], ['--always_compact', '--noflush_code']]
587VARIANT_FLAGS = [[],
588                 ['--stress-opt', '--always-opt'],
589                 ['--nocrankshaft']]
590
591
592class TestRepository(TestSuite):
593
594  def __init__(self, path):
595    normalized_path = abspath(path)
596    super(TestRepository, self).__init__(basename(normalized_path))
597    self.path = normalized_path
598    self.is_loaded = False
599    self.config = None
600
601  def GetConfiguration(self, context):
602    if self.is_loaded:
603      return self.config
604    self.is_loaded = True
605    file = None
606    try:
607      (file, pathname, description) = imp.find_module('testcfg', [ self.path ])
608      module = imp.load_module('testcfg', file, pathname, description)
609      self.config = module.GetConfiguration(context, self.path)
610    finally:
611      if file:
612        file.close()
613    return self.config
614
615  def GetBuildRequirements(self, path, context):
616    return self.GetConfiguration(context).GetBuildRequirements()
617
618  def AddTestsToList(self, result, current_path, path, context, mode):
619    for v in VARIANT_FLAGS:
620      tests = self.GetConfiguration(context).ListTests(current_path, path, mode, v)
621      for t in tests: t.variant_flags = v
622      result += tests
623
624
625  def GetTestStatus(self, context, sections, defs):
626    self.GetConfiguration(context).GetTestStatus(sections, defs)
627
628
629class LiteralTestSuite(TestSuite):
630
631  def __init__(self, tests):
632    super(LiteralTestSuite, self).__init__('root')
633    self.tests = tests
634
635  def GetBuildRequirements(self, path, context):
636    (name, rest) = CarCdr(path)
637    result = [ ]
638    for test in self.tests:
639      if not name or name.match(test.GetName()):
640        result += test.GetBuildRequirements(rest, context)
641    return result
642
643  def ListTests(self, current_path, path, context, mode, variant_flags):
644    (name, rest) = CarCdr(path)
645    result = [ ]
646    for test in self.tests:
647      test_name = test.GetName()
648      if not name or name.match(test_name):
649        full_path = current_path + [test_name]
650        test.AddTestsToList(result, full_path, path, context, mode)
651    return result
652
653  def GetTestStatus(self, context, sections, defs):
654    for test in self.tests:
655      test.GetTestStatus(context, sections, defs)
656
657
658SUFFIX = {
659    'debug'   : '_g',
660    'release' : '' }
661FLAGS = {
662    'debug'   : ['--enable-slow-asserts', '--debug-code', '--verify-heap'],
663    'release' : []}
664TIMEOUT_SCALEFACTOR = {
665    'debug'   : 4,
666    'release' : 1 }
667
668
669class Context(object):
670
671  def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs, store_unexpected_output):
672    self.workspace = workspace
673    self.buildspace = buildspace
674    self.verbose = verbose
675    self.vm_root = vm
676    self.timeout = timeout
677    self.processor = processor
678    self.suppress_dialogs = suppress_dialogs
679    self.store_unexpected_output = store_unexpected_output
680
681  def GetVm(self, mode):
682    name = self.vm_root + SUFFIX[mode]
683    if utils.IsWindows() and not name.endswith('.exe'):
684      name = name + '.exe'
685    return name
686
687  def GetVmCommand(self, testcase, mode):
688    return [self.GetVm(mode)] + self.GetVmFlags(testcase, mode)
689
690  def GetVmFlags(self, testcase, mode):
691    flags = testcase.GetCustomFlags(mode)
692    if flags is None:
693      flags = FLAGS[mode]
694    return testcase.variant_flags + flags
695
696  def GetTimeout(self, testcase, mode):
697    result = self.timeout * TIMEOUT_SCALEFACTOR[mode]
698    if '--stress-opt' in self.GetVmFlags(testcase, mode):
699      return result * 2
700    else:
701      return result
702
703def RunTestCases(cases_to_run, progress, tasks):
704  progress = PROGRESS_INDICATORS[progress](cases_to_run)
705  return progress.Run(tasks)
706
707
708def BuildRequirements(context, requirements, mode, scons_flags):
709  command_line = (['scons', '-Y', context.workspace, 'mode=' + ",".join(mode)]
710                  + requirements
711                  + scons_flags)
712  output = ExecuteNoCapture(command_line, context)
713  return output.exit_code == 0
714
715
716# -------------------------------------------
717# --- T e s t   C o n f i g u r a t i o n ---
718# -------------------------------------------
719
720
721SKIP = 'skip'
722FAIL = 'fail'
723PASS = 'pass'
724OKAY = 'okay'
725TIMEOUT = 'timeout'
726CRASH = 'crash'
727SLOW = 'slow'
728
729
730class Expression(object):
731  pass
732
733
734class Constant(Expression):
735
736  def __init__(self, value):
737    self.value = value
738
739  def Evaluate(self, env, defs):
740    return self.value
741
742
743class Variable(Expression):
744
745  def __init__(self, name):
746    self.name = name
747
748  def GetOutcomes(self, env, defs):
749    if self.name in env: return ListSet([env[self.name]])
750    else: return Nothing()
751
752  def Evaluate(self, env, defs):
753    return env[self.name]
754
755
756class Outcome(Expression):
757
758  def __init__(self, name):
759    self.name = name
760
761  def GetOutcomes(self, env, defs):
762    if self.name in defs:
763      return defs[self.name].GetOutcomes(env, defs)
764    else:
765      return ListSet([self.name])
766
767
768class Set(object):
769  pass
770
771
772class ListSet(Set):
773
774  def __init__(self, elms):
775    self.elms = elms
776
777  def __str__(self):
778    return "ListSet%s" % str(self.elms)
779
780  def Intersect(self, that):
781    if not isinstance(that, ListSet):
782      return that.Intersect(self)
783    return ListSet([ x for x in self.elms if x in that.elms ])
784
785  def Union(self, that):
786    if not isinstance(that, ListSet):
787      return that.Union(self)
788    return ListSet(self.elms + [ x for x in that.elms if x not in self.elms ])
789
790  def IsEmpty(self):
791    return len(self.elms) == 0
792
793
794class Everything(Set):
795
796  def Intersect(self, that):
797    return that
798
799  def Union(self, that):
800    return self
801
802  def IsEmpty(self):
803    return False
804
805
806class Nothing(Set):
807
808  def Intersect(self, that):
809    return self
810
811  def Union(self, that):
812    return that
813
814  def IsEmpty(self):
815    return True
816
817
818class Operation(Expression):
819
820  def __init__(self, left, op, right):
821    self.left = left
822    self.op = op
823    self.right = right
824
825  def Evaluate(self, env, defs):
826    if self.op == '||' or self.op == ',':
827      return self.left.Evaluate(env, defs) or self.right.Evaluate(env, defs)
828    elif self.op == 'if':
829      return False
830    elif self.op == '==':
831      inter = self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs))
832      return not inter.IsEmpty()
833    else:
834      assert self.op == '&&'
835      return self.left.Evaluate(env, defs) and self.right.Evaluate(env, defs)
836
837  def GetOutcomes(self, env, defs):
838    if self.op == '||' or self.op == ',':
839      return self.left.GetOutcomes(env, defs).Union(self.right.GetOutcomes(env, defs))
840    elif self.op == 'if':
841      if self.right.Evaluate(env, defs): return self.left.GetOutcomes(env, defs)
842      else: return Nothing()
843    else:
844      assert self.op == '&&'
845      return self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs))
846
847
848def IsAlpha(str):
849  for char in str:
850    if not (char.isalpha() or char.isdigit() or char == '_'):
851      return False
852  return True
853
854
855class Tokenizer(object):
856  """A simple string tokenizer that chops expressions into variables,
857  parens and operators"""
858
859  def __init__(self, expr):
860    self.index = 0
861    self.expr = expr
862    self.length = len(expr)
863    self.tokens = None
864
865  def Current(self, length = 1):
866    if not self.HasMore(length): return ""
867    return self.expr[self.index:self.index+length]
868
869  def HasMore(self, length = 1):
870    return self.index < self.length + (length - 1)
871
872  def Advance(self, count = 1):
873    self.index = self.index + count
874
875  def AddToken(self, token):
876    self.tokens.append(token)
877
878  def SkipSpaces(self):
879    while self.HasMore() and self.Current().isspace():
880      self.Advance()
881
882  def Tokenize(self):
883    self.tokens = [ ]
884    while self.HasMore():
885      self.SkipSpaces()
886      if not self.HasMore():
887        return None
888      if self.Current() == '(':
889        self.AddToken('(')
890        self.Advance()
891      elif self.Current() == ')':
892        self.AddToken(')')
893        self.Advance()
894      elif self.Current() == '$':
895        self.AddToken('$')
896        self.Advance()
897      elif self.Current() == ',':
898        self.AddToken(',')
899        self.Advance()
900      elif IsAlpha(self.Current()):
901        buf = ""
902        while self.HasMore() and IsAlpha(self.Current()):
903          buf += self.Current()
904          self.Advance()
905        self.AddToken(buf)
906      elif self.Current(2) == '&&':
907        self.AddToken('&&')
908        self.Advance(2)
909      elif self.Current(2) == '||':
910        self.AddToken('||')
911        self.Advance(2)
912      elif self.Current(2) == '==':
913        self.AddToken('==')
914        self.Advance(2)
915      else:
916        return None
917    return self.tokens
918
919
920class Scanner(object):
921  """A simple scanner that can serve out tokens from a given list"""
922
923  def __init__(self, tokens):
924    self.tokens = tokens
925    self.length = len(tokens)
926    self.index = 0
927
928  def HasMore(self):
929    return self.index < self.length
930
931  def Current(self):
932    return self.tokens[self.index]
933
934  def Advance(self):
935    self.index = self.index + 1
936
937
938def ParseAtomicExpression(scan):
939  if scan.Current() == "true":
940    scan.Advance()
941    return Constant(True)
942  elif scan.Current() == "false":
943    scan.Advance()
944    return Constant(False)
945  elif IsAlpha(scan.Current()):
946    name = scan.Current()
947    scan.Advance()
948    return Outcome(name.lower())
949  elif scan.Current() == '$':
950    scan.Advance()
951    if not IsAlpha(scan.Current()):
952      return None
953    name = scan.Current()
954    scan.Advance()
955    return Variable(name.lower())
956  elif scan.Current() == '(':
957    scan.Advance()
958    result = ParseLogicalExpression(scan)
959    if (not result) or (scan.Current() != ')'):
960      return None
961    scan.Advance()
962    return result
963  else:
964    return None
965
966
967BINARIES = ['==']
968def ParseOperatorExpression(scan):
969  left = ParseAtomicExpression(scan)
970  if not left: return None
971  while scan.HasMore() and (scan.Current() in BINARIES):
972    op = scan.Current()
973    scan.Advance()
974    right = ParseOperatorExpression(scan)
975    if not right:
976      return None
977    left = Operation(left, op, right)
978  return left
979
980
981def ParseConditionalExpression(scan):
982  left = ParseOperatorExpression(scan)
983  if not left: return None
984  while scan.HasMore() and (scan.Current() == 'if'):
985    scan.Advance()
986    right = ParseOperatorExpression(scan)
987    if not right:
988      return None
989    left=  Operation(left, 'if', right)
990  return left
991
992
993LOGICALS = ["&&", "||", ","]
994def ParseLogicalExpression(scan):
995  left = ParseConditionalExpression(scan)
996  if not left: return None
997  while scan.HasMore() and (scan.Current() in LOGICALS):
998    op = scan.Current()
999    scan.Advance()
1000    right = ParseConditionalExpression(scan)
1001    if not right:
1002      return None
1003    left = Operation(left, op, right)
1004  return left
1005
1006
1007def ParseCondition(expr):
1008  """Parses a logical expression into an Expression object"""
1009  tokens = Tokenizer(expr).Tokenize()
1010  if not tokens:
1011    print "Malformed expression: '%s'" % expr
1012    return None
1013  scan = Scanner(tokens)
1014  ast = ParseLogicalExpression(scan)
1015  if not ast:
1016    print "Malformed expression: '%s'" % expr
1017    return None
1018  if scan.HasMore():
1019    print "Malformed expression: '%s'" % expr
1020    return None
1021  return ast
1022
1023
1024class ClassifiedTest(object):
1025
1026  def __init__(self, case, outcomes):
1027    self.case = case
1028    self.outcomes = outcomes
1029
1030  def TestsIsolates(self):
1031    return self.case.TestsIsolates()
1032
1033
1034class Configuration(object):
1035  """The parsed contents of a configuration file"""
1036
1037  def __init__(self, sections, defs):
1038    self.sections = sections
1039    self.defs = defs
1040
1041  def ClassifyTests(self, cases, env):
1042    sections = [s for s in self.sections if s.condition.Evaluate(env, self.defs)]
1043    all_rules = reduce(list.__add__, [s.rules for s in sections], [])
1044    unused_rules = set(all_rules)
1045    result = [ ]
1046    all_outcomes = set([])
1047    for case in cases:
1048      matches = [ r for r in all_rules if r.Contains(case.path) ]
1049      outcomes = set([])
1050      for rule in matches:
1051        outcomes = outcomes.union(rule.GetOutcomes(env, self.defs))
1052        unused_rules.discard(rule)
1053      if not outcomes:
1054        outcomes = [PASS]
1055      case.outcomes = outcomes
1056      all_outcomes = all_outcomes.union(outcomes)
1057      result.append(ClassifiedTest(case, outcomes))
1058    return (result, list(unused_rules), all_outcomes)
1059
1060
1061class Section(object):
1062  """A section of the configuration file.  Sections are enabled or
1063  disabled prior to running the tests, based on their conditions"""
1064
1065  def __init__(self, condition):
1066    self.condition = condition
1067    self.rules = [ ]
1068
1069  def AddRule(self, rule):
1070    self.rules.append(rule)
1071
1072
1073class Rule(object):
1074  """A single rule that specifies the expected outcome for a single
1075  test."""
1076
1077  def __init__(self, raw_path, path, value):
1078    self.raw_path = raw_path
1079    self.path = path
1080    self.value = value
1081
1082  def GetOutcomes(self, env, defs):
1083    set = self.value.GetOutcomes(env, defs)
1084    assert isinstance(set, ListSet)
1085    return set.elms
1086
1087  def Contains(self, path):
1088    if len(self.path) > len(path):
1089      return False
1090    for i in xrange(len(self.path)):
1091      if not self.path[i].match(path[i]):
1092        return False
1093    return True
1094
1095
1096HEADER_PATTERN = re.compile(r'\[([^]]+)\]')
1097RULE_PATTERN = re.compile(r'\s*([^: ]*)\s*:(.*)')
1098DEF_PATTERN = re.compile(r'^def\s*(\w+)\s*=(.*)$')
1099PREFIX_PATTERN = re.compile(r'^\s*prefix\s+([\w\_\.\-\/]+)$')
1100
1101
1102def ReadConfigurationInto(path, sections, defs):
1103  current_section = Section(Constant(True))
1104  sections.append(current_section)
1105  prefix = []
1106  for line in utils.ReadLinesFrom(path):
1107    header_match = HEADER_PATTERN.match(line)
1108    if header_match:
1109      condition_str = header_match.group(1).strip()
1110      condition = ParseCondition(condition_str)
1111      new_section = Section(condition)
1112      sections.append(new_section)
1113      current_section = new_section
1114      continue
1115    rule_match = RULE_PATTERN.match(line)
1116    if rule_match:
1117      path = prefix + SplitPath(rule_match.group(1).strip())
1118      value_str = rule_match.group(2).strip()
1119      value = ParseCondition(value_str)
1120      if not value:
1121        return False
1122      current_section.AddRule(Rule(rule_match.group(1), path, value))
1123      continue
1124    def_match = DEF_PATTERN.match(line)
1125    if def_match:
1126      name = def_match.group(1).lower()
1127      value = ParseCondition(def_match.group(2).strip())
1128      if not value:
1129        return False
1130      defs[name] = value
1131      continue
1132    prefix_match = PREFIX_PATTERN.match(line)
1133    if prefix_match:
1134      prefix = SplitPath(prefix_match.group(1).strip())
1135      continue
1136    print "Malformed line: '%s'." % line
1137    return False
1138  return True
1139
1140
1141# ---------------
1142# --- M a i n ---
1143# ---------------
1144
1145
1146ARCH_GUESS = utils.GuessArchitecture()
1147
1148
1149def BuildOptions():
1150  result = optparse.OptionParser()
1151  result.add_option("-m", "--mode", help="The test modes in which to run (comma-separated)",
1152      default='release')
1153  result.add_option("-v", "--verbose", help="Verbose output",
1154      default=False, action="store_true")
1155  result.add_option("-S", dest="scons_flags", help="Flag to pass through to scons",
1156      default=[], action="append")
1157  result.add_option("-p", "--progress",
1158      help="The style of progress indicator (verbose, dots, color, mono)",
1159      choices=PROGRESS_INDICATORS.keys(), default="mono")
1160  result.add_option("--no-build", help="Don't build requirements",
1161      default=False, action="store_true")
1162  result.add_option("--build-only", help="Only build requirements, don't run the tests",
1163      default=False, action="store_true")
1164  result.add_option("--report", help="Print a summary of the tests to be run",
1165      default=False, action="store_true")
1166  result.add_option("-s", "--suite", help="A test suite",
1167      default=[], action="append")
1168  result.add_option("-t", "--timeout", help="Timeout in seconds",
1169      default=60, type="int")
1170  result.add_option("--arch", help='The architecture to run tests for',
1171      default='none')
1172  result.add_option("--snapshot", help="Run the tests with snapshot turned on",
1173      default=False, action="store_true")
1174  result.add_option("--simulator", help="Run tests with architecture simulator",
1175      default='none')
1176  result.add_option("--special-command", default=None)
1177  result.add_option("--valgrind", help="Run tests through valgrind",
1178      default=False, action="store_true")
1179  result.add_option("--cat", help="Print the source of the tests",
1180      default=False, action="store_true")
1181  result.add_option("--warn-unused", help="Report unused rules",
1182      default=False, action="store_true")
1183  result.add_option("-j", help="The number of parallel tasks to run",
1184      default=1, type="int")
1185  result.add_option("--time", help="Print timing information after running",
1186      default=False, action="store_true")
1187  result.add_option("--suppress-dialogs", help="Suppress Windows dialogs for crashing tests",
1188        dest="suppress_dialogs", default=True, action="store_true")
1189  result.add_option("--no-suppress-dialogs", help="Display Windows dialogs for crashing tests",
1190        dest="suppress_dialogs", action="store_false")
1191  result.add_option("--shell", help="Path to V8 shell", default="shell")
1192  result.add_option("--isolates", help="Whether to test isolates", default=False, action="store_true")
1193  result.add_option("--store-unexpected-output",
1194      help="Store the temporary JS files from tests that fails",
1195      dest="store_unexpected_output", default=True, action="store_true")
1196  result.add_option("--no-store-unexpected-output",
1197      help="Deletes the temporary JS files from tests that fails",
1198      dest="store_unexpected_output", action="store_false")
1199  result.add_option("--stress-only",
1200                    help="Only run tests with --always-opt --stress-opt",
1201                    default=False, action="store_true")
1202  result.add_option("--nostress",
1203                    help="Don't run crankshaft --always-opt --stress-op test",
1204                    default=False, action="store_true")
1205  result.add_option("--crankshaft",
1206                    help="Run with the --crankshaft flag",
1207                    default=False, action="store_true")
1208  result.add_option("--shard-count",
1209                    help="Split testsuites into this number of shards",
1210                    default=1, type="int")
1211  result.add_option("--shard-run",
1212                    help="Run this shard from the split up tests.",
1213                    default=1, type="int")
1214  result.add_option("--noprof", help="Disable profiling support",
1215                    default=False)
1216  return result
1217
1218
1219def ProcessOptions(options):
1220  global VERBOSE
1221  VERBOSE = options.verbose
1222  options.mode = options.mode.split(',')
1223  for mode in options.mode:
1224    if not mode in ['debug', 'release']:
1225      print "Unknown mode %s" % mode
1226      return False
1227  if options.simulator != 'none':
1228    # Simulator argument was set. Make sure arch and simulator agree.
1229    if options.simulator != options.arch:
1230      if options.arch == 'none':
1231        options.arch = options.simulator
1232      else:
1233        print "Architecture %s does not match sim %s" %(options.arch, options.simulator)
1234        return False
1235    # Ensure that the simulator argument is handed down to scons.
1236    options.scons_flags.append("simulator=" + options.simulator)
1237  else:
1238    # If options.arch is not set by the command line and no simulator setting
1239    # was found, set the arch to the guess.
1240    if options.arch == 'none':
1241      options.arch = ARCH_GUESS
1242    options.scons_flags.append("arch=" + options.arch)
1243  if options.snapshot:
1244    options.scons_flags.append("snapshot=on")
1245  global VARIANT_FLAGS
1246  if options.stress_only:
1247    VARIANT_FLAGS = [['--stress-opt', '--always-opt']]
1248  if options.nostress:
1249    VARIANT_FLAGS = [[],['--nocrankshaft']]
1250  if options.crankshaft:
1251    if options.special_command:
1252      options.special_command += " --crankshaft"
1253    else:
1254      options.special_command = "@--crankshaft"
1255  if options.noprof:
1256    options.scons_flags.append("prof=off")
1257    options.scons_flags.append("profilingsupport=off")
1258  return True
1259
1260
1261REPORT_TEMPLATE = """\
1262Total: %(total)i tests
1263 * %(skipped)4d tests will be skipped
1264 * %(nocrash)4d tests are expected to be flaky but not crash
1265 * %(pass)4d tests are expected to pass
1266 * %(fail_ok)4d tests are expected to fail that we won't fix
1267 * %(fail)4d tests are expected to fail that we should fix\
1268"""
1269
1270def PrintReport(cases):
1271  def IsFlaky(o):
1272    return (PASS in o) and (FAIL in o) and (not CRASH in o) and (not OKAY in o)
1273  def IsFailOk(o):
1274    return (len(o) == 2) and (FAIL in o) and (OKAY in o)
1275  unskipped = [c for c in cases if not SKIP in c.outcomes]
1276  print REPORT_TEMPLATE % {
1277    'total': len(cases),
1278    'skipped': len(cases) - len(unskipped),
1279    'nocrash': len([t for t in unskipped if IsFlaky(t.outcomes)]),
1280    'pass': len([t for t in unskipped if list(t.outcomes) == [PASS]]),
1281    'fail_ok': len([t for t in unskipped if IsFailOk(t.outcomes)]),
1282    'fail': len([t for t in unskipped if list(t.outcomes) == [FAIL]])
1283  }
1284
1285
1286class Pattern(object):
1287
1288  def __init__(self, pattern):
1289    self.pattern = pattern
1290    self.compiled = None
1291
1292  def match(self, str):
1293    if not self.compiled:
1294      pattern = "^" + self.pattern.replace('*', '.*') + "$"
1295      self.compiled = re.compile(pattern)
1296    return self.compiled.match(str)
1297
1298  def __str__(self):
1299    return self.pattern
1300
1301
1302def SplitPath(s):
1303  stripped = [ c.strip() for c in s.split('/') ]
1304  return [ Pattern(s) for s in stripped if len(s) > 0 ]
1305
1306
1307def GetSpecialCommandProcessor(value):
1308  if (not value) or (value.find('@') == -1):
1309    def ExpandCommand(args):
1310      return args
1311    return ExpandCommand
1312  else:
1313    pos = value.find('@')
1314    import urllib
1315    prefix = urllib.unquote(value[:pos]).split()
1316    suffix = urllib.unquote(value[pos+1:]).split()
1317    def ExpandCommand(args):
1318      return prefix + args + suffix
1319    return ExpandCommand
1320
1321
1322BUILT_IN_TESTS = ['mjsunit', 'cctest', 'message', 'preparser']
1323
1324
1325def GetSuites(test_root):
1326  def IsSuite(path):
1327    return isdir(path) and exists(join(path, 'testcfg.py'))
1328  return [ f for f in os.listdir(test_root) if IsSuite(join(test_root, f)) ]
1329
1330
1331def FormatTime(d):
1332  millis = round(d * 1000) % 1000
1333  return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis)
1334
1335def ShardTests(tests, options):
1336  if options.shard_count < 2:
1337    return tests
1338  if options.shard_run < 1 or options.shard_run > options.shard_count:
1339    print "shard-run not a valid number, should be in [1:shard-count]"
1340    print "defaulting back to running all tests"
1341    return tests
1342  count = 0;
1343  shard = []
1344  for test in tests:
1345    if count % options.shard_count == options.shard_run - 1:
1346      shard.append(test);
1347    count += 1
1348  return shard
1349
1350def Main():
1351  parser = BuildOptions()
1352  (options, args) = parser.parse_args()
1353  if not ProcessOptions(options):
1354    parser.print_help()
1355    return 1
1356
1357  workspace = abspath(join(dirname(sys.argv[0]), '..'))
1358  suites = GetSuites(join(workspace, 'test'))
1359  repositories = [TestRepository(join(workspace, 'test', name)) for name in suites]
1360  repositories += [TestRepository(a) for a in options.suite]
1361
1362  root = LiteralTestSuite(repositories)
1363  if len(args) == 0:
1364    paths = [SplitPath(t) for t in BUILT_IN_TESTS]
1365  else:
1366    paths = [ ]
1367    for arg in args:
1368      path = SplitPath(arg)
1369      paths.append(path)
1370
1371  # Check for --valgrind option. If enabled, we overwrite the special
1372  # command flag with a command that uses the run-valgrind.py script.
1373  if options.valgrind:
1374    run_valgrind = join(workspace, "tools", "run-valgrind.py")
1375    options.special_command = "python -u " + run_valgrind + " @"
1376
1377  shell = abspath(options.shell)
1378  buildspace = dirname(shell)
1379
1380  context = Context(workspace, buildspace, VERBOSE,
1381                    shell,
1382                    options.timeout,
1383                    GetSpecialCommandProcessor(options.special_command),
1384                    options.suppress_dialogs,
1385                    options.store_unexpected_output)
1386  # First build the required targets
1387  if not options.no_build:
1388    reqs = [ ]
1389    for path in paths:
1390      reqs += root.GetBuildRequirements(path, context)
1391    reqs = list(set(reqs))
1392    if len(reqs) > 0:
1393      if options.j != 1:
1394        options.scons_flags += ['-j', str(options.j)]
1395      if not BuildRequirements(context, reqs, options.mode, options.scons_flags):
1396        return 1
1397
1398  # Just return if we are only building the targets for running the tests.
1399  if options.build_only:
1400    return 0
1401
1402  # Get status for tests
1403  sections = [ ]
1404  defs = { }
1405  root.GetTestStatus(context, sections, defs)
1406  config = Configuration(sections, defs)
1407
1408  # List the tests
1409  all_cases = [ ]
1410  all_unused = [ ]
1411  unclassified_tests = [ ]
1412  globally_unused_rules = None
1413  for path in paths:
1414    for mode in options.mode:
1415      env = {
1416        'mode': mode,
1417        'system': utils.GuessOS(),
1418        'arch': options.arch,
1419        'simulator': options.simulator,
1420        'crankshaft': options.crankshaft
1421      }
1422      test_list = root.ListTests([], path, context, mode, [])
1423      unclassified_tests += test_list
1424      (cases, unused_rules, all_outcomes) = config.ClassifyTests(test_list, env)
1425      if globally_unused_rules is None:
1426        globally_unused_rules = set(unused_rules)
1427      else:
1428        globally_unused_rules = globally_unused_rules.intersection(unused_rules)
1429      all_cases += ShardTests(cases, options)
1430      all_unused.append(unused_rules)
1431
1432  if options.cat:
1433    visited = set()
1434    for test in unclassified_tests:
1435      key = tuple(test.path)
1436      if key in visited:
1437        continue
1438      visited.add(key)
1439      print "--- begin source: %s ---" % test.GetLabel()
1440      source = test.GetSource().strip()
1441      print source
1442      print "--- end source: %s ---" % test.GetLabel()
1443    return 0
1444
1445  if options.warn_unused:
1446    for rule in globally_unused_rules:
1447      print "Rule for '%s' was not used." % '/'.join([str(s) for s in rule.path])
1448
1449  if options.report:
1450    PrintReport(all_cases)
1451
1452  result = None
1453  def DoSkip(case):
1454    return SKIP in case.outcomes or SLOW in case.outcomes
1455  cases_to_run = [ c for c in all_cases if not DoSkip(c) ]
1456  if not options.isolates:
1457    cases_to_run = [c for c in cases_to_run if not c.TestsIsolates()]
1458  if len(cases_to_run) == 0:
1459    print "No tests to run."
1460    return 0
1461  else:
1462    try:
1463      start = time.time()
1464      if RunTestCases(cases_to_run, options.progress, options.j):
1465        result = 0
1466      else:
1467        result = 1
1468      duration = time.time() - start
1469    except KeyboardInterrupt:
1470      print "Interrupted"
1471      return 1
1472
1473  if options.time:
1474    # Write the times to stderr to make it easy to separate from the
1475    # test output.
1476    print
1477    sys.stderr.write("--- Total time: %s ---\n" % FormatTime(duration))
1478    timed_tests = [ t.case for t in cases_to_run if not t.case.duration is None ]
1479    timed_tests.sort(lambda a, b: a.CompareTime(b))
1480    index = 1
1481    for entry in timed_tests[:20]:
1482      t = FormatTime(entry.duration)
1483      sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel()))
1484      index += 1
1485
1486  return result
1487
1488
1489if __name__ == '__main__':
1490  sys.exit(Main())
1491