1#!/usr/bin/env python 2# 3# Copyright 2008 the V8 project authors. All rights reserved. 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following 12# disclaimer in the documentation and/or other materials provided 13# with the distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived 16# from this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 31import imp 32import optparse 33import os 34from os.path import join, dirname, abspath, basename, isdir, exists 35import platform 36import re 37import signal 38import subprocess 39import sys 40import tempfile 41import time 42import threading 43import utils 44from Queue import Queue, Empty 45 46 47VERBOSE = False 48 49 50# --------------------------------------------- 51# --- P r o g r e s s I n d i c a t o r s --- 52# --------------------------------------------- 53 54 55class ProgressIndicator(object): 56 57 def __init__(self, cases): 58 self.cases = cases 59 self.queue = Queue(len(cases)) 60 for case in cases: 61 self.queue.put_nowait(case) 62 self.succeeded = 0 63 self.remaining = len(cases) 64 self.total = len(cases) 65 self.failed = [ ] 66 self.crashed = 0 67 self.terminate = False 68 self.lock = threading.Lock() 69 70 def PrintFailureHeader(self, test): 71 if test.IsNegative(): 72 negative_marker = '[negative] ' 73 else: 74 negative_marker = '' 75 print "=== %(label)s %(negative)s===" % { 76 'label': test.GetLabel(), 77 'negative': negative_marker 78 } 79 print "Path: %s" % "/".join(test.path) 80 81 def Run(self, tasks): 82 self.Starting() 83 threads = [] 84 # Spawn N-1 threads and then use this thread as the last one. 85 # That way -j1 avoids threading altogether which is a nice fallback 86 # in case of threading problems. 87 for i in xrange(tasks - 1): 88 thread = threading.Thread(target=self.RunSingle, args=[]) 89 threads.append(thread) 90 thread.start() 91 try: 92 self.RunSingle() 93 # Wait for the remaining threads 94 for thread in threads: 95 # Use a timeout so that signals (ctrl-c) will be processed. 96 thread.join(timeout=10000000) 97 except Exception, e: 98 # If there's an exception we schedule an interruption for any 99 # remaining threads. 100 self.terminate = True 101 # ...and then reraise the exception to bail out 102 raise 103 self.Done() 104 return not self.failed 105 106 def RunSingle(self): 107 while not self.terminate: 108 try: 109 test = self.queue.get_nowait() 110 except Empty: 111 return 112 case = test.case 113 self.lock.acquire() 114 self.AboutToRun(case) 115 self.lock.release() 116 try: 117 start = time.time() 118 output = case.Run() 119 case.duration = (time.time() - start) 120 except IOError, e: 121 assert self.terminate 122 return 123 if self.terminate: 124 return 125 self.lock.acquire() 126 if output.UnexpectedOutput(): 127 self.failed.append(output) 128 if output.HasCrashed(): 129 self.crashed += 1 130 else: 131 self.succeeded += 1 132 self.remaining -= 1 133 self.HasRun(output) 134 self.lock.release() 135 136 137def EscapeCommand(command): 138 parts = [] 139 for part in command: 140 if ' ' in part: 141 # Escape spaces. We may need to escape more characters for this 142 # to work properly. 143 parts.append('"%s"' % part) 144 else: 145 parts.append(part) 146 return " ".join(parts) 147 148 149class SimpleProgressIndicator(ProgressIndicator): 150 151 def Starting(self): 152 print 'Running %i tests' % len(self.cases) 153 154 def Done(self): 155 print 156 for failed in self.failed: 157 self.PrintFailureHeader(failed.test) 158 if failed.output.stderr: 159 print "--- stderr ---" 160 print failed.output.stderr.strip() 161 if failed.output.stdout: 162 print "--- stdout ---" 163 print failed.output.stdout.strip() 164 print "Command: %s" % EscapeCommand(failed.command) 165 if failed.HasCrashed(): 166 print "--- CRASHED ---" 167 if failed.HasTimedOut(): 168 print "--- TIMEOUT ---" 169 if len(self.failed) == 0: 170 print "===" 171 print "=== All tests succeeded" 172 print "===" 173 else: 174 print 175 print "===" 176 print "=== %i tests failed" % len(self.failed) 177 if self.crashed > 0: 178 print "=== %i tests CRASHED" % self.crashed 179 print "===" 180 181 182class VerboseProgressIndicator(SimpleProgressIndicator): 183 184 def AboutToRun(self, case): 185 print 'Starting %s...' % case.GetLabel() 186 sys.stdout.flush() 187 188 def HasRun(self, output): 189 if output.UnexpectedOutput(): 190 if output.HasCrashed(): 191 outcome = 'CRASH' 192 else: 193 outcome = 'FAIL' 194 else: 195 outcome = 'pass' 196 print 'Done running %s: %s' % (output.test.GetLabel(), outcome) 197 198 199class DotsProgressIndicator(SimpleProgressIndicator): 200 201 def AboutToRun(self, case): 202 pass 203 204 def HasRun(self, output): 205 total = self.succeeded + len(self.failed) 206 if (total > 1) and (total % 50 == 1): 207 sys.stdout.write('\n') 208 if output.UnexpectedOutput(): 209 if output.HasCrashed(): 210 sys.stdout.write('C') 211 sys.stdout.flush() 212 elif output.HasTimedOut(): 213 sys.stdout.write('T') 214 sys.stdout.flush() 215 else: 216 sys.stdout.write('F') 217 sys.stdout.flush() 218 else: 219 sys.stdout.write('.') 220 sys.stdout.flush() 221 222 223class CompactProgressIndicator(ProgressIndicator): 224 225 def __init__(self, cases, templates): 226 super(CompactProgressIndicator, self).__init__(cases) 227 self.templates = templates 228 self.last_status_length = 0 229 self.start_time = time.time() 230 231 def Starting(self): 232 pass 233 234 def Done(self): 235 self.PrintProgress('Done') 236 237 def AboutToRun(self, case): 238 self.PrintProgress(case.GetLabel()) 239 240 def HasRun(self, output): 241 if output.UnexpectedOutput(): 242 self.ClearLine(self.last_status_length) 243 self.PrintFailureHeader(output.test) 244 stdout = output.output.stdout.strip() 245 if len(stdout): 246 print self.templates['stdout'] % stdout 247 stderr = output.output.stderr.strip() 248 if len(stderr): 249 print self.templates['stderr'] % stderr 250 print "Command: %s" % EscapeCommand(output.command) 251 if output.HasCrashed(): 252 print "--- CRASHED ---" 253 if output.HasTimedOut(): 254 print "--- TIMEOUT ---" 255 256 def Truncate(self, str, length): 257 if length and (len(str) > (length - 3)): 258 return str[:(length-3)] + "..." 259 else: 260 return str 261 262 def PrintProgress(self, name): 263 self.ClearLine(self.last_status_length) 264 elapsed = time.time() - self.start_time 265 status = self.templates['status_line'] % { 266 'passed': self.succeeded, 267 'remaining': (((self.total - self.remaining) * 100) // self.total), 268 'failed': len(self.failed), 269 'test': name, 270 'mins': int(elapsed) / 60, 271 'secs': int(elapsed) % 60 272 } 273 status = self.Truncate(status, 78) 274 self.last_status_length = len(status) 275 print status, 276 sys.stdout.flush() 277 278 279class ColorProgressIndicator(CompactProgressIndicator): 280 281 def __init__(self, cases): 282 templates = { 283 'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s", 284 'stdout': "\033[1m%s\033[0m", 285 'stderr': "\033[31m%s\033[0m", 286 } 287 super(ColorProgressIndicator, self).__init__(cases, templates) 288 289 def ClearLine(self, last_line_length): 290 print "\033[1K\r", 291 292 293class MonochromeProgressIndicator(CompactProgressIndicator): 294 295 def __init__(self, cases): 296 templates = { 297 'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s", 298 'stdout': '%s', 299 'stderr': '%s', 300 'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"), 301 'max_length': 78 302 } 303 super(MonochromeProgressIndicator, self).__init__(cases, templates) 304 305 def ClearLine(self, last_line_length): 306 print ("\r" + (" " * last_line_length) + "\r"), 307 308 309PROGRESS_INDICATORS = { 310 'verbose': VerboseProgressIndicator, 311 'dots': DotsProgressIndicator, 312 'color': ColorProgressIndicator, 313 'mono': MonochromeProgressIndicator 314} 315 316 317# ------------------------- 318# --- F r a m e w o r k --- 319# ------------------------- 320 321 322class CommandOutput(object): 323 324 def __init__(self, exit_code, timed_out, stdout, stderr): 325 self.exit_code = exit_code 326 self.timed_out = timed_out 327 self.stdout = stdout 328 self.stderr = stderr 329 self.failed = None 330 331 332class TestCase(object): 333 334 def __init__(self, context, path, mode): 335 self.path = path 336 self.context = context 337 self.duration = None 338 self.mode = mode 339 340 def IsNegative(self): 341 return False 342 343 def TestsIsolates(self): 344 return False 345 346 def CompareTime(self, other): 347 return cmp(other.duration, self.duration) 348 349 def DidFail(self, output): 350 if output.failed is None: 351 output.failed = self.IsFailureOutput(output) 352 return output.failed 353 354 def IsFailureOutput(self, output): 355 return output.exit_code != 0 356 357 def GetSource(self): 358 return "(no source available)" 359 360 def RunCommand(self, command): 361 full_command = self.context.processor(command) 362 output = Execute(full_command, 363 self.context, 364 self.context.GetTimeout(self, self.mode)) 365 self.Cleanup() 366 return TestOutput(self, 367 full_command, 368 output, 369 self.context.store_unexpected_output) 370 371 def BeforeRun(self): 372 pass 373 374 def AfterRun(self, result): 375 pass 376 377 def GetCustomFlags(self, mode): 378 return None 379 380 def Run(self): 381 self.BeforeRun() 382 result = "exception" 383 try: 384 result = self.RunCommand(self.GetCommand()) 385 finally: 386 self.AfterRun(result) 387 return result 388 389 def Cleanup(self): 390 return 391 392 393class TestOutput(object): 394 395 def __init__(self, test, command, output, store_unexpected_output): 396 self.test = test 397 self.command = command 398 self.output = output 399 self.store_unexpected_output = store_unexpected_output 400 401 def UnexpectedOutput(self): 402 if self.HasCrashed(): 403 outcome = CRASH 404 elif self.HasTimedOut(): 405 outcome = TIMEOUT 406 elif self.HasFailed(): 407 outcome = FAIL 408 else: 409 outcome = PASS 410 return not outcome in self.test.outcomes 411 412 def HasPreciousOutput(self): 413 return self.UnexpectedOutput() and self.store_unexpected_output 414 415 def HasCrashed(self): 416 if utils.IsWindows(): 417 return 0x80000000 & self.output.exit_code and not (0x3FFFFF00 & self.output.exit_code) 418 else: 419 # Timed out tests will have exit_code -signal.SIGTERM. 420 if self.output.timed_out: 421 return False 422 return self.output.exit_code < 0 and \ 423 self.output.exit_code != -signal.SIGABRT 424 425 def HasTimedOut(self): 426 return self.output.timed_out; 427 428 def HasFailed(self): 429 execution_failed = self.test.DidFail(self.output) 430 if self.test.IsNegative(): 431 return not execution_failed 432 else: 433 return execution_failed 434 435 436def KillProcessWithID(pid): 437 if utils.IsWindows(): 438 os.popen('taskkill /T /F /PID %d' % pid) 439 else: 440 os.kill(pid, signal.SIGTERM) 441 442 443MAX_SLEEP_TIME = 0.1 444INITIAL_SLEEP_TIME = 0.0001 445SLEEP_TIME_FACTOR = 1.25 446 447SEM_INVALID_VALUE = -1 448SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h 449 450def Win32SetErrorMode(mode): 451 prev_error_mode = SEM_INVALID_VALUE 452 try: 453 import ctypes 454 prev_error_mode = ctypes.windll.kernel32.SetErrorMode(mode); 455 except ImportError: 456 pass 457 return prev_error_mode 458 459def RunProcess(context, timeout, args, **rest): 460 if context.verbose: print "#", " ".join(args) 461 popen_args = args 462 prev_error_mode = SEM_INVALID_VALUE; 463 if utils.IsWindows(): 464 popen_args = '"' + subprocess.list2cmdline(args) + '"' 465 if context.suppress_dialogs: 466 # Try to change the error mode to avoid dialogs on fatal errors. Don't 467 # touch any existing error mode flags by merging the existing error mode. 468 # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx. 469 error_mode = SEM_NOGPFAULTERRORBOX; 470 prev_error_mode = Win32SetErrorMode(error_mode); 471 Win32SetErrorMode(error_mode | prev_error_mode); 472 process = subprocess.Popen( 473 shell = utils.IsWindows(), 474 args = popen_args, 475 **rest 476 ) 477 if utils.IsWindows() and context.suppress_dialogs and prev_error_mode != SEM_INVALID_VALUE: 478 Win32SetErrorMode(prev_error_mode) 479 # Compute the end time - if the process crosses this limit we 480 # consider it timed out. 481 if timeout is None: end_time = None 482 else: end_time = time.time() + timeout 483 timed_out = False 484 # Repeatedly check the exit code from the process in a 485 # loop and keep track of whether or not it times out. 486 exit_code = None 487 sleep_time = INITIAL_SLEEP_TIME 488 while exit_code is None: 489 if (not end_time is None) and (time.time() >= end_time): 490 # Kill the process and wait for it to exit. 491 KillProcessWithID(process.pid) 492 exit_code = process.wait() 493 timed_out = True 494 else: 495 exit_code = process.poll() 496 time.sleep(sleep_time) 497 sleep_time = sleep_time * SLEEP_TIME_FACTOR 498 if sleep_time > MAX_SLEEP_TIME: 499 sleep_time = MAX_SLEEP_TIME 500 return (process, exit_code, timed_out) 501 502 503def PrintError(str): 504 sys.stderr.write(str) 505 sys.stderr.write('\n') 506 507 508def CheckedUnlink(name): 509 # On Windows, when run with -jN in parallel processes, 510 # OS often fails to unlink the temp file. Not sure why. 511 # Need to retry. 512 # Idea from https://bugs.webkit.org/attachment.cgi?id=75982&action=prettypatch 513 retry_count = 0 514 while retry_count < 30: 515 try: 516 os.unlink(name) 517 return 518 except OSError, e: 519 retry_count += 1; 520 time.sleep(retry_count * 0.1) 521 PrintError("os.unlink() " + str(e)) 522 523def Execute(args, context, timeout=None): 524 (fd_out, outname) = tempfile.mkstemp() 525 (fd_err, errname) = tempfile.mkstemp() 526 (process, exit_code, timed_out) = RunProcess( 527 context, 528 timeout, 529 args = args, 530 stdout = fd_out, 531 stderr = fd_err, 532 ) 533 os.close(fd_out) 534 os.close(fd_err) 535 output = file(outname).read() 536 errors = file(errname).read() 537 CheckedUnlink(outname) 538 CheckedUnlink(errname) 539 return CommandOutput(exit_code, timed_out, output, errors) 540 541 542def ExecuteNoCapture(args, context, timeout=None): 543 (process, exit_code, timed_out) = RunProcess( 544 context, 545 timeout, 546 args = args, 547 ) 548 return CommandOutput(exit_code, False, "", "") 549 550 551def CarCdr(path): 552 if len(path) == 0: 553 return (None, [ ]) 554 else: 555 return (path[0], path[1:]) 556 557 558class TestConfiguration(object): 559 560 def __init__(self, context, root): 561 self.context = context 562 self.root = root 563 564 def Contains(self, path, file): 565 if len(path) > len(file): 566 return False 567 for i in xrange(len(path)): 568 if not path[i].match(file[i]): 569 return False 570 return True 571 572 def GetTestStatus(self, sections, defs): 573 pass 574 575 576class TestSuite(object): 577 578 def __init__(self, name): 579 self.name = name 580 581 def GetName(self): 582 return self.name 583 584 585# Use this to run several variants of the tests, e.g.: 586# VARIANT_FLAGS = [[], ['--always_compact', '--noflush_code']] 587VARIANT_FLAGS = [[], 588 ['--stress-opt', '--always-opt'], 589 ['--nocrankshaft']] 590 591 592class TestRepository(TestSuite): 593 594 def __init__(self, path): 595 normalized_path = abspath(path) 596 super(TestRepository, self).__init__(basename(normalized_path)) 597 self.path = normalized_path 598 self.is_loaded = False 599 self.config = None 600 601 def GetConfiguration(self, context): 602 if self.is_loaded: 603 return self.config 604 self.is_loaded = True 605 file = None 606 try: 607 (file, pathname, description) = imp.find_module('testcfg', [ self.path ]) 608 module = imp.load_module('testcfg', file, pathname, description) 609 self.config = module.GetConfiguration(context, self.path) 610 finally: 611 if file: 612 file.close() 613 return self.config 614 615 def GetBuildRequirements(self, path, context): 616 return self.GetConfiguration(context).GetBuildRequirements() 617 618 def AddTestsToList(self, result, current_path, path, context, mode): 619 for v in VARIANT_FLAGS: 620 tests = self.GetConfiguration(context).ListTests(current_path, path, mode, v) 621 for t in tests: t.variant_flags = v 622 result += tests 623 624 625 def GetTestStatus(self, context, sections, defs): 626 self.GetConfiguration(context).GetTestStatus(sections, defs) 627 628 629class LiteralTestSuite(TestSuite): 630 631 def __init__(self, tests): 632 super(LiteralTestSuite, self).__init__('root') 633 self.tests = tests 634 635 def GetBuildRequirements(self, path, context): 636 (name, rest) = CarCdr(path) 637 result = [ ] 638 for test in self.tests: 639 if not name or name.match(test.GetName()): 640 result += test.GetBuildRequirements(rest, context) 641 return result 642 643 def ListTests(self, current_path, path, context, mode, variant_flags): 644 (name, rest) = CarCdr(path) 645 result = [ ] 646 for test in self.tests: 647 test_name = test.GetName() 648 if not name or name.match(test_name): 649 full_path = current_path + [test_name] 650 test.AddTestsToList(result, full_path, path, context, mode) 651 return result 652 653 def GetTestStatus(self, context, sections, defs): 654 for test in self.tests: 655 test.GetTestStatus(context, sections, defs) 656 657 658SUFFIX = { 659 'debug' : '_g', 660 'release' : '' } 661FLAGS = { 662 'debug' : ['--enable-slow-asserts', '--debug-code', '--verify-heap'], 663 'release' : []} 664TIMEOUT_SCALEFACTOR = { 665 'debug' : 4, 666 'release' : 1 } 667 668 669class Context(object): 670 671 def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs, store_unexpected_output): 672 self.workspace = workspace 673 self.buildspace = buildspace 674 self.verbose = verbose 675 self.vm_root = vm 676 self.timeout = timeout 677 self.processor = processor 678 self.suppress_dialogs = suppress_dialogs 679 self.store_unexpected_output = store_unexpected_output 680 681 def GetVm(self, mode): 682 name = self.vm_root + SUFFIX[mode] 683 if utils.IsWindows() and not name.endswith('.exe'): 684 name = name + '.exe' 685 return name 686 687 def GetVmCommand(self, testcase, mode): 688 return [self.GetVm(mode)] + self.GetVmFlags(testcase, mode) 689 690 def GetVmFlags(self, testcase, mode): 691 flags = testcase.GetCustomFlags(mode) 692 if flags is None: 693 flags = FLAGS[mode] 694 return testcase.variant_flags + flags 695 696 def GetTimeout(self, testcase, mode): 697 result = self.timeout * TIMEOUT_SCALEFACTOR[mode] 698 if '--stress-opt' in self.GetVmFlags(testcase, mode): 699 return result * 2 700 else: 701 return result 702 703def RunTestCases(cases_to_run, progress, tasks): 704 progress = PROGRESS_INDICATORS[progress](cases_to_run) 705 return progress.Run(tasks) 706 707 708def BuildRequirements(context, requirements, mode, scons_flags): 709 command_line = (['scons', '-Y', context.workspace, 'mode=' + ",".join(mode)] 710 + requirements 711 + scons_flags) 712 output = ExecuteNoCapture(command_line, context) 713 return output.exit_code == 0 714 715 716# ------------------------------------------- 717# --- T e s t C o n f i g u r a t i o n --- 718# ------------------------------------------- 719 720 721SKIP = 'skip' 722FAIL = 'fail' 723PASS = 'pass' 724OKAY = 'okay' 725TIMEOUT = 'timeout' 726CRASH = 'crash' 727SLOW = 'slow' 728 729 730class Expression(object): 731 pass 732 733 734class Constant(Expression): 735 736 def __init__(self, value): 737 self.value = value 738 739 def Evaluate(self, env, defs): 740 return self.value 741 742 743class Variable(Expression): 744 745 def __init__(self, name): 746 self.name = name 747 748 def GetOutcomes(self, env, defs): 749 if self.name in env: return ListSet([env[self.name]]) 750 else: return Nothing() 751 752 def Evaluate(self, env, defs): 753 return env[self.name] 754 755 756class Outcome(Expression): 757 758 def __init__(self, name): 759 self.name = name 760 761 def GetOutcomes(self, env, defs): 762 if self.name in defs: 763 return defs[self.name].GetOutcomes(env, defs) 764 else: 765 return ListSet([self.name]) 766 767 768class Set(object): 769 pass 770 771 772class ListSet(Set): 773 774 def __init__(self, elms): 775 self.elms = elms 776 777 def __str__(self): 778 return "ListSet%s" % str(self.elms) 779 780 def Intersect(self, that): 781 if not isinstance(that, ListSet): 782 return that.Intersect(self) 783 return ListSet([ x for x in self.elms if x in that.elms ]) 784 785 def Union(self, that): 786 if not isinstance(that, ListSet): 787 return that.Union(self) 788 return ListSet(self.elms + [ x for x in that.elms if x not in self.elms ]) 789 790 def IsEmpty(self): 791 return len(self.elms) == 0 792 793 794class Everything(Set): 795 796 def Intersect(self, that): 797 return that 798 799 def Union(self, that): 800 return self 801 802 def IsEmpty(self): 803 return False 804 805 806class Nothing(Set): 807 808 def Intersect(self, that): 809 return self 810 811 def Union(self, that): 812 return that 813 814 def IsEmpty(self): 815 return True 816 817 818class Operation(Expression): 819 820 def __init__(self, left, op, right): 821 self.left = left 822 self.op = op 823 self.right = right 824 825 def Evaluate(self, env, defs): 826 if self.op == '||' or self.op == ',': 827 return self.left.Evaluate(env, defs) or self.right.Evaluate(env, defs) 828 elif self.op == 'if': 829 return False 830 elif self.op == '==': 831 inter = self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs)) 832 return not inter.IsEmpty() 833 else: 834 assert self.op == '&&' 835 return self.left.Evaluate(env, defs) and self.right.Evaluate(env, defs) 836 837 def GetOutcomes(self, env, defs): 838 if self.op == '||' or self.op == ',': 839 return self.left.GetOutcomes(env, defs).Union(self.right.GetOutcomes(env, defs)) 840 elif self.op == 'if': 841 if self.right.Evaluate(env, defs): return self.left.GetOutcomes(env, defs) 842 else: return Nothing() 843 else: 844 assert self.op == '&&' 845 return self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs)) 846 847 848def IsAlpha(str): 849 for char in str: 850 if not (char.isalpha() or char.isdigit() or char == '_'): 851 return False 852 return True 853 854 855class Tokenizer(object): 856 """A simple string tokenizer that chops expressions into variables, 857 parens and operators""" 858 859 def __init__(self, expr): 860 self.index = 0 861 self.expr = expr 862 self.length = len(expr) 863 self.tokens = None 864 865 def Current(self, length = 1): 866 if not self.HasMore(length): return "" 867 return self.expr[self.index:self.index+length] 868 869 def HasMore(self, length = 1): 870 return self.index < self.length + (length - 1) 871 872 def Advance(self, count = 1): 873 self.index = self.index + count 874 875 def AddToken(self, token): 876 self.tokens.append(token) 877 878 def SkipSpaces(self): 879 while self.HasMore() and self.Current().isspace(): 880 self.Advance() 881 882 def Tokenize(self): 883 self.tokens = [ ] 884 while self.HasMore(): 885 self.SkipSpaces() 886 if not self.HasMore(): 887 return None 888 if self.Current() == '(': 889 self.AddToken('(') 890 self.Advance() 891 elif self.Current() == ')': 892 self.AddToken(')') 893 self.Advance() 894 elif self.Current() == '$': 895 self.AddToken('$') 896 self.Advance() 897 elif self.Current() == ',': 898 self.AddToken(',') 899 self.Advance() 900 elif IsAlpha(self.Current()): 901 buf = "" 902 while self.HasMore() and IsAlpha(self.Current()): 903 buf += self.Current() 904 self.Advance() 905 self.AddToken(buf) 906 elif self.Current(2) == '&&': 907 self.AddToken('&&') 908 self.Advance(2) 909 elif self.Current(2) == '||': 910 self.AddToken('||') 911 self.Advance(2) 912 elif self.Current(2) == '==': 913 self.AddToken('==') 914 self.Advance(2) 915 else: 916 return None 917 return self.tokens 918 919 920class Scanner(object): 921 """A simple scanner that can serve out tokens from a given list""" 922 923 def __init__(self, tokens): 924 self.tokens = tokens 925 self.length = len(tokens) 926 self.index = 0 927 928 def HasMore(self): 929 return self.index < self.length 930 931 def Current(self): 932 return self.tokens[self.index] 933 934 def Advance(self): 935 self.index = self.index + 1 936 937 938def ParseAtomicExpression(scan): 939 if scan.Current() == "true": 940 scan.Advance() 941 return Constant(True) 942 elif scan.Current() == "false": 943 scan.Advance() 944 return Constant(False) 945 elif IsAlpha(scan.Current()): 946 name = scan.Current() 947 scan.Advance() 948 return Outcome(name.lower()) 949 elif scan.Current() == '$': 950 scan.Advance() 951 if not IsAlpha(scan.Current()): 952 return None 953 name = scan.Current() 954 scan.Advance() 955 return Variable(name.lower()) 956 elif scan.Current() == '(': 957 scan.Advance() 958 result = ParseLogicalExpression(scan) 959 if (not result) or (scan.Current() != ')'): 960 return None 961 scan.Advance() 962 return result 963 else: 964 return None 965 966 967BINARIES = ['=='] 968def ParseOperatorExpression(scan): 969 left = ParseAtomicExpression(scan) 970 if not left: return None 971 while scan.HasMore() and (scan.Current() in BINARIES): 972 op = scan.Current() 973 scan.Advance() 974 right = ParseOperatorExpression(scan) 975 if not right: 976 return None 977 left = Operation(left, op, right) 978 return left 979 980 981def ParseConditionalExpression(scan): 982 left = ParseOperatorExpression(scan) 983 if not left: return None 984 while scan.HasMore() and (scan.Current() == 'if'): 985 scan.Advance() 986 right = ParseOperatorExpression(scan) 987 if not right: 988 return None 989 left= Operation(left, 'if', right) 990 return left 991 992 993LOGICALS = ["&&", "||", ","] 994def ParseLogicalExpression(scan): 995 left = ParseConditionalExpression(scan) 996 if not left: return None 997 while scan.HasMore() and (scan.Current() in LOGICALS): 998 op = scan.Current() 999 scan.Advance() 1000 right = ParseConditionalExpression(scan) 1001 if not right: 1002 return None 1003 left = Operation(left, op, right) 1004 return left 1005 1006 1007def ParseCondition(expr): 1008 """Parses a logical expression into an Expression object""" 1009 tokens = Tokenizer(expr).Tokenize() 1010 if not tokens: 1011 print "Malformed expression: '%s'" % expr 1012 return None 1013 scan = Scanner(tokens) 1014 ast = ParseLogicalExpression(scan) 1015 if not ast: 1016 print "Malformed expression: '%s'" % expr 1017 return None 1018 if scan.HasMore(): 1019 print "Malformed expression: '%s'" % expr 1020 return None 1021 return ast 1022 1023 1024class ClassifiedTest(object): 1025 1026 def __init__(self, case, outcomes): 1027 self.case = case 1028 self.outcomes = outcomes 1029 1030 def TestsIsolates(self): 1031 return self.case.TestsIsolates() 1032 1033 1034class Configuration(object): 1035 """The parsed contents of a configuration file""" 1036 1037 def __init__(self, sections, defs): 1038 self.sections = sections 1039 self.defs = defs 1040 1041 def ClassifyTests(self, cases, env): 1042 sections = [s for s in self.sections if s.condition.Evaluate(env, self.defs)] 1043 all_rules = reduce(list.__add__, [s.rules for s in sections], []) 1044 unused_rules = set(all_rules) 1045 result = [ ] 1046 all_outcomes = set([]) 1047 for case in cases: 1048 matches = [ r for r in all_rules if r.Contains(case.path) ] 1049 outcomes = set([]) 1050 for rule in matches: 1051 outcomes = outcomes.union(rule.GetOutcomes(env, self.defs)) 1052 unused_rules.discard(rule) 1053 if not outcomes: 1054 outcomes = [PASS] 1055 case.outcomes = outcomes 1056 all_outcomes = all_outcomes.union(outcomes) 1057 result.append(ClassifiedTest(case, outcomes)) 1058 return (result, list(unused_rules), all_outcomes) 1059 1060 1061class Section(object): 1062 """A section of the configuration file. Sections are enabled or 1063 disabled prior to running the tests, based on their conditions""" 1064 1065 def __init__(self, condition): 1066 self.condition = condition 1067 self.rules = [ ] 1068 1069 def AddRule(self, rule): 1070 self.rules.append(rule) 1071 1072 1073class Rule(object): 1074 """A single rule that specifies the expected outcome for a single 1075 test.""" 1076 1077 def __init__(self, raw_path, path, value): 1078 self.raw_path = raw_path 1079 self.path = path 1080 self.value = value 1081 1082 def GetOutcomes(self, env, defs): 1083 set = self.value.GetOutcomes(env, defs) 1084 assert isinstance(set, ListSet) 1085 return set.elms 1086 1087 def Contains(self, path): 1088 if len(self.path) > len(path): 1089 return False 1090 for i in xrange(len(self.path)): 1091 if not self.path[i].match(path[i]): 1092 return False 1093 return True 1094 1095 1096HEADER_PATTERN = re.compile(r'\[([^]]+)\]') 1097RULE_PATTERN = re.compile(r'\s*([^: ]*)\s*:(.*)') 1098DEF_PATTERN = re.compile(r'^def\s*(\w+)\s*=(.*)$') 1099PREFIX_PATTERN = re.compile(r'^\s*prefix\s+([\w\_\.\-\/]+)$') 1100 1101 1102def ReadConfigurationInto(path, sections, defs): 1103 current_section = Section(Constant(True)) 1104 sections.append(current_section) 1105 prefix = [] 1106 for line in utils.ReadLinesFrom(path): 1107 header_match = HEADER_PATTERN.match(line) 1108 if header_match: 1109 condition_str = header_match.group(1).strip() 1110 condition = ParseCondition(condition_str) 1111 new_section = Section(condition) 1112 sections.append(new_section) 1113 current_section = new_section 1114 continue 1115 rule_match = RULE_PATTERN.match(line) 1116 if rule_match: 1117 path = prefix + SplitPath(rule_match.group(1).strip()) 1118 value_str = rule_match.group(2).strip() 1119 value = ParseCondition(value_str) 1120 if not value: 1121 return False 1122 current_section.AddRule(Rule(rule_match.group(1), path, value)) 1123 continue 1124 def_match = DEF_PATTERN.match(line) 1125 if def_match: 1126 name = def_match.group(1).lower() 1127 value = ParseCondition(def_match.group(2).strip()) 1128 if not value: 1129 return False 1130 defs[name] = value 1131 continue 1132 prefix_match = PREFIX_PATTERN.match(line) 1133 if prefix_match: 1134 prefix = SplitPath(prefix_match.group(1).strip()) 1135 continue 1136 print "Malformed line: '%s'." % line 1137 return False 1138 return True 1139 1140 1141# --------------- 1142# --- M a i n --- 1143# --------------- 1144 1145 1146ARCH_GUESS = utils.GuessArchitecture() 1147 1148 1149def BuildOptions(): 1150 result = optparse.OptionParser() 1151 result.add_option("-m", "--mode", help="The test modes in which to run (comma-separated)", 1152 default='release') 1153 result.add_option("-v", "--verbose", help="Verbose output", 1154 default=False, action="store_true") 1155 result.add_option("-S", dest="scons_flags", help="Flag to pass through to scons", 1156 default=[], action="append") 1157 result.add_option("-p", "--progress", 1158 help="The style of progress indicator (verbose, dots, color, mono)", 1159 choices=PROGRESS_INDICATORS.keys(), default="mono") 1160 result.add_option("--no-build", help="Don't build requirements", 1161 default=False, action="store_true") 1162 result.add_option("--build-only", help="Only build requirements, don't run the tests", 1163 default=False, action="store_true") 1164 result.add_option("--report", help="Print a summary of the tests to be run", 1165 default=False, action="store_true") 1166 result.add_option("-s", "--suite", help="A test suite", 1167 default=[], action="append") 1168 result.add_option("-t", "--timeout", help="Timeout in seconds", 1169 default=60, type="int") 1170 result.add_option("--arch", help='The architecture to run tests for', 1171 default='none') 1172 result.add_option("--snapshot", help="Run the tests with snapshot turned on", 1173 default=False, action="store_true") 1174 result.add_option("--simulator", help="Run tests with architecture simulator", 1175 default='none') 1176 result.add_option("--special-command", default=None) 1177 result.add_option("--valgrind", help="Run tests through valgrind", 1178 default=False, action="store_true") 1179 result.add_option("--cat", help="Print the source of the tests", 1180 default=False, action="store_true") 1181 result.add_option("--warn-unused", help="Report unused rules", 1182 default=False, action="store_true") 1183 result.add_option("-j", help="The number of parallel tasks to run", 1184 default=1, type="int") 1185 result.add_option("--time", help="Print timing information after running", 1186 default=False, action="store_true") 1187 result.add_option("--suppress-dialogs", help="Suppress Windows dialogs for crashing tests", 1188 dest="suppress_dialogs", default=True, action="store_true") 1189 result.add_option("--no-suppress-dialogs", help="Display Windows dialogs for crashing tests", 1190 dest="suppress_dialogs", action="store_false") 1191 result.add_option("--shell", help="Path to V8 shell", default="shell") 1192 result.add_option("--isolates", help="Whether to test isolates", default=False, action="store_true") 1193 result.add_option("--store-unexpected-output", 1194 help="Store the temporary JS files from tests that fails", 1195 dest="store_unexpected_output", default=True, action="store_true") 1196 result.add_option("--no-store-unexpected-output", 1197 help="Deletes the temporary JS files from tests that fails", 1198 dest="store_unexpected_output", action="store_false") 1199 result.add_option("--stress-only", 1200 help="Only run tests with --always-opt --stress-opt", 1201 default=False, action="store_true") 1202 result.add_option("--nostress", 1203 help="Don't run crankshaft --always-opt --stress-op test", 1204 default=False, action="store_true") 1205 result.add_option("--crankshaft", 1206 help="Run with the --crankshaft flag", 1207 default=False, action="store_true") 1208 result.add_option("--shard-count", 1209 help="Split testsuites into this number of shards", 1210 default=1, type="int") 1211 result.add_option("--shard-run", 1212 help="Run this shard from the split up tests.", 1213 default=1, type="int") 1214 result.add_option("--noprof", help="Disable profiling support", 1215 default=False) 1216 return result 1217 1218 1219def ProcessOptions(options): 1220 global VERBOSE 1221 VERBOSE = options.verbose 1222 options.mode = options.mode.split(',') 1223 for mode in options.mode: 1224 if not mode in ['debug', 'release']: 1225 print "Unknown mode %s" % mode 1226 return False 1227 if options.simulator != 'none': 1228 # Simulator argument was set. Make sure arch and simulator agree. 1229 if options.simulator != options.arch: 1230 if options.arch == 'none': 1231 options.arch = options.simulator 1232 else: 1233 print "Architecture %s does not match sim %s" %(options.arch, options.simulator) 1234 return False 1235 # Ensure that the simulator argument is handed down to scons. 1236 options.scons_flags.append("simulator=" + options.simulator) 1237 else: 1238 # If options.arch is not set by the command line and no simulator setting 1239 # was found, set the arch to the guess. 1240 if options.arch == 'none': 1241 options.arch = ARCH_GUESS 1242 options.scons_flags.append("arch=" + options.arch) 1243 if options.snapshot: 1244 options.scons_flags.append("snapshot=on") 1245 global VARIANT_FLAGS 1246 if options.stress_only: 1247 VARIANT_FLAGS = [['--stress-opt', '--always-opt']] 1248 if options.nostress: 1249 VARIANT_FLAGS = [[],['--nocrankshaft']] 1250 if options.crankshaft: 1251 if options.special_command: 1252 options.special_command += " --crankshaft" 1253 else: 1254 options.special_command = "@--crankshaft" 1255 if options.noprof: 1256 options.scons_flags.append("prof=off") 1257 options.scons_flags.append("profilingsupport=off") 1258 return True 1259 1260 1261REPORT_TEMPLATE = """\ 1262Total: %(total)i tests 1263 * %(skipped)4d tests will be skipped 1264 * %(nocrash)4d tests are expected to be flaky but not crash 1265 * %(pass)4d tests are expected to pass 1266 * %(fail_ok)4d tests are expected to fail that we won't fix 1267 * %(fail)4d tests are expected to fail that we should fix\ 1268""" 1269 1270def PrintReport(cases): 1271 def IsFlaky(o): 1272 return (PASS in o) and (FAIL in o) and (not CRASH in o) and (not OKAY in o) 1273 def IsFailOk(o): 1274 return (len(o) == 2) and (FAIL in o) and (OKAY in o) 1275 unskipped = [c for c in cases if not SKIP in c.outcomes] 1276 print REPORT_TEMPLATE % { 1277 'total': len(cases), 1278 'skipped': len(cases) - len(unskipped), 1279 'nocrash': len([t for t in unskipped if IsFlaky(t.outcomes)]), 1280 'pass': len([t for t in unskipped if list(t.outcomes) == [PASS]]), 1281 'fail_ok': len([t for t in unskipped if IsFailOk(t.outcomes)]), 1282 'fail': len([t for t in unskipped if list(t.outcomes) == [FAIL]]) 1283 } 1284 1285 1286class Pattern(object): 1287 1288 def __init__(self, pattern): 1289 self.pattern = pattern 1290 self.compiled = None 1291 1292 def match(self, str): 1293 if not self.compiled: 1294 pattern = "^" + self.pattern.replace('*', '.*') + "$" 1295 self.compiled = re.compile(pattern) 1296 return self.compiled.match(str) 1297 1298 def __str__(self): 1299 return self.pattern 1300 1301 1302def SplitPath(s): 1303 stripped = [ c.strip() for c in s.split('/') ] 1304 return [ Pattern(s) for s in stripped if len(s) > 0 ] 1305 1306 1307def GetSpecialCommandProcessor(value): 1308 if (not value) or (value.find('@') == -1): 1309 def ExpandCommand(args): 1310 return args 1311 return ExpandCommand 1312 else: 1313 pos = value.find('@') 1314 import urllib 1315 prefix = urllib.unquote(value[:pos]).split() 1316 suffix = urllib.unquote(value[pos+1:]).split() 1317 def ExpandCommand(args): 1318 return prefix + args + suffix 1319 return ExpandCommand 1320 1321 1322BUILT_IN_TESTS = ['mjsunit', 'cctest', 'message', 'preparser'] 1323 1324 1325def GetSuites(test_root): 1326 def IsSuite(path): 1327 return isdir(path) and exists(join(path, 'testcfg.py')) 1328 return [ f for f in os.listdir(test_root) if IsSuite(join(test_root, f)) ] 1329 1330 1331def FormatTime(d): 1332 millis = round(d * 1000) % 1000 1333 return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis) 1334 1335def ShardTests(tests, options): 1336 if options.shard_count < 2: 1337 return tests 1338 if options.shard_run < 1 or options.shard_run > options.shard_count: 1339 print "shard-run not a valid number, should be in [1:shard-count]" 1340 print "defaulting back to running all tests" 1341 return tests 1342 count = 0; 1343 shard = [] 1344 for test in tests: 1345 if count % options.shard_count == options.shard_run - 1: 1346 shard.append(test); 1347 count += 1 1348 return shard 1349 1350def Main(): 1351 parser = BuildOptions() 1352 (options, args) = parser.parse_args() 1353 if not ProcessOptions(options): 1354 parser.print_help() 1355 return 1 1356 1357 workspace = abspath(join(dirname(sys.argv[0]), '..')) 1358 suites = GetSuites(join(workspace, 'test')) 1359 repositories = [TestRepository(join(workspace, 'test', name)) for name in suites] 1360 repositories += [TestRepository(a) for a in options.suite] 1361 1362 root = LiteralTestSuite(repositories) 1363 if len(args) == 0: 1364 paths = [SplitPath(t) for t in BUILT_IN_TESTS] 1365 else: 1366 paths = [ ] 1367 for arg in args: 1368 path = SplitPath(arg) 1369 paths.append(path) 1370 1371 # Check for --valgrind option. If enabled, we overwrite the special 1372 # command flag with a command that uses the run-valgrind.py script. 1373 if options.valgrind: 1374 run_valgrind = join(workspace, "tools", "run-valgrind.py") 1375 options.special_command = "python -u " + run_valgrind + " @" 1376 1377 shell = abspath(options.shell) 1378 buildspace = dirname(shell) 1379 1380 context = Context(workspace, buildspace, VERBOSE, 1381 shell, 1382 options.timeout, 1383 GetSpecialCommandProcessor(options.special_command), 1384 options.suppress_dialogs, 1385 options.store_unexpected_output) 1386 # First build the required targets 1387 if not options.no_build: 1388 reqs = [ ] 1389 for path in paths: 1390 reqs += root.GetBuildRequirements(path, context) 1391 reqs = list(set(reqs)) 1392 if len(reqs) > 0: 1393 if options.j != 1: 1394 options.scons_flags += ['-j', str(options.j)] 1395 if not BuildRequirements(context, reqs, options.mode, options.scons_flags): 1396 return 1 1397 1398 # Just return if we are only building the targets for running the tests. 1399 if options.build_only: 1400 return 0 1401 1402 # Get status for tests 1403 sections = [ ] 1404 defs = { } 1405 root.GetTestStatus(context, sections, defs) 1406 config = Configuration(sections, defs) 1407 1408 # List the tests 1409 all_cases = [ ] 1410 all_unused = [ ] 1411 unclassified_tests = [ ] 1412 globally_unused_rules = None 1413 for path in paths: 1414 for mode in options.mode: 1415 env = { 1416 'mode': mode, 1417 'system': utils.GuessOS(), 1418 'arch': options.arch, 1419 'simulator': options.simulator, 1420 'crankshaft': options.crankshaft 1421 } 1422 test_list = root.ListTests([], path, context, mode, []) 1423 unclassified_tests += test_list 1424 (cases, unused_rules, all_outcomes) = config.ClassifyTests(test_list, env) 1425 if globally_unused_rules is None: 1426 globally_unused_rules = set(unused_rules) 1427 else: 1428 globally_unused_rules = globally_unused_rules.intersection(unused_rules) 1429 all_cases += ShardTests(cases, options) 1430 all_unused.append(unused_rules) 1431 1432 if options.cat: 1433 visited = set() 1434 for test in unclassified_tests: 1435 key = tuple(test.path) 1436 if key in visited: 1437 continue 1438 visited.add(key) 1439 print "--- begin source: %s ---" % test.GetLabel() 1440 source = test.GetSource().strip() 1441 print source 1442 print "--- end source: %s ---" % test.GetLabel() 1443 return 0 1444 1445 if options.warn_unused: 1446 for rule in globally_unused_rules: 1447 print "Rule for '%s' was not used." % '/'.join([str(s) for s in rule.path]) 1448 1449 if options.report: 1450 PrintReport(all_cases) 1451 1452 result = None 1453 def DoSkip(case): 1454 return SKIP in case.outcomes or SLOW in case.outcomes 1455 cases_to_run = [ c for c in all_cases if not DoSkip(c) ] 1456 if not options.isolates: 1457 cases_to_run = [c for c in cases_to_run if not c.TestsIsolates()] 1458 if len(cases_to_run) == 0: 1459 print "No tests to run." 1460 return 0 1461 else: 1462 try: 1463 start = time.time() 1464 if RunTestCases(cases_to_run, options.progress, options.j): 1465 result = 0 1466 else: 1467 result = 1 1468 duration = time.time() - start 1469 except KeyboardInterrupt: 1470 print "Interrupted" 1471 return 1 1472 1473 if options.time: 1474 # Write the times to stderr to make it easy to separate from the 1475 # test output. 1476 print 1477 sys.stderr.write("--- Total time: %s ---\n" % FormatTime(duration)) 1478 timed_tests = [ t.case for t in cases_to_run if not t.case.duration is None ] 1479 timed_tests.sort(lambda a, b: a.CompareTime(b)) 1480 index = 1 1481 for entry in timed_tests[:20]: 1482 t = FormatTime(entry.duration) 1483 sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel())) 1484 index += 1 1485 1486 return result 1487 1488 1489if __name__ == '__main__': 1490 sys.exit(Main()) 1491