1# Copyright 2012 the V8 project authors. All rights reserved. 2# Redistribution and use in source and binary forms, with or without 3# modification, are permitted provided that the following conditions are 4# met: 5# 6# * Redistributions of source code must retain the above copyright 7# notice, this list of conditions and the following disclaimer. 8# * Redistributions in binary form must reproduce the above 9# copyright notice, this list of conditions and the following 10# disclaimer in the documentation and/or other materials provided 11# with the distribution. 12# * Neither the name of Google Inc. nor the names of its 13# contributors may be used to endorse or promote products derived 14# from this software without specific prior written permission. 15# 16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28 29import hashlib 30import imp 31import os 32import shutil 33import sys 34import tarfile 35 36 37from testrunner.local import statusfile 38from testrunner.local import testsuite 39from testrunner.local import utils 40from testrunner.objects import testcase 41 42TEST_262_HARNESS_FILES = ["sta.js", "assert.js"] 43 44TEST_262_SUITE_PATH = ["data", "test"] 45TEST_262_HARNESS_PATH = ["data", "harness"] 46TEST_262_TOOLS_PATH = ["data", "tools", "packaging"] 47 48ALL_VARIANT_FLAGS_STRICT = dict( 49 (v, [flags + ["--use-strict"] for flags in flag_sets]) 50 for v, flag_sets in testsuite.ALL_VARIANT_FLAGS.iteritems() 51) 52 53FAST_VARIANT_FLAGS_STRICT = dict( 54 (v, [flags + ["--use-strict"] for flags in flag_sets]) 55 for v, flag_sets in testsuite.FAST_VARIANT_FLAGS.iteritems() 56) 57 58ALL_VARIANT_FLAGS_BOTH = dict( 59 (v, [flags for flags in testsuite.ALL_VARIANT_FLAGS[v] + 60 ALL_VARIANT_FLAGS_STRICT[v]]) 61 for v in testsuite.ALL_VARIANT_FLAGS 62) 63 64FAST_VARIANT_FLAGS_BOTH = dict( 65 (v, [flags for flags in testsuite.FAST_VARIANT_FLAGS[v] + 66 FAST_VARIANT_FLAGS_STRICT[v]]) 67 for v in testsuite.FAST_VARIANT_FLAGS 68) 69 70ALL_VARIANTS = { 71 'nostrict': testsuite.ALL_VARIANT_FLAGS, 72 'strict': ALL_VARIANT_FLAGS_STRICT, 73 'both': ALL_VARIANT_FLAGS_BOTH, 74} 75 76FAST_VARIANTS = { 77 'nostrict': testsuite.FAST_VARIANT_FLAGS, 78 'strict': FAST_VARIANT_FLAGS_STRICT, 79 'both': FAST_VARIANT_FLAGS_BOTH, 80} 81 82class Test262VariantGenerator(testsuite.VariantGenerator): 83 def GetFlagSets(self, testcase, variant): 84 if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes): 85 variant_flags = FAST_VARIANTS 86 else: 87 variant_flags = ALL_VARIANTS 88 89 test_record = self.suite.GetTestRecord(testcase) 90 if "noStrict" in test_record: 91 return variant_flags["nostrict"][variant] 92 if "onlyStrict" in test_record: 93 return variant_flags["strict"][variant] 94 return variant_flags["both"][variant] 95 96 97class Test262TestSuite(testsuite.TestSuite): 98 99 def __init__(self, name, root): 100 super(Test262TestSuite, self).__init__(name, root) 101 self.testroot = os.path.join(self.root, *TEST_262_SUITE_PATH) 102 self.harnesspath = os.path.join(self.root, *TEST_262_HARNESS_PATH) 103 self.harness = [os.path.join(self.harnesspath, f) 104 for f in TEST_262_HARNESS_FILES] 105 self.harness += [os.path.join(self.root, "harness-adapt.js")] 106 self.ParseTestRecord = None 107 108 def ListTests(self, context): 109 tests = [] 110 for dirname, dirs, files in os.walk(self.testroot): 111 for dotted in [x for x in dirs if x.startswith(".")]: 112 dirs.remove(dotted) 113 if context.noi18n and "intl402" in dirs: 114 dirs.remove("intl402") 115 dirs.sort() 116 files.sort() 117 for filename in files: 118 if filename.endswith(".js"): 119 fullpath = os.path.join(dirname, filename) 120 relpath = fullpath[len(self.testroot) + 1 : -3] 121 testname = relpath.replace(os.path.sep, "/") 122 case = testcase.TestCase(self, testname) 123 tests.append(case) 124 return tests 125 126 def GetFlagsForTestCase(self, testcase, context): 127 return (testcase.flags + context.mode_flags + self.harness + 128 self.GetIncludesForTest(testcase) + ["--harmony"] + 129 [os.path.join(self.testroot, testcase.path + ".js")]) 130 131 def _VariantGeneratorFactory(self): 132 return Test262VariantGenerator 133 134 def LoadParseTestRecord(self): 135 if not self.ParseTestRecord: 136 root = os.path.join(self.root, *TEST_262_TOOLS_PATH) 137 f = None 138 try: 139 (f, pathname, description) = imp.find_module("parseTestRecord", [root]) 140 module = imp.load_module("parseTestRecord", f, pathname, description) 141 self.ParseTestRecord = module.parseTestRecord 142 except: 143 raise ImportError("Cannot load parseTestRecord; you may need to " 144 "--download-data for test262") 145 finally: 146 if f: 147 f.close() 148 return self.ParseTestRecord 149 150 def GetTestRecord(self, testcase): 151 if not hasattr(testcase, "test_record"): 152 ParseTestRecord = self.LoadParseTestRecord() 153 testcase.test_record = ParseTestRecord(self.GetSourceForTest(testcase), 154 testcase.path) 155 return testcase.test_record 156 157 def GetIncludesForTest(self, testcase): 158 test_record = self.GetTestRecord(testcase) 159 if "includes" in test_record: 160 includes = [os.path.join(self.harnesspath, f) 161 for f in test_record["includes"]] 162 else: 163 includes = [] 164 return includes 165 166 def GetSourceForTest(self, testcase): 167 filename = os.path.join(self.testroot, testcase.path + ".js") 168 with open(filename) as f: 169 return f.read() 170 171 def IsNegativeTest(self, testcase): 172 test_record = self.GetTestRecord(testcase) 173 return "negative" in test_record 174 175 def IsFailureOutput(self, output, testpath): 176 if output.exit_code != 0: 177 return True 178 return "FAILED!" in output.stdout 179 180 def HasUnexpectedOutput(self, testcase): 181 outcome = self.GetOutcome(testcase) 182 if (statusfile.FAIL_SLOPPY in testcase.outcomes and 183 "--use-strict" not in testcase.flags): 184 return outcome != statusfile.FAIL 185 return not outcome in (testcase.outcomes or [statusfile.PASS]) 186 187 def DownloadData(self): 188 print "Test262 download is deprecated. It's part of DEPS." 189 190 # Clean up old directories and archive files. 191 directory_old_name = os.path.join(self.root, "data.old") 192 if os.path.exists(directory_old_name): 193 shutil.rmtree(directory_old_name) 194 195 archive_files = [f for f in os.listdir(self.root) 196 if f.startswith("tc39-test262-")] 197 if len(archive_files) > 0: 198 print "Clobber outdated test archives ..." 199 for f in archive_files: 200 os.remove(os.path.join(self.root, f)) 201 202 203def GetSuite(name, root): 204 return Test262TestSuite(name, root) 205