1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28
29import hashlib
30import imp
31import os
32import shutil
33import sys
34import tarfile
35
36
37from testrunner.local import statusfile
38from testrunner.local import testsuite
39from testrunner.local import utils
40from testrunner.objects import testcase
41
42TEST_262_HARNESS_FILES = ["sta.js", "assert.js"]
43
44TEST_262_SUITE_PATH = ["data", "test"]
45TEST_262_HARNESS_PATH = ["data", "harness"]
46TEST_262_TOOLS_PATH = ["data", "tools", "packaging"]
47
48ALL_VARIANT_FLAGS_STRICT = dict(
49    (v, [flags + ["--use-strict"] for flags in flag_sets])
50    for v, flag_sets in testsuite.ALL_VARIANT_FLAGS.iteritems()
51)
52
53FAST_VARIANT_FLAGS_STRICT = dict(
54    (v, [flags + ["--use-strict"] for flags in flag_sets])
55    for v, flag_sets in testsuite.FAST_VARIANT_FLAGS.iteritems()
56)
57
58ALL_VARIANT_FLAGS_BOTH = dict(
59    (v, [flags for flags in testsuite.ALL_VARIANT_FLAGS[v] +
60                            ALL_VARIANT_FLAGS_STRICT[v]])
61    for v in testsuite.ALL_VARIANT_FLAGS
62)
63
64FAST_VARIANT_FLAGS_BOTH = dict(
65    (v, [flags for flags in testsuite.FAST_VARIANT_FLAGS[v] +
66                            FAST_VARIANT_FLAGS_STRICT[v]])
67    for v in testsuite.FAST_VARIANT_FLAGS
68)
69
70ALL_VARIANTS = {
71  'nostrict': testsuite.ALL_VARIANT_FLAGS,
72  'strict': ALL_VARIANT_FLAGS_STRICT,
73  'both': ALL_VARIANT_FLAGS_BOTH,
74}
75
76FAST_VARIANTS = {
77  'nostrict': testsuite.FAST_VARIANT_FLAGS,
78  'strict': FAST_VARIANT_FLAGS_STRICT,
79  'both': FAST_VARIANT_FLAGS_BOTH,
80}
81
82class Test262VariantGenerator(testsuite.VariantGenerator):
83  def GetFlagSets(self, testcase, variant):
84    if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
85      variant_flags = FAST_VARIANTS
86    else:
87      variant_flags = ALL_VARIANTS
88
89    test_record = self.suite.GetTestRecord(testcase)
90    if "noStrict" in test_record:
91      return variant_flags["nostrict"][variant]
92    if "onlyStrict" in test_record:
93      return variant_flags["strict"][variant]
94    return variant_flags["both"][variant]
95
96
97class Test262TestSuite(testsuite.TestSuite):
98
99  def __init__(self, name, root):
100    super(Test262TestSuite, self).__init__(name, root)
101    self.testroot = os.path.join(self.root, *TEST_262_SUITE_PATH)
102    self.harnesspath = os.path.join(self.root, *TEST_262_HARNESS_PATH)
103    self.harness = [os.path.join(self.harnesspath, f)
104                    for f in TEST_262_HARNESS_FILES]
105    self.harness += [os.path.join(self.root, "harness-adapt.js")]
106    self.ParseTestRecord = None
107
108  def ListTests(self, context):
109    tests = []
110    for dirname, dirs, files in os.walk(self.testroot):
111      for dotted in [x for x in dirs if x.startswith(".")]:
112        dirs.remove(dotted)
113      if context.noi18n and "intl402" in dirs:
114        dirs.remove("intl402")
115      dirs.sort()
116      files.sort()
117      for filename in files:
118        if filename.endswith(".js"):
119          fullpath = os.path.join(dirname, filename)
120          relpath = fullpath[len(self.testroot) + 1 : -3]
121          testname = relpath.replace(os.path.sep, "/")
122          case = testcase.TestCase(self, testname)
123          tests.append(case)
124    return tests
125
126  def GetFlagsForTestCase(self, testcase, context):
127    return (testcase.flags + context.mode_flags + self.harness +
128            self.GetIncludesForTest(testcase) + ["--harmony"] +
129            [os.path.join(self.testroot, testcase.path + ".js")])
130
131  def _VariantGeneratorFactory(self):
132    return Test262VariantGenerator
133
134  def LoadParseTestRecord(self):
135    if not self.ParseTestRecord:
136      root = os.path.join(self.root, *TEST_262_TOOLS_PATH)
137      f = None
138      try:
139        (f, pathname, description) = imp.find_module("parseTestRecord", [root])
140        module = imp.load_module("parseTestRecord", f, pathname, description)
141        self.ParseTestRecord = module.parseTestRecord
142      except:
143        raise ImportError("Cannot load parseTestRecord; you may need to "
144                          "--download-data for test262")
145      finally:
146        if f:
147          f.close()
148    return self.ParseTestRecord
149
150  def GetTestRecord(self, testcase):
151    if not hasattr(testcase, "test_record"):
152      ParseTestRecord = self.LoadParseTestRecord()
153      testcase.test_record = ParseTestRecord(self.GetSourceForTest(testcase),
154                                             testcase.path)
155    return testcase.test_record
156
157  def GetIncludesForTest(self, testcase):
158    test_record = self.GetTestRecord(testcase)
159    if "includes" in test_record:
160      includes = [os.path.join(self.harnesspath, f)
161                  for f in test_record["includes"]]
162    else:
163      includes = []
164    return includes
165
166  def GetSourceForTest(self, testcase):
167    filename = os.path.join(self.testroot, testcase.path + ".js")
168    with open(filename) as f:
169      return f.read()
170
171  def IsNegativeTest(self, testcase):
172    test_record = self.GetTestRecord(testcase)
173    return "negative" in test_record
174
175  def IsFailureOutput(self, output, testpath):
176    if output.exit_code != 0:
177      return True
178    return "FAILED!" in output.stdout
179
180  def HasUnexpectedOutput(self, testcase):
181    outcome = self.GetOutcome(testcase)
182    if (statusfile.FAIL_SLOPPY in testcase.outcomes and
183        "--use-strict" not in testcase.flags):
184      return outcome != statusfile.FAIL
185    return not outcome in (testcase.outcomes or [statusfile.PASS])
186
187  def DownloadData(self):
188    print "Test262 download is deprecated. It's part of DEPS."
189
190    # Clean up old directories and archive files.
191    directory_old_name = os.path.join(self.root, "data.old")
192    if os.path.exists(directory_old_name):
193      shutil.rmtree(directory_old_name)
194
195    archive_files = [f for f in os.listdir(self.root)
196                     if f.startswith("tc39-test262-")]
197    if len(archive_files) > 0:
198      print "Clobber outdated test archives ..."
199      for f in archive_files:
200        os.remove(os.path.join(self.root, f))
201
202
203def GetSuite(name, root):
204  return Test262TestSuite(name, root)
205