1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28
29import imp
30import os
31
32from . import commands
33from . import statusfile
34from . import utils
35from ..objects import testcase
36
37# Use this to run several variants of the tests.
38ALL_VARIANT_FLAGS = {
39  "default": [[]],
40  "stress": [["--stress-opt", "--always-opt"]],
41  "turbofan": [["--turbo"]],
42  "turbofan_opt": [["--turbo", "--always-opt"]],
43  "nocrankshaft": [["--nocrankshaft"]],
44  "ignition": [["--ignition"]],
45  "ignition_turbofan": [["--ignition", "--turbo", "--turbo-from-bytecode"]],
46  "preparser": [["--min-preparse-length=0"]],
47}
48
49# FAST_VARIANTS implies no --always-opt.
50FAST_VARIANT_FLAGS = {
51  "default": [[]],
52  "stress": [["--stress-opt"]],
53  "turbofan": [["--turbo"]],
54  "nocrankshaft": [["--nocrankshaft"]],
55  "ignition": [["--ignition"]],
56  "ignition_turbofan": [["--ignition", "--turbo", "--turbo-from-bytecode"]],
57  "preparser": [["--min-preparse-length=0"]],
58}
59
60ALL_VARIANTS = set(["default", "stress", "turbofan", "turbofan_opt",
61                    "nocrankshaft", "ignition", "ignition_turbofan",
62                    "preparser"])
63FAST_VARIANTS = set(["default", "turbofan"])
64STANDARD_VARIANT = set(["default"])
65IGNITION_VARIANT = set(["ignition"])
66
67
68class VariantGenerator(object):
69  def __init__(self, suite, variants):
70    self.suite = suite
71    self.all_variants = ALL_VARIANTS & variants
72    self.fast_variants = FAST_VARIANTS & variants
73    self.standard_variant = STANDARD_VARIANT & variants
74
75  def FilterVariantsByTest(self, testcase):
76    result = self.all_variants
77    if testcase.outcomes:
78      if statusfile.OnlyStandardVariant(testcase.outcomes):
79        return self.standard_variant
80      if statusfile.OnlyFastVariants(testcase.outcomes):
81        result = self.fast_variants
82      if statusfile.NoIgnitionVariant(testcase.outcomes):
83        result = result - IGNITION_VARIANT
84    return result
85
86  def GetFlagSets(self, testcase, variant):
87    if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
88      return FAST_VARIANT_FLAGS[variant]
89    else:
90      return ALL_VARIANT_FLAGS[variant]
91
92
93class TestSuite(object):
94
95  @staticmethod
96  def LoadTestSuite(root, global_init=True):
97    name = root.split(os.path.sep)[-1]
98    f = None
99    try:
100      (f, pathname, description) = imp.find_module("testcfg", [root])
101      module = imp.load_module("testcfg", f, pathname, description)
102      return module.GetSuite(name, root)
103    except ImportError:
104      # Use default if no testcfg is present.
105      return GoogleTestSuite(name, root)
106    finally:
107      if f:
108        f.close()
109
110  def __init__(self, name, root):
111    # Note: This might be called concurrently from different processes.
112    self.name = name  # string
113    self.root = root  # string containing path
114    self.tests = None  # list of TestCase objects
115    self.rules = None  # dictionary mapping test path to list of outcomes
116    self.wildcards = None  # dictionary mapping test paths to list of outcomes
117    self.total_duration = None  # float, assigned on demand
118
119  def shell(self):
120    return "d8"
121
122  def suffix(self):
123    return ".js"
124
125  def status_file(self):
126    return "%s/%s.status" % (self.root, self.name)
127
128  # Used in the status file and for stdout printing.
129  def CommonTestName(self, testcase):
130    if utils.IsWindows():
131      return testcase.path.replace("\\", "/")
132    else:
133      return testcase.path
134
135  def ListTests(self, context):
136    raise NotImplementedError
137
138  def _VariantGeneratorFactory(self):
139    """The variant generator class to be used."""
140    return VariantGenerator
141
142  def CreateVariantGenerator(self, variants):
143    """Return a generator for the testing variants of this suite.
144
145    Args:
146      variants: List of variant names to be run as specified by the test
147                runner.
148    Returns: An object of type VariantGenerator.
149    """
150    return self._VariantGeneratorFactory()(self, set(variants))
151
152  def DownloadData(self):
153    pass
154
155  def ReadStatusFile(self, variables):
156    (self.rules, self.wildcards) = \
157        statusfile.ReadStatusFile(self.status_file(), variables)
158
159  def ReadTestCases(self, context):
160    self.tests = self.ListTests(context)
161
162  @staticmethod
163  def _FilterSlow(slow, mode):
164    return (mode == "run" and not slow) or (mode == "skip" and slow)
165
166  @staticmethod
167  def _FilterPassFail(pass_fail, mode):
168    return (mode == "run" and not pass_fail) or (mode == "skip" and pass_fail)
169
170  def FilterTestCasesByStatus(self, warn_unused_rules,
171                              slow_tests="dontcare",
172                              pass_fail_tests="dontcare"):
173    filtered = []
174    used_rules = set()
175    for t in self.tests:
176      slow = False
177      pass_fail = False
178      testname = self.CommonTestName(t)
179      if testname in self.rules:
180        used_rules.add(testname)
181        # Even for skipped tests, as the TestCase object stays around and
182        # PrintReport() uses it.
183        t.outcomes = self.rules[testname]
184        if statusfile.DoSkip(t.outcomes):
185          continue  # Don't add skipped tests to |filtered|.
186        for outcome in t.outcomes:
187          if outcome.startswith('Flags: '):
188            t.flags += outcome[7:].split()
189        slow = statusfile.IsSlow(t.outcomes)
190        pass_fail = statusfile.IsPassOrFail(t.outcomes)
191      skip = False
192      for rule in self.wildcards:
193        assert rule[-1] == '*'
194        if testname.startswith(rule[:-1]):
195          used_rules.add(rule)
196          t.outcomes |= self.wildcards[rule]
197          if statusfile.DoSkip(t.outcomes):
198            skip = True
199            break  # "for rule in self.wildcards"
200          slow = slow or statusfile.IsSlow(t.outcomes)
201          pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes)
202      if (skip
203          or self._FilterSlow(slow, slow_tests)
204          or self._FilterPassFail(pass_fail, pass_fail_tests)):
205        continue  # "for t in self.tests"
206      filtered.append(t)
207    self.tests = filtered
208
209    if not warn_unused_rules:
210      return
211
212    for rule in self.rules:
213      if rule not in used_rules:
214        print("Unused rule: %s -> %s" % (rule, self.rules[rule]))
215    for rule in self.wildcards:
216      if rule not in used_rules:
217        print("Unused rule: %s -> %s" % (rule, self.wildcards[rule]))
218
219  def FilterTestCasesByArgs(self, args):
220    """Filter test cases based on command-line arguments.
221
222    An argument with an asterisk in the end will match all test cases
223    that have the argument as a prefix. Without asterisk, only exact matches
224    will be used with the exeption of the test-suite name as argument.
225    """
226    filtered = []
227    globs = []
228    exact_matches = []
229    for a in args:
230      argpath = a.split('/')
231      if argpath[0] != self.name:
232        continue
233      if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'):
234        return  # Don't filter, run all tests in this suite.
235      path = '/'.join(argpath[1:])
236      if path[-1] == '*':
237        path = path[:-1]
238        globs.append(path)
239      else:
240        exact_matches.append(path)
241    for t in self.tests:
242      for a in globs:
243        if t.path.startswith(a):
244          filtered.append(t)
245          break
246      for a in exact_matches:
247        if t.path == a:
248          filtered.append(t)
249          break
250    self.tests = filtered
251
252  def GetFlagsForTestCase(self, testcase, context):
253    raise NotImplementedError
254
255  def GetSourceForTest(self, testcase):
256    return "(no source available)"
257
258  def IsFailureOutput(self, testcase):
259    return testcase.output.exit_code != 0
260
261  def IsNegativeTest(self, testcase):
262    return False
263
264  def HasFailed(self, testcase):
265    execution_failed = self.IsFailureOutput(testcase)
266    if self.IsNegativeTest(testcase):
267      return not execution_failed
268    else:
269      return execution_failed
270
271  def GetOutcome(self, testcase):
272    if testcase.output.HasCrashed():
273      return statusfile.CRASH
274    elif testcase.output.HasTimedOut():
275      return statusfile.TIMEOUT
276    elif self.HasFailed(testcase):
277      return statusfile.FAIL
278    else:
279      return statusfile.PASS
280
281  def HasUnexpectedOutput(self, testcase):
282    outcome = self.GetOutcome(testcase)
283    return not outcome in (testcase.outcomes or [statusfile.PASS])
284
285  def StripOutputForTransmit(self, testcase):
286    if not self.HasUnexpectedOutput(testcase):
287      testcase.output.stdout = ""
288      testcase.output.stderr = ""
289
290  def CalculateTotalDuration(self):
291    self.total_duration = 0.0
292    for t in self.tests:
293      self.total_duration += t.duration
294    return self.total_duration
295
296
297class StandardVariantGenerator(VariantGenerator):
298  def FilterVariantsByTest(self, testcase):
299    return self.standard_variant
300
301
302class GoogleTestSuite(TestSuite):
303  def __init__(self, name, root):
304    super(GoogleTestSuite, self).__init__(name, root)
305
306  def ListTests(self, context):
307    shell = os.path.abspath(os.path.join(context.shell_dir, self.shell()))
308    if utils.IsWindows():
309      shell += ".exe"
310    output = commands.Execute(context.command_prefix +
311                              [shell, "--gtest_list_tests"] +
312                              context.extra_flags)
313    if output.exit_code != 0:
314      print output.stdout
315      print output.stderr
316      raise Exception("Test executable failed to list the tests.")
317    tests = []
318    test_case = ''
319    for line in output.stdout.splitlines():
320      test_desc = line.strip().split()[0]
321      if test_desc.endswith('.'):
322        test_case = test_desc
323      elif test_case and test_desc:
324        test = testcase.TestCase(self, test_case + test_desc)
325        tests.append(test)
326    tests.sort(key=lambda t: t.path)
327    return tests
328
329  def GetFlagsForTestCase(self, testcase, context):
330    return (testcase.flags + ["--gtest_filter=" + testcase.path] +
331            ["--gtest_random_seed=%s" % context.random_seed] +
332            ["--gtest_print_time=0"] +
333            context.mode_flags)
334
335  def _VariantGeneratorFactory(self):
336    return StandardVariantGenerator
337
338  def shell(self):
339    return self.name
340