1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28
29import imp
30import os
31
32from . import statusfile
33from . import utils
34
35class TestSuite(object):
36
37  @staticmethod
38  def LoadTestSuite(root):
39    name = root.split(os.path.sep)[-1]
40    f = None
41    try:
42      (f, pathname, description) = imp.find_module("testcfg", [root])
43      module = imp.load_module("testcfg", f, pathname, description)
44      suite = module.GetSuite(name, root)
45    finally:
46      if f:
47        f.close()
48    return suite
49
50  def __init__(self, name, root):
51    self.name = name  # string
52    self.root = root  # string containing path
53    self.tests = None  # list of TestCase objects
54    self.rules = None  # dictionary mapping test path to list of outcomes
55    self.wildcards = None  # dictionary mapping test paths to list of outcomes
56    self.total_duration = None  # float, assigned on demand
57
58  def shell(self):
59    return "d8"
60
61  def suffix(self):
62    return ".js"
63
64  def status_file(self):
65    return "%s/%s.status" % (self.root, self.name)
66
67  # Used in the status file and for stdout printing.
68  def CommonTestName(self, testcase):
69    if utils.IsWindows():
70      return testcase.path.replace("\\", "/")
71    else:
72      return testcase.path
73
74  def ListTests(self, context):
75    raise NotImplementedError
76
77  def VariantFlags(self, testcase, default_flags):
78    if testcase.outcomes and statusfile.OnlyStandardVariant(testcase.outcomes):
79      return [[]]
80    return default_flags
81
82  def DownloadData(self):
83    pass
84
85  def ReadStatusFile(self, variables):
86    (self.rules, self.wildcards) = \
87        statusfile.ReadStatusFile(self.status_file(), variables)
88
89  def ReadTestCases(self, context):
90    self.tests = self.ListTests(context)
91
92  @staticmethod
93  def _FilterFlaky(flaky, mode):
94    return (mode == "run" and not flaky) or (mode == "skip" and flaky)
95
96  @staticmethod
97  def _FilterSlow(slow, mode):
98    return (mode == "run" and not slow) or (mode == "skip" and slow)
99
100  @staticmethod
101  def _FilterPassFail(pass_fail, mode):
102    return (mode == "run" and not pass_fail) or (mode == "skip" and pass_fail)
103
104  def FilterTestCasesByStatus(self, warn_unused_rules,
105                              flaky_tests="dontcare",
106                              slow_tests="dontcare",
107                              pass_fail_tests="dontcare"):
108    filtered = []
109    used_rules = set()
110    for t in self.tests:
111      flaky = False
112      slow = False
113      pass_fail = False
114      testname = self.CommonTestName(t)
115      if testname in self.rules:
116        used_rules.add(testname)
117        # Even for skipped tests, as the TestCase object stays around and
118        # PrintReport() uses it.
119        t.outcomes = self.rules[testname]
120        if statusfile.DoSkip(t.outcomes):
121          continue  # Don't add skipped tests to |filtered|.
122        flaky = statusfile.IsFlaky(t.outcomes)
123        slow = statusfile.IsSlow(t.outcomes)
124        pass_fail = statusfile.IsPassOrFail(t.outcomes)
125      skip = False
126      for rule in self.wildcards:
127        assert rule[-1] == '*'
128        if testname.startswith(rule[:-1]):
129          used_rules.add(rule)
130          t.outcomes = self.wildcards[rule]
131          if statusfile.DoSkip(t.outcomes):
132            skip = True
133            break  # "for rule in self.wildcards"
134          flaky = flaky or statusfile.IsFlaky(t.outcomes)
135          slow = slow or statusfile.IsSlow(t.outcomes)
136          pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes)
137      if (skip or self._FilterFlaky(flaky, flaky_tests)
138          or self._FilterSlow(slow, slow_tests)
139          or self._FilterPassFail(pass_fail, pass_fail_tests)):
140        continue  # "for t in self.tests"
141      filtered.append(t)
142    self.tests = filtered
143
144    if not warn_unused_rules:
145      return
146
147    for rule in self.rules:
148      if rule not in used_rules:
149        print("Unused rule: %s -> %s" % (rule, self.rules[rule]))
150    for rule in self.wildcards:
151      if rule not in used_rules:
152        print("Unused rule: %s -> %s" % (rule, self.wildcards[rule]))
153
154  def FilterTestCasesByArgs(self, args):
155    filtered = []
156    filtered_args = []
157    for a in args:
158      argpath = a.split(os.path.sep)
159      if argpath[0] != self.name:
160        continue
161      if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'):
162        return  # Don't filter, run all tests in this suite.
163      path = os.path.sep.join(argpath[1:])
164      if path[-1] == '*':
165        path = path[:-1]
166      filtered_args.append(path)
167    for t in self.tests:
168      for a in filtered_args:
169        if t.path.startswith(a):
170          filtered.append(t)
171          break
172    self.tests = filtered
173
174  def GetFlagsForTestCase(self, testcase, context):
175    raise NotImplementedError
176
177  def GetSourceForTest(self, testcase):
178    return "(no source available)"
179
180  def IsFailureOutput(self, output, testpath):
181    return output.exit_code != 0
182
183  def IsNegativeTest(self, testcase):
184    return False
185
186  def HasFailed(self, testcase):
187    execution_failed = self.IsFailureOutput(testcase.output, testcase.path)
188    if self.IsNegativeTest(testcase):
189      return not execution_failed
190    else:
191      return execution_failed
192
193  def HasUnexpectedOutput(self, testcase):
194    if testcase.output.HasCrashed():
195      outcome = statusfile.CRASH
196    elif testcase.output.HasTimedOut():
197      outcome = statusfile.TIMEOUT
198    elif self.HasFailed(testcase):
199      outcome = statusfile.FAIL
200    else:
201      outcome = statusfile.PASS
202    if not testcase.outcomes:
203      return outcome != statusfile.PASS
204    return not outcome in testcase.outcomes
205
206  def StripOutputForTransmit(self, testcase):
207    if not self.HasUnexpectedOutput(testcase):
208      testcase.output.stdout = ""
209      testcase.output.stderr = ""
210
211  def CalculateTotalDuration(self):
212    self.total_duration = 0.0
213    for t in self.tests:
214      self.total_duration += t.duration
215    return self.total_duration
216