1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28
29import imp
30import os
31
32from . import commands
33from . import statusfile
34from . import utils
35from ..objects import testcase
36
37class TestSuite(object):
38
39  @staticmethod
40  def LoadTestSuite(root):
41    name = root.split(os.path.sep)[-1]
42    f = None
43    try:
44      (f, pathname, description) = imp.find_module("testcfg", [root])
45      module = imp.load_module("testcfg", f, pathname, description)
46      return module.GetSuite(name, root)
47    except:
48      # Use default if no testcfg is present.
49      return GoogleTestSuite(name, root)
50    finally:
51      if f:
52        f.close()
53
54  def __init__(self, name, root):
55    self.name = name  # string
56    self.root = root  # string containing path
57    self.tests = None  # list of TestCase objects
58    self.rules = None  # dictionary mapping test path to list of outcomes
59    self.wildcards = None  # dictionary mapping test paths to list of outcomes
60    self.total_duration = None  # float, assigned on demand
61
62  def shell(self):
63    return "d8"
64
65  def suffix(self):
66    return ".js"
67
68  def status_file(self):
69    return "%s/%s.status" % (self.root, self.name)
70
71  # Used in the status file and for stdout printing.
72  def CommonTestName(self, testcase):
73    if utils.IsWindows():
74      return testcase.path.replace("\\", "/")
75    else:
76      return testcase.path
77
78  def ListTests(self, context):
79    raise NotImplementedError
80
81  def VariantFlags(self, testcase, default_flags):
82    if testcase.outcomes and statusfile.OnlyStandardVariant(testcase.outcomes):
83      return [[]]
84    return default_flags
85
86  def DownloadData(self):
87    pass
88
89  def ReadStatusFile(self, variables):
90    (self.rules, self.wildcards) = \
91        statusfile.ReadStatusFile(self.status_file(), variables)
92
93  def ReadTestCases(self, context):
94    self.tests = self.ListTests(context)
95
96  @staticmethod
97  def _FilterFlaky(flaky, mode):
98    return (mode == "run" and not flaky) or (mode == "skip" and flaky)
99
100  @staticmethod
101  def _FilterSlow(slow, mode):
102    return (mode == "run" and not slow) or (mode == "skip" and slow)
103
104  @staticmethod
105  def _FilterPassFail(pass_fail, mode):
106    return (mode == "run" and not pass_fail) or (mode == "skip" and pass_fail)
107
108  def FilterTestCasesByStatus(self, warn_unused_rules,
109                              flaky_tests="dontcare",
110                              slow_tests="dontcare",
111                              pass_fail_tests="dontcare"):
112    filtered = []
113    used_rules = set()
114    for t in self.tests:
115      flaky = False
116      slow = False
117      pass_fail = False
118      testname = self.CommonTestName(t)
119      if testname in self.rules:
120        used_rules.add(testname)
121        # Even for skipped tests, as the TestCase object stays around and
122        # PrintReport() uses it.
123        t.outcomes = self.rules[testname]
124        if statusfile.DoSkip(t.outcomes):
125          continue  # Don't add skipped tests to |filtered|.
126        flaky = statusfile.IsFlaky(t.outcomes)
127        slow = statusfile.IsSlow(t.outcomes)
128        pass_fail = statusfile.IsPassOrFail(t.outcomes)
129      skip = False
130      for rule in self.wildcards:
131        assert rule[-1] == '*'
132        if testname.startswith(rule[:-1]):
133          used_rules.add(rule)
134          t.outcomes = self.wildcards[rule]
135          if statusfile.DoSkip(t.outcomes):
136            skip = True
137            break  # "for rule in self.wildcards"
138          flaky = flaky or statusfile.IsFlaky(t.outcomes)
139          slow = slow or statusfile.IsSlow(t.outcomes)
140          pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes)
141      if (skip or self._FilterFlaky(flaky, flaky_tests)
142          or self._FilterSlow(slow, slow_tests)
143          or self._FilterPassFail(pass_fail, pass_fail_tests)):
144        continue  # "for t in self.tests"
145      filtered.append(t)
146    self.tests = filtered
147
148    if not warn_unused_rules:
149      return
150
151    for rule in self.rules:
152      if rule not in used_rules:
153        print("Unused rule: %s -> %s" % (rule, self.rules[rule]))
154    for rule in self.wildcards:
155      if rule not in used_rules:
156        print("Unused rule: %s -> %s" % (rule, self.wildcards[rule]))
157
158  def FilterTestCasesByArgs(self, args):
159    filtered = []
160    filtered_args = []
161    for a in args:
162      argpath = a.split(os.path.sep)
163      if argpath[0] != self.name:
164        continue
165      if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'):
166        return  # Don't filter, run all tests in this suite.
167      path = os.path.sep.join(argpath[1:])
168      if path[-1] == '*':
169        path = path[:-1]
170      filtered_args.append(path)
171    for t in self.tests:
172      for a in filtered_args:
173        if t.path.startswith(a):
174          filtered.append(t)
175          break
176    self.tests = filtered
177
178  def GetFlagsForTestCase(self, testcase, context):
179    raise NotImplementedError
180
181  def GetSourceForTest(self, testcase):
182    return "(no source available)"
183
184  def IsFailureOutput(self, output, testpath):
185    return output.exit_code != 0
186
187  def IsNegativeTest(self, testcase):
188    return False
189
190  def HasFailed(self, testcase):
191    execution_failed = self.IsFailureOutput(testcase.output, testcase.path)
192    if self.IsNegativeTest(testcase):
193      return not execution_failed
194    else:
195      return execution_failed
196
197  def GetOutcome(self, testcase):
198    if testcase.output.HasCrashed():
199      return statusfile.CRASH
200    elif testcase.output.HasTimedOut():
201      return statusfile.TIMEOUT
202    elif self.HasFailed(testcase):
203      return statusfile.FAIL
204    else:
205      return statusfile.PASS
206
207  def HasUnexpectedOutput(self, testcase):
208    outcome = self.GetOutcome(testcase)
209    return not outcome in (testcase.outcomes or [statusfile.PASS])
210
211  def StripOutputForTransmit(self, testcase):
212    if not self.HasUnexpectedOutput(testcase):
213      testcase.output.stdout = ""
214      testcase.output.stderr = ""
215
216  def CalculateTotalDuration(self):
217    self.total_duration = 0.0
218    for t in self.tests:
219      self.total_duration += t.duration
220    return self.total_duration
221
222
223class GoogleTestSuite(TestSuite):
224  def __init__(self, name, root):
225    super(GoogleTestSuite, self).__init__(name, root)
226
227  def ListTests(self, context):
228    shell = os.path.abspath(os.path.join(context.shell_dir, self.shell()))
229    if utils.IsWindows():
230      shell += ".exe"
231    output = commands.Execute(context.command_prefix +
232                              [shell, "--gtest_list_tests"] +
233                              context.extra_flags)
234    if output.exit_code != 0:
235      print output.stdout
236      print output.stderr
237      return []
238    tests = []
239    test_case = ''
240    for line in output.stdout.splitlines():
241      test_desc = line.strip().split()[0]
242      if test_desc.endswith('.'):
243        test_case = test_desc
244      elif test_case and test_desc:
245        test = testcase.TestCase(self, test_case + test_desc, dependency=None)
246        tests.append(test)
247    tests.sort()
248    return tests
249
250  def GetFlagsForTestCase(self, testcase, context):
251    return (testcase.flags + ["--gtest_filter=" + testcase.path] +
252            ["--gtest_random_seed=%s" % context.random_seed] +
253            ["--gtest_print_time=0"] +
254            context.mode_flags)
255
256  def shell(self):
257    return self.name
258