1b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# Copyright 2013 the V8 project authors. All rights reserved. 2b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# Redistribution and use in source and binary forms, with or without 3b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# modification, are permitted provided that the following conditions are 4b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# met: 5b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# 6b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# * Redistributions of source code must retain the above copyright 7b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# notice, this list of conditions and the following disclaimer. 8b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# * Redistributions in binary form must reproduce the above 9b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# copyright notice, this list of conditions and the following 10b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# disclaimer in the documentation and/or other materials provided 11b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# with the distribution. 12b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# * Neither the name of Google Inc. nor the names of its 13b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# contributors may be used to endorse or promote products derived 14b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# from this software without specific prior written permission. 15b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# 16b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 28b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport itertools 29b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport os 30b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport re 31b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 32b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom testrunner.local import testsuite 33b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom testrunner.objects import testcase 34b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 35b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochFLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)") 36b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochFILES_PATTERN = re.compile(r"//\s+Files:(.*)") 37b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochSELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME") 38b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 39b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 40b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# TODO (machenbach): Share commonalities with mjstest. 41b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochclass WebkitTestSuite(testsuite.TestSuite): 42b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 43b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def __init__(self, name, root): 44b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch super(WebkitTestSuite, self).__init__(name, root) 45b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 46b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def ListTests(self, context): 47b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch tests = [] 48b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for dirname, dirs, files in os.walk(self.root): 49b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for dotted in [x for x in dirs if x.startswith('.')]: 50b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch dirs.remove(dotted) 51b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if 'resources' in dirs: 52b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch dirs.remove('resources') 53b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 54b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch dirs.sort() 55b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch files.sort() 56b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for filename in files: 57b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if filename.endswith(".js"): 58b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch testname = os.path.join(dirname[len(self.root) + 1:], filename[:-3]) 59b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch test = testcase.TestCase(self, testname) 60b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch tests.append(test) 61b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return tests 62b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 63b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def GetFlagsForTestCase(self, testcase, context): 64b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch source = self.GetSourceForTest(testcase) 65b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch flags = [] + context.mode_flags 66b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch flags_match = re.findall(FLAGS_PATTERN, source) 67b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for match in flags_match: 68b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch flags += match.strip().split() 69b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 70b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch files_list = [] # List of file names to append to command arguments. 71b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch files_match = FILES_PATTERN.search(source); 72b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Accept several lines of 'Files:'. 73b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while True: 74b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if files_match: 75b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch files_list += files_match.group(1).strip().split() 76b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch files_match = FILES_PATTERN.search(source, files_match.end()) 77b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch else: 78b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch break 79b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch files = [ os.path.normpath(os.path.join(self.root, '..', '..', f)) 80b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for f in files_list ] 81b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch testfilename = os.path.join(self.root, testcase.path + self.suffix()) 82b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if SELF_SCRIPT_PATTERN.search(source): 83b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch env = ["-e", "TEST_FILE_NAME=\"%s\"" % testfilename.replace("\\", "\\\\")] 84b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch files = env + files 85b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch files.append(os.path.join(self.root, "resources/standalone-pre.js")) 86b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch files.append(testfilename) 87b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch files.append(os.path.join(self.root, "resources/standalone-post.js")) 88b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 89b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch flags += files 90b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if context.isolates: 91b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch flags.append("--isolate") 92b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch flags += files 93b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 94b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return testcase.flags + flags 95b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 96b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def GetSourceForTest(self, testcase): 97b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch filename = os.path.join(self.root, testcase.path + self.suffix()) 98b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch with open(filename) as f: 99b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return f.read() 100b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 101b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # TODO(machenbach): Share with test/message/testcfg.py 102b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def _IgnoreLine(self, string): 103b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch """Ignore empty lines, valgrind output and Android output.""" 104b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not string: return True 105b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return (string.startswith("==") or string.startswith("**") or 106b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch string.startswith("ANDROID") or 107b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # These five patterns appear in normal Native Client output. 108b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch string.startswith("DEBUG MODE ENABLED") or 109b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch string.startswith("tools/nacl-run.py") or 110b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch string.find("BYPASSING ALL ACL CHECKS") > 0 or 111b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch string.find("Native Client module will be loaded") > 0 or 112b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch string.find("NaClHostDescOpen:") > 0) 113b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 114b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def IsFailureOutput(self, output, testpath): 115b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if super(WebkitTestSuite, self).IsFailureOutput(output, testpath): 116b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return True 117b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch file_name = os.path.join(self.root, testpath) + "-expected.txt" 118b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch with file(file_name, "r") as expected: 119b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch expected_lines = expected.readlines() 120b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 121b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def ExpIterator(): 122b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for line in expected_lines: 123b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if line.startswith("#") or not line.strip(): continue 124b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch yield line.strip() 125b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 126b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def ActIterator(lines): 127b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for line in lines: 128b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if self._IgnoreLine(line.strip()): continue 129b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch yield line.strip() 130b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 131b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def ActBlockIterator(): 132b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch """Iterates over blocks of actual output lines.""" 133b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch lines = output.stdout.splitlines() 134b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch start_index = 0 135b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch found_eqeq = False 136b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for index, line in enumerate(lines): 137b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # If a stress test separator is found: 138b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if line.startswith("=="): 139b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Iterate over all lines before a separator except the first. 140b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not found_eqeq: 141b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch found_eqeq = True 142b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch else: 143b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch yield ActIterator(lines[start_index:index]) 144b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # The next block of ouput lines starts after the separator. 145b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch start_index = index + 1 146b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Iterate over complete output if no separator was found. 147b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if not found_eqeq: 148b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch yield ActIterator(lines) 149b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 150b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for act_iterator in ActBlockIterator(): 151b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for (expected, actual) in itertools.izip_longest( 152b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ExpIterator(), act_iterator, fillvalue=''): 153b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if expected != actual: 154b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return True 155b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return False 156b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 157b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 158b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochdef GetSuite(name, root): 159b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return WebkitTestSuite(name, root) 160