1#===----------------------------------------------------------------------===##
2#
3#                     The LLVM Compiler Infrastructure
4#
5# This file is dual licensed under the MIT and the University of Illinois Open
6# Source Licenses. See LICENSE.TXT for details.
7#
8#===----------------------------------------------------------------------===##
9
10import copy
11import errno
12import os
13import time
14import random
15
16import lit.Test        # pylint: disable=import-error
17import lit.TestRunner  # pylint: disable=import-error
18from lit.TestRunner import ParserKind, IntegratedTestKeywordParser  \
19    # pylint: disable=import-error
20
21from libcxx.test.executor import LocalExecutor as LocalExecutor
22import libcxx.util
23
24
25class LibcxxTestFormat(object):
26    """
27    Custom test format handler for use with the test format use by libc++.
28
29    Tests fall into two categories:
30      FOO.pass.cpp - Executable test which should compile, run, and exit with
31                     code 0.
32      FOO.fail.cpp - Negative test case which is expected to fail compilation.
33      FOO.sh.cpp   - A test that uses LIT's ShTest format.
34    """
35
36    def __init__(self, cxx, use_verify_for_fail, execute_external,
37                 executor, exec_env):
38        self.cxx = copy.deepcopy(cxx)
39        self.use_verify_for_fail = use_verify_for_fail
40        self.execute_external = execute_external
41        self.executor = executor
42        self.exec_env = dict(exec_env)
43
44    @staticmethod
45    def _make_custom_parsers():
46        return [
47            IntegratedTestKeywordParser('FLAKY_TEST.', ParserKind.TAG,
48                                        initial_value=False),
49            IntegratedTestKeywordParser('MODULES_DEFINES:', ParserKind.LIST,
50                                        initial_value=[])
51        ]
52
53    @staticmethod
54    def _get_parser(key, parsers):
55        for p in parsers:
56            if p.keyword == key:
57                return p
58        assert False and "parser not found"
59
60    # TODO: Move this into lit's FileBasedTest
61    def getTestsInDirectory(self, testSuite, path_in_suite,
62                            litConfig, localConfig):
63        source_path = testSuite.getSourcePath(path_in_suite)
64        for filename in os.listdir(source_path):
65            # Ignore dot files and excluded tests.
66            if filename.startswith('.') or filename in localConfig.excludes:
67                continue
68
69            filepath = os.path.join(source_path, filename)
70            if not os.path.isdir(filepath):
71                if any([filename.endswith(ext)
72                        for ext in localConfig.suffixes]):
73                    yield lit.Test.Test(testSuite, path_in_suite + (filename,),
74                                        localConfig)
75
76    def execute(self, test, lit_config):
77        while True:
78            try:
79                return self._execute(test, lit_config)
80            except OSError as oe:
81                if oe.errno != errno.ETXTBSY:
82                    raise
83                time.sleep(0.1)
84
85    def _execute(self, test, lit_config):
86        name = test.path_in_suite[-1]
87        name_root, name_ext = os.path.splitext(name)
88        is_libcxx_test = test.path_in_suite[0] == 'libcxx'
89        is_sh_test = name_root.endswith('.sh')
90        is_pass_test = name.endswith('.pass.cpp') or name.endswith('.pass.mm')
91        is_fail_test = name.endswith('.fail.cpp') or name.endswith('.fail.mm')
92        is_objcxx_test = name.endswith('.mm')
93        is_objcxx_arc_test = name.endswith('.arc.pass.mm') or \
94                             name.endswith('.arc.fail.mm')
95        assert is_sh_test or name_ext == '.cpp' or name_ext == '.mm', \
96            'non-cpp file must be sh test'
97
98        if test.config.unsupported:
99            return (lit.Test.UNSUPPORTED,
100                    "A lit.local.cfg marked this unsupported")
101
102        if is_objcxx_test and not \
103           'objective-c++' in test.config.available_features:
104            return (lit.Test.UNSUPPORTED, "Objective-C++ is not supported")
105
106        parsers = self._make_custom_parsers()
107        script = lit.TestRunner.parseIntegratedTestScript(
108            test, additional_parsers=parsers, require_script=is_sh_test)
109        # Check if a result for the test was returned. If so return that
110        # result.
111        if isinstance(script, lit.Test.Result):
112            return script
113        if lit_config.noExecute:
114            return lit.Test.Result(lit.Test.PASS)
115
116        # Check that we don't have run lines on tests that don't support them.
117        if not is_sh_test and len(script) != 0:
118            lit_config.fatal('Unsupported RUN line found in test %s' % name)
119
120        tmpDir, tmpBase = lit.TestRunner.getTempPaths(test)
121        substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir,
122                                                               tmpBase)
123        script = lit.TestRunner.applySubstitutions(script, substitutions)
124
125        test_cxx = copy.deepcopy(self.cxx)
126        if is_fail_test:
127            test_cxx.useCCache(False)
128            test_cxx.useWarnings(False)
129        extra_modules_defines = self._get_parser('MODULES_DEFINES:',
130                                                 parsers).getValue()
131        if '-fmodules' in test.config.available_features:
132            test_cxx.compile_flags += [('-D%s' % mdef.strip()) for
133                                       mdef in extra_modules_defines]
134            test_cxx.addWarningFlagIfSupported('-Wno-macro-redefined')
135            # FIXME: libc++ debug tests #define _LIBCPP_ASSERT to override it
136            # If we see this we need to build the test against uniquely built
137            # modules.
138            if is_libcxx_test:
139                with open(test.getSourcePath(), 'r') as f:
140                    contents = f.read()
141                if '#define _LIBCPP_ASSERT' in contents:
142                    test_cxx.useModules(False)
143
144        if is_objcxx_test:
145            test_cxx.source_lang = 'objective-c++'
146            if is_objcxx_arc_test:
147                test_cxx.compile_flags += ['-fobjc-arc']
148            else:
149                test_cxx.compile_flags += ['-fno-objc-arc']
150            test_cxx.link_flags += ['-framework', 'Foundation']
151
152        # Dispatch the test based on its suffix.
153        if is_sh_test:
154            if not isinstance(self.executor, LocalExecutor):
155                # We can't run ShTest tests with a executor yet.
156                # For now, bail on trying to run them
157                return lit.Test.UNSUPPORTED, 'ShTest format not yet supported'
158            test.config.environment = dict(self.exec_env)
159            return lit.TestRunner._runShTest(test, lit_config,
160                                             self.execute_external, script,
161                                             tmpBase)
162        elif is_fail_test:
163            return self._evaluate_fail_test(test, test_cxx, parsers)
164        elif is_pass_test:
165            return self._evaluate_pass_test(test, tmpBase, lit_config,
166                                            test_cxx, parsers)
167        else:
168            # No other test type is supported
169            assert False
170
171    def _clean(self, exec_path):  # pylint: disable=no-self-use
172        libcxx.util.cleanFile(exec_path)
173
174    def _evaluate_pass_test(self, test, tmpBase, lit_config,
175                            test_cxx, parsers):
176        execDir = os.path.dirname(test.getExecPath())
177        source_path = test.getSourcePath()
178        exec_path = tmpBase + '.exe'
179        object_path = tmpBase + '.o'
180        # Create the output directory if it does not already exist.
181        libcxx.util.mkdir_p(os.path.dirname(tmpBase))
182        try:
183            # Compile the test
184            cmd, out, err, rc = test_cxx.compileLinkTwoSteps(
185                source_path, out=exec_path, object_file=object_path,
186                cwd=execDir)
187            compile_cmd = cmd
188            if rc != 0:
189                report = libcxx.util.makeReport(cmd, out, err, rc)
190                report += "Compilation failed unexpectedly!"
191                return lit.Test.FAIL, report
192            # Run the test
193            local_cwd = os.path.dirname(source_path)
194            env = None
195            if self.exec_env:
196                env = self.exec_env
197            # TODO: Only list actually needed files in file_deps.
198            # Right now we just mark all of the .dat files in the same
199            # directory as dependencies, but it's likely less than that. We
200            # should add a `// FILE-DEP: foo.dat` to each test to track this.
201            data_files = [os.path.join(local_cwd, f)
202                          for f in os.listdir(local_cwd) if f.endswith('.dat')]
203            is_flaky = self._get_parser('FLAKY_TEST.', parsers).getValue()
204            max_retry = 3 if is_flaky else 1
205            for retry_count in range(max_retry):
206                cmd, out, err, rc = self.executor.run(exec_path, [exec_path],
207                                                      local_cwd, data_files,
208                                                      env)
209                if rc == 0:
210                    res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
211                    return res, ''
212                elif rc != 0 and retry_count + 1 == max_retry:
213                    report = libcxx.util.makeReport(cmd, out, err, rc)
214                    report = "Compiled With: %s\n%s" % (compile_cmd, report)
215                    report += "Compiled test failed unexpectedly!"
216                    return lit.Test.FAIL, report
217
218            assert False # Unreachable
219        finally:
220            # Note that cleanup of exec_file happens in `_clean()`. If you
221            # override this, cleanup is your reponsibility.
222            libcxx.util.cleanFile(object_path)
223            self._clean(exec_path)
224
225    def _evaluate_fail_test(self, test, test_cxx, parsers):
226        source_path = test.getSourcePath()
227        # FIXME: lift this detection into LLVM/LIT.
228        with open(source_path, 'r') as f:
229            contents = f.read()
230        verify_tags = ['expected-note', 'expected-remark', 'expected-warning',
231                       'expected-error', 'expected-no-diagnostics']
232        use_verify = self.use_verify_for_fail and \
233                     any([tag in contents for tag in verify_tags])
234        # FIXME(EricWF): GCC 5 does not evaluate static assertions that
235        # are dependant on a template parameter when '-fsyntax-only' is passed.
236        # This is fixed in GCC 6. However for now we only pass "-fsyntax-only"
237        # when using Clang.
238        if test_cxx.type != 'gcc':
239            test_cxx.flags += ['-fsyntax-only']
240        if use_verify:
241            test_cxx.useVerify()
242            test_cxx.useWarnings()
243            if '-Wuser-defined-warnings' in test_cxx.warning_flags:
244                test_cxx.warning_flags += ['-Wno-error=user-defined-warnings']
245
246        cmd, out, err, rc = test_cxx.compile(source_path, out=os.devnull)
247        expected_rc = 0 if use_verify else 1
248        if rc == expected_rc:
249            return lit.Test.PASS, ''
250        else:
251            report = libcxx.util.makeReport(cmd, out, err, rc)
252            report_msg = ('Expected compilation to fail!' if not use_verify else
253                          'Expected compilation using verify to pass!')
254            return lit.Test.FAIL, report + report_msg + '\n'
255