TestRunner.py revision b35a1733607b07f36c3617932b67debb375a15c5 
1from __future__ import absolute_import
2import os, signal, subprocess, sys
3import StringIO
4
5import platform
6import tempfile
7
8import re
9import lit.ShUtil as ShUtil
10import lit.Test as Test
11import lit.Util as Util
12
13class InternalShellError(Exception):
14    def __init__(self, command, message):
15        self.command = command
16        self.message = message
17
18kIsWindows = platform.system() == 'Windows'
19
20# Don't use close_fds on Windows.
21kUseCloseFDs = not kIsWindows
22
23# Use temporary files to replace /dev/null on Windows.
24kAvoidDevNull = kIsWindows
25
26def executeCommand(command, cwd=None, env=None):
27    # Close extra file handles on UNIX (on Windows this cannot be done while
28    # also redirecting input).
29    close_fds = not kIsWindows
30
31    p = subprocess.Popen(command, cwd=cwd,
32                         stdin=subprocess.PIPE,
33                         stdout=subprocess.PIPE,
34                         stderr=subprocess.PIPE,
35                         env=env, close_fds=close_fds)
36    out,err = p.communicate()
37    exitCode = p.wait()
38
39    # Detect Ctrl-C in subprocess.
40    if exitCode == -signal.SIGINT:
41        raise KeyboardInterrupt
42
43    return out, err, exitCode
44
45def executeShCmd(cmd, cfg, cwd, results):
46    if isinstance(cmd, ShUtil.Seq):
47        if cmd.op == ';':
48            res = executeShCmd(cmd.lhs, cfg, cwd, results)
49            return executeShCmd(cmd.rhs, cfg, cwd, results)
50
51        if cmd.op == '&':
52            raise InternalShellError(cmd,"unsupported shell operator: '&'")
53
54        if cmd.op == '||':
55            res = executeShCmd(cmd.lhs, cfg, cwd, results)
56            if res != 0:
57                res = executeShCmd(cmd.rhs, cfg, cwd, results)
58            return res
59
60        if cmd.op == '&&':
61            res = executeShCmd(cmd.lhs, cfg, cwd, results)
62            if res is None:
63                return res
64
65            if res == 0:
66                res = executeShCmd(cmd.rhs, cfg, cwd, results)
67            return res
68
69        raise ValueError('Unknown shell command: %r' % cmd.op)
70
71    assert isinstance(cmd, ShUtil.Pipeline)
72    procs = []
73    input = subprocess.PIPE
74    stderrTempFiles = []
75    opened_files = []
76    named_temp_files = []
77    # To avoid deadlock, we use a single stderr stream for piped
78    # output. This is null until we have seen some output using
79    # stderr.
80    for i,j in enumerate(cmd.commands):
81        # Apply the redirections, we use (N,) as a sentinel to indicate stdin,
82        # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
83        # from a file are represented with a list [file, mode, file-object]
84        # where file-object is initially None.
85        redirects = [(0,), (1,), (2,)]
86        for r in j.redirects:
87            if r[0] == ('>',2):
88                redirects[2] = [r[1], 'w', None]
89            elif r[0] == ('>>',2):
90                redirects[2] = [r[1], 'a', None]
91            elif r[0] == ('>&',2) and r[1] in '012':
92                redirects[2] = redirects[int(r[1])]
93            elif r[0] == ('>&',) or r[0] == ('&>',):
94                redirects[1] = redirects[2] = [r[1], 'w', None]
95            elif r[0] == ('>',):
96                redirects[1] = [r[1], 'w', None]
97            elif r[0] == ('>>',):
98                redirects[1] = [r[1], 'a', None]
99            elif r[0] == ('<',):
100                redirects[0] = [r[1], 'r', None]
101            else:
102                raise InternalShellError(j,"Unsupported redirect: %r" % (r,))
103
104        # Map from the final redirections to something subprocess can handle.
105        final_redirects = []
106        for index,r in enumerate(redirects):
107            if r == (0,):
108                result = input
109            elif r == (1,):
110                if index == 0:
111                    raise InternalShellError(j,"Unsupported redirect for stdin")
112                elif index == 1:
113                    result = subprocess.PIPE
114                else:
115                    result = subprocess.STDOUT
116            elif r == (2,):
117                if index != 2:
118                    raise InternalShellError(j,"Unsupported redirect on stdout")
119                result = subprocess.PIPE
120            else:
121                if r[2] is None:
122                    if kAvoidDevNull and r[0] == '/dev/null':
123                        r[2] = tempfile.TemporaryFile(mode=r[1])
124                    else:
125                        r[2] = open(r[0], r[1])
126                    # Workaround a Win32 and/or subprocess bug when appending.
127                    #
128                    # FIXME: Actually, this is probably an instance of PR6753.
129                    if r[1] == 'a':
130                        r[2].seek(0, 2)
131                    opened_files.append(r[2])
132                result = r[2]
133            final_redirects.append(result)
134
135        stdin, stdout, stderr = final_redirects
136
137        # If stderr wants to come from stdout, but stdout isn't a pipe, then put
138        # stderr on a pipe and treat it as stdout.
139        if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
140            stderr = subprocess.PIPE
141            stderrIsStdout = True
142        else:
143            stderrIsStdout = False
144
145            # Don't allow stderr on a PIPE except for the last
146            # process, this could deadlock.
147            #
148            # FIXME: This is slow, but so is deadlock.
149            if stderr == subprocess.PIPE and j != cmd.commands[-1]:
150                stderr = tempfile.TemporaryFile(mode='w+b')
151                stderrTempFiles.append((i, stderr))
152
153        # Resolve the executable path ourselves.
154        args = list(j.args)
155        args[0] = Util.which(args[0], cfg.environment['PATH'])
156        if not args[0]:
157            raise InternalShellError(j, '%r: command not found' % j.args[0])
158
159        # Replace uses of /dev/null with temporary files.
160        if kAvoidDevNull:
161            for i,arg in enumerate(args):
162                if arg == "/dev/null":
163                    f = tempfile.NamedTemporaryFile(delete=False)
164                    f.close()
165                    named_temp_files.append(f.name)
166                    args[i] = f.name
167
168        procs.append(subprocess.Popen(args, cwd=cwd,
169                                      stdin = stdin,
170                                      stdout = stdout,
171                                      stderr = stderr,
172                                      env = cfg.environment,
173                                      close_fds = kUseCloseFDs))
174
175        # Immediately close stdin for any process taking stdin from us.
176        if stdin == subprocess.PIPE:
177            procs[-1].stdin.close()
178            procs[-1].stdin = None
179
180        # Update the current stdin source.
181        if stdout == subprocess.PIPE:
182            input = procs[-1].stdout
183        elif stderrIsStdout:
184            input = procs[-1].stderr
185        else:
186            input = subprocess.PIPE
187
188    # Explicitly close any redirected files. We need to do this now because we
189    # need to release any handles we may have on the temporary files (important
190    # on Win32, for example). Since we have already spawned the subprocess, our
191    # handles have already been transferred so we do not need them anymore.
192    for f in opened_files:
193        f.close()
194
195    # FIXME: There is probably still deadlock potential here. Yawn.
196    procData = [None] * len(procs)
197    procData[-1] = procs[-1].communicate()
198
199    for i in range(len(procs) - 1):
200        if procs[i].stdout is not None:
201            out = procs[i].stdout.read()
202        else:
203            out = ''
204        if procs[i].stderr is not None:
205            err = procs[i].stderr.read()
206        else:
207            err = ''
208        procData[i] = (out,err)
209
210    # Read stderr out of the temp files.
211    for i,f in stderrTempFiles:
212        f.seek(0, 0)
213        procData[i] = (procData[i][0], f.read())
214
215    exitCode = None
216    for i,(out,err) in enumerate(procData):
217        res = procs[i].wait()
218        # Detect Ctrl-C in subprocess.
219        if res == -signal.SIGINT:
220            raise KeyboardInterrupt
221
222        results.append((cmd.commands[i], out, err, res))
223        if cmd.pipe_err:
224            # Python treats the exit code as a signed char.
225            if res < 0:
226                exitCode = min(exitCode, res)
227            else:
228                exitCode = max(exitCode, res)
229        else:
230            exitCode = res
231
232    # Remove any named temporary files we created.
233    for f in named_temp_files:
234        try:
235            os.remove(f)
236        except OSError:
237            pass
238
239    if cmd.negate:
240        exitCode = not exitCode
241
242    return exitCode
243
244def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
245    cmds = []
246    for ln in commands:
247        try:
248            cmds.append(ShUtil.ShParser(ln, litConfig.isWindows,
249                                        test.config.pipefail).parse())
250        except:
251            return (Test.FAIL, "shell parser error on: %r" % ln)
252
253    cmd = cmds[0]
254    for c in cmds[1:]:
255        cmd = ShUtil.Seq(cmd, '&&', c)
256
257    results = []
258    try:
259        exitCode = executeShCmd(cmd, test.config, cwd, results)
260    except InternalShellError:
261        e = sys.exc_info()[1]
262        exitCode = 127
263        results.append((e.command, '', e.message, exitCode))
264
265    out = err = ''
266    for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
267        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
268        out += 'Command %d Result: %r\n' % (i, res)
269        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
270        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
271
272    return out, err, exitCode
273
274def executeScript(test, litConfig, tmpBase, commands, cwd):
275    bashPath = litConfig.getBashPath();
276    isWin32CMDEXE = (litConfig.isWindows and not bashPath)
277    script = tmpBase + '.script'
278    if isWin32CMDEXE:
279        script += '.bat'
280
281    # Write script file
282    mode = 'w'
283    if litConfig.isWindows and not isWin32CMDEXE:
284      mode += 'b'  # Avoid CRLFs when writing bash scripts.
285    f = open(script, mode)
286    if isWin32CMDEXE:
287        f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
288    else:
289        if test.config.pipefail:
290            f.write('set -o pipefail;')
291        f.write('{ ' + '; } &&\n{ '.join(commands) + '; }')
292    f.write('\n')
293    f.close()
294
295    if isWin32CMDEXE:
296        command = ['cmd','/c', script]
297    else:
298        if bashPath:
299            command = [bashPath, script]
300        else:
301            command = ['/bin/sh', script]
302        if litConfig.useValgrind:
303            # FIXME: Running valgrind on sh is overkill. We probably could just
304            # run on clang with no real loss.
305            command = litConfig.valgrindArgs + command
306
307    return executeCommand(command, cwd=cwd, env=test.config.environment)
308
309def isExpectedFail(test, xfails):
310    # Check if any of the xfails match an available feature or the target.
311    for item in xfails:
312        # If this is the wildcard, it always fails.
313        if item == '*':
314            return True
315
316        # If this is an exact match for one of the features, it fails.
317        if item in test.config.available_features:
318            return True
319
320        # If this is a part of the target triple, it fails.
321        if item in test.suite.config.target_triple:
322            return True
323
324    return False
325
326def parseIntegratedTestScript(test, normalize_slashes=False,
327                              extra_substitutions=[]):
328    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
329    script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES'
330    information. The RUN lines also will have variable substitution performed.
331    """
332
333    # Get the temporary location, this is always relative to the test suite
334    # root, not test source root.
335    #
336    # FIXME: This should not be here?
337    sourcepath = test.getSourcePath()
338    sourcedir = os.path.dirname(sourcepath)
339    execpath = test.getExecPath()
340    execdir,execbase = os.path.split(execpath)
341    tmpDir = os.path.join(execdir, 'Output')
342    tmpBase = os.path.join(tmpDir, execbase)
343    if test.index is not None:
344        tmpBase += '_%d' % test.index
345
346    # Normalize slashes, if requested.
347    if normalize_slashes:
348        sourcepath = sourcepath.replace('\\', '/')
349        sourcedir = sourcedir.replace('\\', '/')
350        tmpDir = tmpDir.replace('\\', '/')
351        tmpBase = tmpBase.replace('\\', '/')
352
353    # We use #_MARKER_# to hide %% while we do the other substitutions.
354    substitutions = list(extra_substitutions)
355    substitutions.extend([('%%', '#_MARKER_#')])
356    substitutions.extend(test.config.substitutions)
357    substitutions.extend([('%s', sourcepath),
358                          ('%S', sourcedir),
359                          ('%p', sourcedir),
360                          ('%{pathsep}', os.pathsep),
361                          ('%t', tmpBase + '.tmp'),
362                          ('%T', tmpDir),
363                          ('#_MARKER_#', '%')])
364
365    # Collect the test lines from the script.
366    script = []
367    xfails = []
368    requires = []
369    line_number = 0
370    for ln in open(sourcepath):
371        line_number += 1
372        if 'RUN:' in ln:
373            # Isolate the command to run.
374            index = ln.index('RUN:')
375            ln = ln[index+4:]
376
377            # Trim trailing whitespace.
378            ln = ln.rstrip()
379
380            # Substitute line number expressions
381            ln = re.sub('%\(line\)', str(line_number), ln)
382            def replace_line_number(match):
383                if match.group(1) == '+':
384                    return str(line_number + int(match.group(2)))
385                if match.group(1) == '-':
386                    return str(line_number - int(match.group(2)))
387            ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln)
388
389            # Collapse lines with trailing '\\'.
390            if script and script[-1][-1] == '\\':
391                script[-1] = script[-1][:-1] + ln
392            else:
393                script.append(ln)
394        elif 'XFAIL:' in ln:
395            items = ln[ln.index('XFAIL:') + 6:].split(',')
396            xfails.extend([s.strip() for s in items])
397        elif 'REQUIRES:' in ln:
398            items = ln[ln.index('REQUIRES:') + 9:].split(',')
399            requires.extend([s.strip() for s in items])
400        elif 'END.' in ln:
401            # Check for END. lines.
402            if ln[ln.index('END.'):].strip() == 'END.':
403                break
404
405    # Apply substitutions to the script.  Allow full regular
406    # expression syntax.  Replace each matching occurrence of regular
407    # expression pattern a with substitution b in line ln.
408    def processLine(ln):
409        # Apply substitutions
410        for a,b in substitutions:
411            if kIsWindows:
412                b = b.replace("\\","\\\\")
413            ln = re.sub(a, b, ln)
414
415        # Strip the trailing newline and any extra whitespace.
416        return ln.strip()
417    script = map(processLine, script)
418
419    # Verify the script contains a run line.
420    if not script:
421        return (Test.UNRESOLVED, "Test has no run line!")
422
423    # Check for unterminated run lines.
424    if script[-1][-1] == '\\':
425        return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')")
426
427    # Check that we have the required features:
428    missing_required_features = [f for f in requires
429                                 if f not in test.config.available_features]
430    if missing_required_features:
431        msg = ', '.join(missing_required_features)
432        return (Test.UNSUPPORTED,
433                "Test requires the following features: %s" % msg)
434
435    isXFail = isExpectedFail(test, xfails)
436    return script,isXFail,tmpBase,execdir
437
438def formatTestOutput(status, out, err, exitCode, script):
439    output = StringIO.StringIO()
440    output.write("Script:\n")
441    output.write("--\n")
442    output.write('\n'.join(script))
443    output.write("\n--\n")
444    output.write("Exit Code: %r\n\n" % exitCode)
445    if out:
446        output.write("Command Output (stdout):\n")
447        output.write("--\n")
448        output.write(out)
449        output.write("--\n")
450    if err:
451        output.write("Command Output (stderr):\n")
452        output.write("--\n")
453        output.write(err)
454        output.write("--\n")
455    return (status, output.getvalue())
456
457def executeShTest(test, litConfig, useExternalSh,
458                  extra_substitutions=[]):
459    if test.config.unsupported:
460        return (Test.UNSUPPORTED, 'Test is unsupported')
461
462    res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions)
463    if len(res) == 2:
464        return res
465
466    script, isXFail, tmpBase, execdir = res
467
468    # Create the output directory if it does not already exist.
469    Util.mkdir_p(os.path.dirname(tmpBase))
470
471    if useExternalSh:
472        res = executeScript(test, litConfig, tmpBase, script, execdir)
473    else:
474        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
475    if len(res) == 2:
476        return res
477
478    out,err,exitCode = res
479    if isXFail:
480        ok = exitCode != 0
481        if ok:
482            status = Test.XFAIL
483        else:
484            status = Test.XPASS
485    else:
486        ok = exitCode == 0
487        if ok:
488            status = Test.PASS
489        else:
490            status = Test.FAIL
491
492    if ok:
493        return (status,'')
494
495    return formatTestOutput(status, out, err, exitCode, script)
496