TestRunner.py revision 44a83f092029fa76bce05ff0c0598afc55215767 
1import os, signal, subprocess, sys
2import StringIO
3
4import ShUtil
5import Test
6import Util
7
8import platform
9import tempfile
10
11import re
12
13class InternalShellError(Exception):
14    def __init__(self, command, message):
15        self.command = command
16        self.message = message
17
18kIsWindows = platform.system() == 'Windows'
19
20# Don't use close_fds on Windows.
21kUseCloseFDs = not kIsWindows
22
23# Use temporary files to replace /dev/null on Windows.
24kAvoidDevNull = kIsWindows
25
26def executeCommand(command, cwd=None, env=None):
27    # Close extra file handles on UNIX (on Windows this cannot be done while
28    # also redirecting input).
29    close_fds = not kIsWindows
30
31    p = subprocess.Popen(command, cwd=cwd,
32                         stdin=subprocess.PIPE,
33                         stdout=subprocess.PIPE,
34                         stderr=subprocess.PIPE,
35                         env=env, close_fds=close_fds)
36    out,err = p.communicate()
37    exitCode = p.wait()
38
39    # Detect Ctrl-C in subprocess.
40    if exitCode == -signal.SIGINT:
41        raise KeyboardInterrupt
42
43    return out, err, exitCode
44
45def executeShCmd(cmd, cfg, cwd, results):
46    if isinstance(cmd, ShUtil.Seq):
47        if cmd.op == ';':
48            res = executeShCmd(cmd.lhs, cfg, cwd, results)
49            return executeShCmd(cmd.rhs, cfg, cwd, results)
50
51        if cmd.op == '&':
52            raise NotImplementedError,"unsupported test command: '&'"
53
54        if cmd.op == '||':
55            res = executeShCmd(cmd.lhs, cfg, cwd, results)
56            if res != 0:
57                res = executeShCmd(cmd.rhs, cfg, cwd, results)
58            return res
59        if cmd.op == '&&':
60            res = executeShCmd(cmd.lhs, cfg, cwd, results)
61            if res is None:
62                return res
63
64            if res == 0:
65                res = executeShCmd(cmd.rhs, cfg, cwd, results)
66            return res
67
68        raise ValueError,'Unknown shell command: %r' % cmd.op
69
70    assert isinstance(cmd, ShUtil.Pipeline)
71    procs = []
72    input = subprocess.PIPE
73    stderrTempFiles = []
74    opened_files = []
75    named_temp_files = []
76    # To avoid deadlock, we use a single stderr stream for piped
77    # output. This is null until we have seen some output using
78    # stderr.
79    for i,j in enumerate(cmd.commands):
80        # Apply the redirections, we use (N,) as a sentinal to indicate stdin,
81        # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
82        # from a file are represented with a list [file, mode, file-object]
83        # where file-object is initially None.
84        redirects = [(0,), (1,), (2,)]
85        for r in j.redirects:
86            if r[0] == ('>',2):
87                redirects[2] = [r[1], 'w', None]
88            elif r[0] == ('>>',2):
89                redirects[2] = [r[1], 'a', None]
90            elif r[0] == ('>&',2) and r[1] in '012':
91                redirects[2] = redirects[int(r[1])]
92            elif r[0] == ('>&',) or r[0] == ('&>',):
93                redirects[1] = redirects[2] = [r[1], 'w', None]
94            elif r[0] == ('>',):
95                redirects[1] = [r[1], 'w', None]
96            elif r[0] == ('>>',):
97                redirects[1] = [r[1], 'a', None]
98            elif r[0] == ('<',):
99                redirects[0] = [r[1], 'r', None]
100            else:
101                raise NotImplementedError,"Unsupported redirect: %r" % (r,)
102
103        # Map from the final redirections to something subprocess can handle.
104        final_redirects = []
105        for index,r in enumerate(redirects):
106            if r == (0,):
107                result = input
108            elif r == (1,):
109                if index == 0:
110                    raise NotImplementedError,"Unsupported redirect for stdin"
111                elif index == 1:
112                    result = subprocess.PIPE
113                else:
114                    result = subprocess.STDOUT
115            elif r == (2,):
116                if index != 2:
117                    raise NotImplementedError,"Unsupported redirect on stdout"
118                result = subprocess.PIPE
119            else:
120                if r[2] is None:
121                    if kAvoidDevNull and r[0] == '/dev/null':
122                        r[2] = tempfile.TemporaryFile(mode=r[1])
123                    else:
124                        r[2] = open(r[0], r[1])
125                    # Workaround a Win32 and/or subprocess bug when appending.
126                    #
127                    # FIXME: Actually, this is probably an instance of PR6753.
128                    if r[1] == 'a':
129                        r[2].seek(0, 2)
130                    opened_files.append(r[2])
131                result = r[2]
132            final_redirects.append(result)
133
134        stdin, stdout, stderr = final_redirects
135
136        # If stderr wants to come from stdout, but stdout isn't a pipe, then put
137        # stderr on a pipe and treat it as stdout.
138        if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
139            stderr = subprocess.PIPE
140            stderrIsStdout = True
141        else:
142            stderrIsStdout = False
143
144            # Don't allow stderr on a PIPE except for the last
145            # process, this could deadlock.
146            #
147            # FIXME: This is slow, but so is deadlock.
148            if stderr == subprocess.PIPE and j != cmd.commands[-1]:
149                stderr = tempfile.TemporaryFile(mode='w+b')
150                stderrTempFiles.append((i, stderr))
151
152        # Resolve the executable path ourselves.
153        args = list(j.args)
154        args[0] = Util.which(args[0], cfg.environment['PATH'])
155        if not args[0]:
156            raise InternalShellError(j, '%r: command not found' % j.args[0])
157
158        # Replace uses of /dev/null with temporary files.
159        if kAvoidDevNull:
160            for i,arg in enumerate(args):
161                if arg == "/dev/null":
162                    f = tempfile.NamedTemporaryFile(delete=False)
163                    f.close()
164                    named_temp_files.append(f.name)
165                    args[i] = f.name
166
167        procs.append(subprocess.Popen(args, cwd=cwd,
168                                      stdin = stdin,
169                                      stdout = stdout,
170                                      stderr = stderr,
171                                      env = cfg.environment,
172                                      close_fds = kUseCloseFDs))
173
174        # Immediately close stdin for any process taking stdin from us.
175        if stdin == subprocess.PIPE:
176            procs[-1].stdin.close()
177            procs[-1].stdin = None
178
179        # Update the current stdin source.
180        if stdout == subprocess.PIPE:
181            input = procs[-1].stdout
182        elif stderrIsStdout:
183            input = procs[-1].stderr
184        else:
185            input = subprocess.PIPE
186
187    # Explicitly close any redirected files. We need to do this now because we
188    # need to release any handles we may have on the temporary files (important
189    # on Win32, for example). Since we have already spawned the subprocess, our
190    # handles have already been transferred so we do not need them anymore.
191    for f in opened_files:
192        f.close()
193
194    # FIXME: There is probably still deadlock potential here. Yawn.
195    procData = [None] * len(procs)
196    procData[-1] = procs[-1].communicate()
197
198    for i in range(len(procs) - 1):
199        if procs[i].stdout is not None:
200            out = procs[i].stdout.read()
201        else:
202            out = ''
203        if procs[i].stderr is not None:
204            err = procs[i].stderr.read()
205        else:
206            err = ''
207        procData[i] = (out,err)
208
209    # Read stderr out of the temp files.
210    for i,f in stderrTempFiles:
211        f.seek(0, 0)
212        procData[i] = (procData[i][0], f.read())
213
214    exitCode = None
215    for i,(out,err) in enumerate(procData):
216        res = procs[i].wait()
217        # Detect Ctrl-C in subprocess.
218        if res == -signal.SIGINT:
219            raise KeyboardInterrupt
220
221        results.append((cmd.commands[i], out, err, res))
222        if cmd.pipe_err:
223            # Python treats the exit code as a signed char.
224            if res < 0:
225                exitCode = min(exitCode, res)
226            else:
227                exitCode = max(exitCode, res)
228        else:
229            exitCode = res
230
231    # Remove any named temporary files we created.
232    for f in named_temp_files:
233        try:
234            os.remove(f)
235        except OSError:
236            pass
237
238    if cmd.negate:
239        exitCode = not exitCode
240
241    return exitCode
242
243def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
244    ln = ' &&\n'.join(commands)
245    try:
246        cmd = ShUtil.ShParser(ln, litConfig.isWindows).parse()
247    except:
248        return (Test.FAIL, "shell parser error on: %r" % ln)
249
250    results = []
251    try:
252        exitCode = executeShCmd(cmd, test.config, cwd, results)
253    except InternalShellError,e:
254        out = ''
255        err = e.message
256        exitCode = 255
257
258    out = err = ''
259    for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
260        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
261        out += 'Command %d Result: %r\n' % (i, res)
262        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
263        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
264
265    return out, err, exitCode
266
267def executeTclScriptInternal(test, litConfig, tmpBase, commands, cwd):
268    import TclUtil
269    cmds = []
270    for ln in commands:
271        # Given the unfortunate way LLVM's test are written, the line gets
272        # backslash substitution done twice.
273        ln = TclUtil.TclLexer(ln).lex_unquoted(process_all = True)
274
275        try:
276            tokens = list(TclUtil.TclLexer(ln).lex())
277        except:
278            return (Test.FAIL, "Tcl lexer error on: %r" % ln)
279
280        # Validate there are no control tokens.
281        for t in tokens:
282            if not isinstance(t, str):
283                return (Test.FAIL,
284                        "Invalid test line: %r containing %r" % (ln, t))
285
286        try:
287            cmds.append(TclUtil.TclExecCommand(tokens).parse_pipeline())
288        except:
289            return (Test.FAIL, "Tcl 'exec' parse error on: %r" % ln)
290
291    if litConfig.useValgrind:
292        for pipeline in cmds:
293            if pipeline.commands:
294                # Only valgrind the first command in each pipeline, to avoid
295                # valgrinding things like grep, not, and FileCheck.
296                cmd = pipeline.commands[0]
297                cmd.args = litConfig.valgrindArgs + cmd.args
298
299    cmd = cmds[0]
300    for c in cmds[1:]:
301        cmd = ShUtil.Seq(cmd, '&&', c)
302
303    # FIXME: This is lame, we shouldn't need bash. See PR5240.
304    bashPath = litConfig.getBashPath()
305    if litConfig.useTclAsSh and bashPath:
306        script = tmpBase + '.script'
307
308        # Write script file
309        f = open(script,'w')
310        print >>f, 'set -o pipefail'
311        cmd.toShell(f, pipefail = True)
312        f.close()
313
314        if 0:
315            print >>sys.stdout, cmd
316            print >>sys.stdout, open(script).read()
317            print >>sys.stdout
318            return '', '', 0
319
320        command = [litConfig.getBashPath(), script]
321        out,err,exitCode = executeCommand(command, cwd=cwd,
322                                          env=test.config.environment)
323
324        return out,err,exitCode
325    else:
326        results = []
327        try:
328            exitCode = executeShCmd(cmd, test.config, cwd, results)
329        except InternalShellError,e:
330            results.append((e.command, '', e.message + '\n', 255))
331            exitCode = 255
332
333    out = err = ''
334
335    for i,(cmd, cmd_out, cmd_err, res) in enumerate(results):
336        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
337        out += 'Command %d Result: %r\n' % (i, res)
338        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
339        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
340
341    return out, err, exitCode
342
343def executeScript(test, litConfig, tmpBase, commands, cwd):
344    bashPath = litConfig.getBashPath();
345    isWin32CMDEXE = (litConfig.isWindows and not bashPath)
346    script = tmpBase + '.script'
347    if isWin32CMDEXE:
348        script += '.bat'
349
350    # Write script file
351    f = open(script,'w')
352    if isWin32CMDEXE:
353        f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
354    else:
355        f.write(' &&\n'.join(commands))
356    f.write('\n')
357    f.close()
358
359    if isWin32CMDEXE:
360        command = ['cmd','/c', script]
361    else:
362        if bashPath:
363            command = [bashPath, script]
364        else:
365            command = ['/bin/sh', script]
366        if litConfig.useValgrind:
367            # FIXME: Running valgrind on sh is overkill. We probably could just
368            # run on clang with no real loss.
369            command = litConfig.valgrindArgs + command
370
371    return executeCommand(command, cwd=cwd, env=test.config.environment)
372
373def isExpectedFail(test, xfails, xtargets):
374    # If the xfail matches an available feature, it always fails.
375    for item in xfails:
376        if item in test.config.available_features:
377            return True
378
379    # Otherwise, check if any xfail matches this target.
380    for item in xfails:
381        if item == '*' or item in test.suite.config.target_triple:
382            break
383    else:
384        return False
385
386    # If so, see if it is expected to pass on this target.
387    #
388    # FIXME: Rename XTARGET to something that makes sense, like XPASS.
389    for item in xtargets:
390        if item == '*' or item in test.suite.config.target_triple:
391            return False
392
393    return True
394
395def parseIntegratedTestScript(test, normalize_slashes=False,
396                              extra_substitutions=[]):
397    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
398    script and extract the lines to 'RUN' as well as 'XFAIL' and 'XTARGET'
399    information. The RUN lines also will have variable substitution performed.
400    """
401
402    # Get the temporary location, this is always relative to the test suite
403    # root, not test source root.
404    #
405    # FIXME: This should not be here?
406    sourcepath = test.getSourcePath()
407    sourcedir = os.path.dirname(sourcepath)
408    execpath = test.getExecPath()
409    execdir,execbase = os.path.split(execpath)
410    tmpDir = os.path.join(execdir, 'Output')
411    tmpBase = os.path.join(tmpDir, execbase)
412    if test.index is not None:
413        tmpBase += '_%d' % test.index
414
415    # Normalize slashes, if requested.
416    if normalize_slashes:
417        sourcepath = sourcepath.replace('\\', '/')
418        sourcedir = sourcedir.replace('\\', '/')
419        tmpDir = tmpDir.replace('\\', '/')
420        tmpBase = tmpBase.replace('\\', '/')
421
422    # We use #_MARKER_# to hide %% while we do the other substitutions.
423    substitutions = list(extra_substitutions)
424    substitutions.extend([('%%', '#_MARKER_#')])
425    substitutions.extend(test.config.substitutions)
426    substitutions.extend([('%s', sourcepath),
427                          ('%S', sourcedir),
428                          ('%p', sourcedir),
429                          ('%{pathsep}', os.pathsep),
430                          ('%t', tmpBase + '.tmp'),
431                          ('%T', tmpDir),
432                          # FIXME: Remove this once we kill DejaGNU.
433                          ('%abs_tmp', tmpBase + '.tmp'),
434                          ('#_MARKER_#', '%')])
435
436    # Collect the test lines from the script.
437    script = []
438    xfails = []
439    xtargets = []
440    requires = []
441    for ln in open(sourcepath):
442        if 'RUN:' in ln:
443            # Isolate the command to run.
444            index = ln.index('RUN:')
445            ln = ln[index+4:]
446
447            # Trim trailing whitespace.
448            ln = ln.rstrip()
449
450            # Collapse lines with trailing '\\'.
451            if script and script[-1][-1] == '\\':
452                script[-1] = script[-1][:-1] + ln
453            else:
454                script.append(ln)
455        elif 'XFAIL:' in ln:
456            items = ln[ln.index('XFAIL:') + 6:].split(',')
457            xfails.extend([s.strip() for s in items])
458        elif 'XTARGET:' in ln:
459            items = ln[ln.index('XTARGET:') + 8:].split(',')
460            xtargets.extend([s.strip() for s in items])
461        elif 'REQUIRES:' in ln:
462            items = ln[ln.index('REQUIRES:') + 9:].split(',')
463            requires.extend([s.strip() for s in items])
464        elif 'END.' in ln:
465            # Check for END. lines.
466            if ln[ln.index('END.'):].strip() == 'END.':
467                break
468
469    # Apply substitutions to the script.  Allow full regular
470    # expression syntax.  Replace each matching occurrence of regular
471    # expression pattern a with substitution b in line ln.
472    def processLine(ln):
473        # Apply substitutions
474        for a,b in substitutions:
475            if kIsWindows:
476                b = b.replace("\\","\\\\")
477            ln = re.sub(a, b, ln)
478
479        # Strip the trailing newline and any extra whitespace.
480        return ln.strip()
481    script = map(processLine, script)
482
483    # Verify the script contains a run line.
484    if not script:
485        return (Test.UNRESOLVED, "Test has no run line!")
486
487    # Check for unterminated run lines.
488    if script[-1][-1] == '\\':
489        return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')")
490
491    # Check that we have the required features:
492    missing_required_features = [f for f in requires
493                                 if f not in test.config.available_features]
494    if missing_required_features:
495        msg = ', '.join(missing_required_features)
496        return (Test.UNSUPPORTED,
497                "Test requires the following features: %s" % msg)
498
499    isXFail = isExpectedFail(test, xfails, xtargets)
500    return script,isXFail,tmpBase,execdir
501
502def formatTestOutput(status, out, err, exitCode, failDueToStderr, script):
503    output = StringIO.StringIO()
504    print >>output, "Script:"
505    print >>output, "--"
506    print >>output, '\n'.join(script)
507    print >>output, "--"
508    print >>output, "Exit Code: %r" % exitCode,
509    if failDueToStderr:
510        print >>output, "(but there was output on stderr)"
511    else:
512        print >>output
513    if out:
514        print >>output, "Command Output (stdout):"
515        print >>output, "--"
516        output.write(out)
517        print >>output, "--"
518    if err:
519        print >>output, "Command Output (stderr):"
520        print >>output, "--"
521        output.write(err)
522        print >>output, "--"
523    return (status, output.getvalue())
524
525def executeTclTest(test, litConfig):
526    if test.config.unsupported:
527        return (Test.UNSUPPORTED, 'Test is unsupported')
528
529    # Parse the test script, normalizing slashes in substitutions on Windows
530    # (since otherwise Tcl style lexing will treat them as escapes).
531    res = parseIntegratedTestScript(test, normalize_slashes=kIsWindows)
532    if len(res) == 2:
533        return res
534
535    script, isXFail, tmpBase, execdir = res
536
537    if litConfig.noExecute:
538        return (Test.PASS, '')
539
540    # Create the output directory if it does not already exist.
541    Util.mkdir_p(os.path.dirname(tmpBase))
542
543    res = executeTclScriptInternal(test, litConfig, tmpBase, script, execdir)
544    if len(res) == 2:
545        return res
546
547    # Test for failure. In addition to the exit code, Tcl commands are
548    # considered to fail if there is any standard error output.
549    out,err,exitCode = res
550    if isXFail:
551        ok = exitCode != 0 or err and not litConfig.ignoreStdErr
552        if ok:
553            status = Test.XFAIL
554        else:
555            status = Test.XPASS
556    else:
557        ok = exitCode == 0 and (not err or litConfig.ignoreStdErr)
558        if ok:
559            status = Test.PASS
560        else:
561            status = Test.FAIL
562
563    if ok:
564        return (status,'')
565
566    # Set a flag for formatTestOutput so it can explain why the test was
567    # considered to have failed, despite having an exit code of 0.
568    failDueToStderr = exitCode == 0 and err and not litConfig.ignoreStdErr
569
570    return formatTestOutput(status, out, err, exitCode, failDueToStderr, script)
571
572def executeShTest(test, litConfig, useExternalSh,
573                  extra_substitutions=[]):
574    if test.config.unsupported:
575        return (Test.UNSUPPORTED, 'Test is unsupported')
576
577    res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions)
578    if len(res) == 2:
579        return res
580
581    script, isXFail, tmpBase, execdir = res
582
583    if litConfig.noExecute:
584        return (Test.PASS, '')
585
586    # Create the output directory if it does not already exist.
587    Util.mkdir_p(os.path.dirname(tmpBase))
588
589    if useExternalSh:
590        res = executeScript(test, litConfig, tmpBase, script, execdir)
591    else:
592        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
593    if len(res) == 2:
594        return res
595
596    out,err,exitCode = res
597    if isXFail:
598        ok = exitCode != 0
599        if ok:
600            status = Test.XFAIL
601        else:
602            status = Test.XPASS
603    else:
604        ok = exitCode == 0
605        if ok:
606            status = Test.PASS
607        else:
608            status = Test.FAIL
609
610    if ok:
611        return (status,'')
612
613    # Sh tests are not considered to fail just from stderr output.
614    failDueToStderr = False
615
616    return formatTestOutput(status, out, err, exitCode, failDueToStderr, script)
617