TestRunner.py revision e09ccab0ce2986d1ba6950f2eccc46fd1935ae68 
1import os, signal, subprocess, sys
2import StringIO
3
4import ShUtil
5import Test
6import Util
7
8import platform
9import tempfile
10
11class InternalShellError(Exception):
12    def __init__(self, command, message):
13        self.command = command
14        self.message = message
15
16# Don't use close_fds on Windows.
17kUseCloseFDs = platform.system() != 'Windows'
18
19# Use temporary files to replace /dev/null on Windows.
20kAvoidDevNull = platform.system() == 'Windows'
21
22def executeCommand(command, cwd=None, env=None):
23    p = subprocess.Popen(command, cwd=cwd,
24                         stdin=subprocess.PIPE,
25                         stdout=subprocess.PIPE,
26                         stderr=subprocess.PIPE,
27                         env=env)
28    out,err = p.communicate()
29    exitCode = p.wait()
30
31    # Detect Ctrl-C in subprocess.
32    if exitCode == -signal.SIGINT:
33        raise KeyboardInterrupt
34
35    return out, err, exitCode
36
37def executeShCmd(cmd, cfg, cwd, results):
38    if isinstance(cmd, ShUtil.Seq):
39        if cmd.op == ';':
40            res = executeShCmd(cmd.lhs, cfg, cwd, results)
41            return executeShCmd(cmd.rhs, cfg, cwd, results)
42
43        if cmd.op == '&':
44            raise NotImplementedError,"unsupported test command: '&'"
45
46        if cmd.op == '||':
47            res = executeShCmd(cmd.lhs, cfg, cwd, results)
48            if res != 0:
49                res = executeShCmd(cmd.rhs, cfg, cwd, results)
50            return res
51        if cmd.op == '&&':
52            res = executeShCmd(cmd.lhs, cfg, cwd, results)
53            if res is None:
54                return res
55
56            if res == 0:
57                res = executeShCmd(cmd.rhs, cfg, cwd, results)
58            return res
59
60        raise ValueError,'Unknown shell command: %r' % cmd.op
61
62    assert isinstance(cmd, ShUtil.Pipeline)
63    procs = []
64    input = subprocess.PIPE
65    stderrTempFiles = []
66    opened_files = []
67    # To avoid deadlock, we use a single stderr stream for piped
68    # output. This is null until we have seen some output using
69    # stderr.
70    for i,j in enumerate(cmd.commands):
71        # Apply the redirections, we use (N,) as a sentinal to indicate stdin,
72        # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
73        # from a file are represented with a list [file, mode, file-object]
74        # where file-object is initially None.
75        redirects = [(0,), (1,), (2,)]
76        for r in j.redirects:
77            if r[0] == ('>',2):
78                redirects[2] = [r[1], 'w', None]
79            elif r[0] == ('>>',2):
80                redirects[2] = [r[1], 'a', None]
81            elif r[0] == ('>&',2) and r[1] in '012':
82                redirects[2] = redirects[int(r[1])]
83            elif r[0] == ('>&',) or r[0] == ('&>',):
84                redirects[1] = redirects[2] = [r[1], 'w', None]
85            elif r[0] == ('>',):
86                redirects[1] = [r[1], 'w', None]
87            elif r[0] == ('>>',):
88                redirects[1] = [r[1], 'a', None]
89            elif r[0] == ('<',):
90                redirects[0] = [r[1], 'r', None]
91            else:
92                raise NotImplementedError,"Unsupported redirect: %r" % (r,)
93
94        # Map from the final redirections to something subprocess can handle.
95        final_redirects = []
96        for index,r in enumerate(redirects):
97            if r == (0,):
98                result = input
99            elif r == (1,):
100                if index == 0:
101                    raise NotImplementedError,"Unsupported redirect for stdin"
102                elif index == 1:
103                    result = subprocess.PIPE
104                else:
105                    result = subprocess.STDOUT
106            elif r == (2,):
107                if index != 2:
108                    raise NotImplementedError,"Unsupported redirect on stdout"
109                result = subprocess.PIPE
110            else:
111                if r[2] is None:
112                    if kAvoidDevNull and r[0] == '/dev/null':
113                        r[2] = tempfile.TemporaryFile(mode=r[1])
114                    else:
115                        r[2] = open(r[0], r[1])
116                    # Workaround a Win32 and/or subprocess bug when appending.
117                    #
118                    # FIXME: Actually, this is probably an instance of PR6753.
119                    if r[1] == 'a':
120                        r[2].seek(0, 2)
121                    opened_files.append(r[2])
122                result = r[2]
123            final_redirects.append(result)
124
125        stdin, stdout, stderr = final_redirects
126
127        # If stderr wants to come from stdout, but stdout isn't a pipe, then put
128        # stderr on a pipe and treat it as stdout.
129        if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
130            stderr = subprocess.PIPE
131            stderrIsStdout = True
132        else:
133            stderrIsStdout = False
134
135            # Don't allow stderr on a PIPE except for the last
136            # process, this could deadlock.
137            #
138            # FIXME: This is slow, but so is deadlock.
139            if stderr == subprocess.PIPE and j != cmd.commands[-1]:
140                stderr = tempfile.TemporaryFile(mode='w+b')
141                stderrTempFiles.append((i, stderr))
142
143        # Resolve the executable path ourselves.
144        args = list(j.args)
145        args[0] = Util.which(args[0], cfg.environment['PATH'])
146        if not args[0]:
147            raise InternalShellError(j, '%r: command not found' % j.args[0])
148
149        procs.append(subprocess.Popen(args, cwd=cwd,
150                                      stdin = stdin,
151                                      stdout = stdout,
152                                      stderr = stderr,
153                                      env = cfg.environment,
154                                      close_fds = kUseCloseFDs))
155
156        # Immediately close stdin for any process taking stdin from us.
157        if stdin == subprocess.PIPE:
158            procs[-1].stdin.close()
159            procs[-1].stdin = None
160
161        # Update the current stdin source.
162        if stdout == subprocess.PIPE:
163            input = procs[-1].stdout
164        elif stderrIsStdout:
165            input = procs[-1].stderr
166        else:
167            input = subprocess.PIPE
168
169    # FIXME: There is probably still deadlock potential here. Yawn.
170    procData = [None] * len(procs)
171    procData[-1] = procs[-1].communicate()
172
173    for i in range(len(procs) - 1):
174        if procs[i].stdout is not None:
175            out = procs[i].stdout.read()
176        else:
177            out = ''
178        if procs[i].stderr is not None:
179            err = procs[i].stderr.read()
180        else:
181            err = ''
182        procData[i] = (out,err)
183
184    # Read stderr out of the temp files.
185    for i,f in stderrTempFiles:
186        f.seek(0, 0)
187        procData[i] = (procData[i][0], f.read())
188
189    exitCode = None
190    for i,(out,err) in enumerate(procData):
191        res = procs[i].wait()
192        # Detect Ctrl-C in subprocess.
193        if res == -signal.SIGINT:
194            raise KeyboardInterrupt
195
196        results.append((cmd.commands[i], out, err, res))
197        if cmd.pipe_err:
198            # Python treats the exit code as a signed char.
199            if res < 0:
200                exitCode = min(exitCode, res)
201            else:
202                exitCode = max(exitCode, res)
203        else:
204            exitCode = res
205
206    # Explicitly close any redirected files.
207    for f in opened_files:
208        f.close()
209
210    if cmd.negate:
211        exitCode = not exitCode
212
213    return exitCode
214
215def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
216    ln = ' &&\n'.join(commands)
217    try:
218        cmd = ShUtil.ShParser(ln, litConfig.isWindows).parse()
219    except:
220        return (Test.FAIL, "shell parser error on: %r" % ln)
221
222    results = []
223    try:
224        exitCode = executeShCmd(cmd, test.config, cwd, results)
225    except InternalShellError,e:
226        out = ''
227        err = e.message
228        exitCode = 255
229
230    out = err = ''
231    for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
232        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
233        out += 'Command %d Result: %r\n' % (i, res)
234        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
235        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
236
237    return out, err, exitCode
238
239def executeTclScriptInternal(test, litConfig, tmpBase, commands, cwd):
240    import TclUtil
241    cmds = []
242    for ln in commands:
243        # Given the unfortunate way LLVM's test are written, the line gets
244        # backslash substitution done twice.
245        ln = TclUtil.TclLexer(ln).lex_unquoted(process_all = True)
246
247        try:
248            tokens = list(TclUtil.TclLexer(ln).lex())
249        except:
250            return (Test.FAIL, "Tcl lexer error on: %r" % ln)
251
252        # Validate there are no control tokens.
253        for t in tokens:
254            if not isinstance(t, str):
255                return (Test.FAIL,
256                        "Invalid test line: %r containing %r" % (ln, t))
257
258        try:
259            cmds.append(TclUtil.TclExecCommand(tokens).parse_pipeline())
260        except:
261            return (Test.FAIL, "Tcl 'exec' parse error on: %r" % ln)
262
263    if litConfig.useValgrind:
264        for pipeline in cmds:
265            if pipeline.commands:
266                # Only valgrind the first command in each pipeline, to avoid
267                # valgrinding things like grep, not, and FileCheck.
268                cmd = pipeline.commands[0]
269                cmd.args = litConfig.valgrindArgs + cmd.args
270
271    cmd = cmds[0]
272    for c in cmds[1:]:
273        cmd = ShUtil.Seq(cmd, '&&', c)
274
275    # FIXME: This is lame, we shouldn't need bash. See PR5240.
276    bashPath = litConfig.getBashPath()
277    if litConfig.useTclAsSh and bashPath:
278        script = tmpBase + '.script'
279
280        # Write script file
281        f = open(script,'w')
282        print >>f, 'set -o pipefail'
283        cmd.toShell(f, pipefail = True)
284        f.close()
285
286        if 0:
287            print >>sys.stdout, cmd
288            print >>sys.stdout, open(script).read()
289            print >>sys.stdout
290            return '', '', 0
291
292        command = [litConfig.getBashPath(), script]
293        out,err,exitCode = executeCommand(command, cwd=cwd,
294                                          env=test.config.environment)
295
296        # Tcl commands fail on standard error output.
297        if err:
298            exitCode = 1
299            out = 'Command has output on stderr!\n\n' + out
300
301        return out,err,exitCode
302    else:
303        results = []
304        try:
305            exitCode = executeShCmd(cmd, test.config, cwd, results)
306        except InternalShellError,e:
307            results.append((e.command, '', e.message + '\n', 255))
308            exitCode = 255
309
310    out = err = ''
311
312    # Tcl commands fail on standard error output.
313    if [True for _,_,err,res in results if err]:
314        exitCode = 1
315        out += 'Command has output on stderr!\n\n'
316
317    for i,(cmd, cmd_out, cmd_err, res) in enumerate(results):
318        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
319        out += 'Command %d Result: %r\n' % (i, res)
320        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
321        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
322
323    return out, err, exitCode
324
325def executeScript(test, litConfig, tmpBase, commands, cwd):
326    script = tmpBase + '.script'
327    if litConfig.isWindows:
328        script += '.bat'
329
330    # Write script file
331    f = open(script,'w')
332    if litConfig.isWindows:
333        f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
334    else:
335        f.write(' &&\n'.join(commands))
336    f.write('\n')
337    f.close()
338
339    if litConfig.isWindows:
340        command = ['cmd','/c', script]
341    else:
342        command = ['/bin/sh', script]
343        if litConfig.useValgrind:
344            # FIXME: Running valgrind on sh is overkill. We probably could just
345            # run on clang with no real loss.
346            command = litConfig.valgrindArgs + command
347
348    return executeCommand(command, cwd=cwd, env=test.config.environment)
349
350def isExpectedFail(xfails, xtargets, target_triple):
351    # Check if any xfail matches this target.
352    for item in xfails:
353        if item == '*' or item in target_triple:
354            break
355    else:
356        return False
357
358    # If so, see if it is expected to pass on this target.
359    #
360    # FIXME: Rename XTARGET to something that makes sense, like XPASS.
361    for item in xtargets:
362        if item == '*' or item in target_triple:
363            return False
364
365    return True
366
367def parseIntegratedTestScript(test):
368    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
369    script and extract the lines to 'RUN' as well as 'XFAIL' and 'XTARGET'
370    information. The RUN lines also will have variable substitution performed.
371    """
372
373    # Get the temporary location, this is always relative to the test suite
374    # root, not test source root.
375    #
376    # FIXME: This should not be here?
377    sourcepath = test.getSourcePath()
378    execpath = test.getExecPath()
379    execdir,execbase = os.path.split(execpath)
380    tmpBase = os.path.join(execdir, 'Output', execbase)
381    if test.index is not None:
382        tmpBase += '_%d' % test.index
383
384    # We use #_MARKER_# to hide %% while we do the other substitutions.
385    substitutions = [('%%', '#_MARKER_#')]
386    substitutions.extend(test.config.substitutions)
387    substitutions.extend([('%s', sourcepath),
388                          ('%S', os.path.dirname(sourcepath)),
389                          ('%p', os.path.dirname(sourcepath)),
390                          ('%t', tmpBase + '.tmp'),
391                          # FIXME: Remove this once we kill DejaGNU.
392                          ('%abs_tmp', tmpBase + '.tmp'),
393                          ('#_MARKER_#', '%')])
394
395    # Collect the test lines from the script.
396    script = []
397    xfails = []
398    xtargets = []
399    for ln in open(sourcepath):
400        if 'RUN:' in ln:
401            # Isolate the command to run.
402            index = ln.index('RUN:')
403            ln = ln[index+4:]
404
405            # Trim trailing whitespace.
406            ln = ln.rstrip()
407
408            # Collapse lines with trailing '\\'.
409            if script and script[-1][-1] == '\\':
410                script[-1] = script[-1][:-1] + ln
411            else:
412                script.append(ln)
413        elif 'XFAIL:' in ln:
414            items = ln[ln.index('XFAIL:') + 6:].split(',')
415            xfails.extend([s.strip() for s in items])
416        elif 'XTARGET:' in ln:
417            items = ln[ln.index('XTARGET:') + 8:].split(',')
418            xtargets.extend([s.strip() for s in items])
419        elif 'END.' in ln:
420            # Check for END. lines.
421            if ln[ln.index('END.'):].strip() == 'END.':
422                break
423
424    # Apply substitutions to the script.
425    def processLine(ln):
426        # Apply substitutions
427        for a,b in substitutions:
428            ln = ln.replace(a,b)
429
430        # Strip the trailing newline and any extra whitespace.
431        return ln.strip()
432    script = map(processLine, script)
433
434    # Verify the script contains a run line.
435    if not script:
436        return (Test.UNRESOLVED, "Test has no run line!")
437
438    if script[-1][-1] == '\\':
439        return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')")
440
441    isXFail = isExpectedFail(xfails, xtargets, test.suite.config.target_triple)
442    return script,isXFail,tmpBase,execdir
443
444def formatTestOutput(status, out, err, exitCode, script):
445    output = StringIO.StringIO()
446    print >>output, "Script:"
447    print >>output, "--"
448    print >>output, '\n'.join(script)
449    print >>output, "--"
450    print >>output, "Exit Code: %r" % exitCode
451    print >>output, "Command Output (stdout):"
452    print >>output, "--"
453    output.write(out)
454    print >>output, "--"
455    print >>output, "Command Output (stderr):"
456    print >>output, "--"
457    output.write(err)
458    print >>output, "--"
459    return (status, output.getvalue())
460
461def executeTclTest(test, litConfig):
462    if test.config.unsupported:
463        return (Test.UNSUPPORTED, 'Test is unsupported')
464
465    res = parseIntegratedTestScript(test)
466    if len(res) == 2:
467        return res
468
469    script, isXFail, tmpBase, execdir = res
470
471    if litConfig.noExecute:
472        return (Test.PASS, '')
473
474    # Create the output directory if it does not already exist.
475    Util.mkdir_p(os.path.dirname(tmpBase))
476
477    res = executeTclScriptInternal(test, litConfig, tmpBase, script, execdir)
478    if len(res) == 2:
479        return res
480
481    out,err,exitCode = res
482    if isXFail:
483        ok = exitCode != 0
484        status = (Test.XPASS, Test.XFAIL)[ok]
485    else:
486        ok = exitCode == 0
487        status = (Test.FAIL, Test.PASS)[ok]
488
489    if ok:
490        return (status,'')
491
492    return formatTestOutput(status, out, err, exitCode, script)
493
494def executeShTest(test, litConfig, useExternalSh):
495    if test.config.unsupported:
496        return (Test.UNSUPPORTED, 'Test is unsupported')
497
498    res = parseIntegratedTestScript(test)
499    if len(res) == 2:
500        return res
501
502    script, isXFail, tmpBase, execdir = res
503
504    if litConfig.noExecute:
505        return (Test.PASS, '')
506
507    # Create the output directory if it does not already exist.
508    Util.mkdir_p(os.path.dirname(tmpBase))
509
510    if useExternalSh:
511        res = executeScript(test, litConfig, tmpBase, script, execdir)
512    else:
513        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
514    if len(res) == 2:
515        return res
516
517    out,err,exitCode = res
518    if isXFail:
519        ok = exitCode != 0
520        status = (Test.XPASS, Test.XFAIL)[ok]
521    else:
522        ok = exitCode == 0
523        status = (Test.FAIL, Test.PASS)[ok]
524
525    if ok:
526        return (status,'')
527
528    return formatTestOutput(status, out, err, exitCode, script)
529