TestRunner.py revision 7b7b90769ac1b1ed4ac3d506b57866d44130a36e 
1import os, signal, subprocess, sys
2import StringIO
3
4import ShUtil
5import Test
6import Util
7
8import platform
9import tempfile
10
11class InternalShellError(Exception):
12    def __init__(self, command, message):
13        self.command = command
14        self.message = message
15
16# Don't use close_fds on Windows.
17kUseCloseFDs = platform.system() != 'Windows'
18
19# Use temporary files to replace /dev/null on Windows.
20kAvoidDevNull = platform.system() == 'Windows'
21
22def executeCommand(command, cwd=None, env=None):
23    p = subprocess.Popen(command, cwd=cwd,
24                         stdin=subprocess.PIPE,
25                         stdout=subprocess.PIPE,
26                         stderr=subprocess.PIPE,
27                         env=env)
28    out,err = p.communicate()
29    exitCode = p.wait()
30
31    # Detect Ctrl-C in subprocess.
32    if exitCode == -signal.SIGINT:
33        raise KeyboardInterrupt
34
35    return out, err, exitCode
36
37def executeShCmd(cmd, cfg, cwd, results):
38    if isinstance(cmd, ShUtil.Seq):
39        if cmd.op == ';':
40            res = executeShCmd(cmd.lhs, cfg, cwd, results)
41            return executeShCmd(cmd.rhs, cfg, cwd, results)
42
43        if cmd.op == '&':
44            raise NotImplementedError,"unsupported test command: '&'"
45
46        if cmd.op == '||':
47            res = executeShCmd(cmd.lhs, cfg, cwd, results)
48            if res != 0:
49                res = executeShCmd(cmd.rhs, cfg, cwd, results)
50            return res
51        if cmd.op == '&&':
52            res = executeShCmd(cmd.lhs, cfg, cwd, results)
53            if res is None:
54                return res
55
56            if res == 0:
57                res = executeShCmd(cmd.rhs, cfg, cwd, results)
58            return res
59
60        raise ValueError,'Unknown shell command: %r' % cmd.op
61
62    assert isinstance(cmd, ShUtil.Pipeline)
63    procs = []
64    input = subprocess.PIPE
65    stderrTempFiles = []
66    # To avoid deadlock, we use a single stderr stream for piped
67    # output. This is null until we have seen some output using
68    # stderr.
69    for i,j in enumerate(cmd.commands):
70        # Apply the redirections, we use (N,) as a sentinal to indicate stdin,
71        # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
72        # from a file are represented with a list [file, mode, file-object]
73        # where file-object is initially None.
74        redirects = [(0,), (1,), (2,)]
75        for r in j.redirects:
76            if r[0] == ('>',2):
77                redirects[2] = [r[1], 'w', None]
78            elif r[0] == ('>>',2):
79                redirects[2] = [r[1], 'a', None]
80            elif r[0] == ('>&',2) and r[1] in '012':
81                redirects[2] = redirects[int(r[1])]
82            elif r[0] == ('>&',) or r[0] == ('&>',):
83                redirects[1] = redirects[2] = [r[1], 'w', None]
84            elif r[0] == ('>',):
85                redirects[1] = [r[1], 'w', None]
86            elif r[0] == ('>>',):
87                redirects[1] = [r[1], 'a', None]
88            elif r[0] == ('<',):
89                redirects[0] = [r[1], 'r', None]
90            else:
91                raise NotImplementedError,"Unsupported redirect: %r" % (r,)
92
93        # Map from the final redirections to something subprocess can handle.
94        final_redirects = []
95        for index,r in enumerate(redirects):
96            if r == (0,):
97                result = input
98            elif r == (1,):
99                if index == 0:
100                    raise NotImplementedError,"Unsupported redirect for stdin"
101                elif index == 1:
102                    result = subprocess.PIPE
103                else:
104                    result = subprocess.STDOUT
105            elif r == (2,):
106                if index != 2:
107                    raise NotImplementedError,"Unsupported redirect on stdout"
108                result = subprocess.PIPE
109            else:
110                if r[2] is None:
111                    if kAvoidDevNull and r[0] == '/dev/null':
112                        r[2] = tempfile.TemporaryFile(mode=r[1])
113                    else:
114                        r[2] = open(r[0], r[1])
115                    # Workaround a Win32 and/or subprocess bug when appending.
116                    if r[1] == 'a':
117                        r[2].seek(0, 2)
118                result = r[2]
119            final_redirects.append(result)
120
121        stdin, stdout, stderr = final_redirects
122
123        # If stderr wants to come from stdout, but stdout isn't a pipe, then put
124        # stderr on a pipe and treat it as stdout.
125        if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
126            stderr = subprocess.PIPE
127            stderrIsStdout = True
128        else:
129            stderrIsStdout = False
130
131            # Don't allow stderr on a PIPE except for the last
132            # process, this could deadlock.
133            #
134            # FIXME: This is slow, but so is deadlock.
135            if stderr == subprocess.PIPE and j != cmd.commands[-1]:
136                stderr = tempfile.TemporaryFile(mode='w+b')
137                stderrTempFiles.append((i, stderr))
138
139        # Resolve the executable path ourselves.
140        args = list(j.args)
141        args[0] = Util.which(args[0], cfg.environment['PATH'])
142        if not args[0]:
143            raise InternalShellError(j, '%r: command not found' % j.args[0])
144
145        procs.append(subprocess.Popen(args, cwd=cwd,
146                                      stdin = stdin,
147                                      stdout = stdout,
148                                      stderr = stderr,
149                                      env = cfg.environment,
150                                      close_fds = kUseCloseFDs))
151
152        # Immediately close stdin for any process taking stdin from us.
153        if stdin == subprocess.PIPE:
154            procs[-1].stdin.close()
155            procs[-1].stdin = None
156
157        # Update the current stdin source.
158        if stdout == subprocess.PIPE:
159            input = procs[-1].stdout
160        elif stderrIsStdout:
161            input = procs[-1].stderr
162        else:
163            input = subprocess.PIPE
164
165    # FIXME: There is probably still deadlock potential here. Yawn.
166    procData = [None] * len(procs)
167    procData[-1] = procs[-1].communicate()
168
169    for i in range(len(procs) - 1):
170        if procs[i].stdout is not None:
171            out = procs[i].stdout.read()
172        else:
173            out = ''
174        if procs[i].stderr is not None:
175            err = procs[i].stderr.read()
176        else:
177            err = ''
178        procData[i] = (out,err)
179
180    # Read stderr out of the temp files.
181    for i,f in stderrTempFiles:
182        f.seek(0, 0)
183        procData[i] = (procData[i][0], f.read())
184
185    exitCode = None
186    for i,(out,err) in enumerate(procData):
187        res = procs[i].wait()
188        # Detect Ctrl-C in subprocess.
189        if res == -signal.SIGINT:
190            raise KeyboardInterrupt
191
192        results.append((cmd.commands[i], out, err, res))
193        if cmd.pipe_err:
194            # Python treats the exit code as a signed char.
195            if res < 0:
196                exitCode = min(exitCode, res)
197            else:
198                exitCode = max(exitCode, res)
199        else:
200            exitCode = res
201
202    if cmd.negate:
203        exitCode = not exitCode
204
205    return exitCode
206
207def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
208    ln = ' &&\n'.join(commands)
209    try:
210        cmd = ShUtil.ShParser(ln, litConfig.isWindows).parse()
211    except:
212        return (Test.FAIL, "shell parser error on: %r" % ln)
213
214    results = []
215    try:
216        exitCode = executeShCmd(cmd, test.config, cwd, results)
217    except InternalShellError,e:
218        out = ''
219        err = e.message
220        exitCode = 255
221
222    out = err = ''
223    for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
224        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
225        out += 'Command %d Result: %r\n' % (i, res)
226        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
227        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
228
229    return out, err, exitCode
230
231def executeTclScriptInternal(test, litConfig, tmpBase, commands, cwd):
232    import TclUtil
233    cmds = []
234    for ln in commands:
235        # Given the unfortunate way LLVM's test are written, the line gets
236        # backslash substitution done twice.
237        ln = TclUtil.TclLexer(ln).lex_unquoted(process_all = True)
238
239        try:
240            tokens = list(TclUtil.TclLexer(ln).lex())
241        except:
242            return (Test.FAIL, "Tcl lexer error on: %r" % ln)
243
244        # Validate there are no control tokens.
245        for t in tokens:
246            if not isinstance(t, str):
247                return (Test.FAIL,
248                        "Invalid test line: %r containing %r" % (ln, t))
249
250        try:
251            cmds.append(TclUtil.TclExecCommand(tokens).parse_pipeline())
252        except:
253            return (Test.FAIL, "Tcl 'exec' parse error on: %r" % ln)
254
255    cmd = cmds[0]
256    for c in cmds[1:]:
257        cmd = ShUtil.Seq(cmd, '&&', c)
258
259    # FIXME: This is lame, we shouldn't need bash. See PR5240.
260    bashPath = litConfig.getBashPath()
261    if litConfig.useTclAsSh and bashPath:
262        script = tmpBase + '.script'
263
264        # Write script file
265        f = open(script,'w')
266        print >>f, 'set -o pipefail'
267        cmd.toShell(f, pipefail = True)
268        f.close()
269
270        if 0:
271            print >>sys.stdout, cmd
272            print >>sys.stdout, open(script).read()
273            print >>sys.stdout
274            return '', '', 0
275
276        command = [litConfig.getBashPath(), script]
277        out,err,exitCode = executeCommand(command, cwd=cwd,
278                                          env=test.config.environment)
279
280        # Tcl commands fail on standard error output.
281        if err:
282            exitCode = 1
283            out = 'Command has output on stderr!\n\n' + out
284
285        return out,err,exitCode
286    else:
287        results = []
288        try:
289            exitCode = executeShCmd(cmd, test.config, cwd, results)
290        except InternalShellError,e:
291            results.append((e.command, '', e.message + '\n', 255))
292            exitCode = 255
293
294    out = err = ''
295
296    # Tcl commands fail on standard error output.
297    if [True for _,_,err,res in results if err]:
298        exitCode = 1
299        out += 'Command has output on stderr!\n\n'
300
301    for i,(cmd, cmd_out, cmd_err, res) in enumerate(results):
302        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
303        out += 'Command %d Result: %r\n' % (i, res)
304        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
305        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
306
307    return out, err, exitCode
308
309def executeScript(test, litConfig, tmpBase, commands, cwd):
310    script = tmpBase + '.script'
311    if litConfig.isWindows:
312        script += '.bat'
313
314    # Write script file
315    f = open(script,'w')
316    if litConfig.isWindows:
317        f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
318    else:
319        f.write(' &&\n'.join(commands))
320    f.write('\n')
321    f.close()
322
323    if litConfig.isWindows:
324        command = ['cmd','/c', script]
325    else:
326        command = ['/bin/sh', script]
327        if litConfig.useValgrind:
328            # FIXME: Running valgrind on sh is overkill. We probably could just
329            # run on clang with no real loss.
330            valgrindArgs = ['valgrind', '-q',
331                            '--tool=memcheck', '--trace-children=yes',
332                            '--error-exitcode=123']
333            valgrindArgs.extend(litConfig.valgrindArgs)
334
335            command = valgrindArgs + command
336
337    return executeCommand(command, cwd=cwd, env=test.config.environment)
338
339def isExpectedFail(xfails, xtargets, target_triple):
340    # Check if any xfail matches this target.
341    for item in xfails:
342        if item == '*' or item in target_triple:
343            break
344    else:
345        return False
346
347    # If so, see if it is expected to pass on this target.
348    #
349    # FIXME: Rename XTARGET to something that makes sense, like XPASS.
350    for item in xtargets:
351        if item == '*' or item in target_triple:
352            return False
353
354    return True
355
356def parseIntegratedTestScript(test):
357    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
358    script and extract the lines to 'RUN' as well as 'XFAIL' and 'XTARGET'
359    information. The RUN lines also will have variable substitution performed.
360    """
361
362    # Get the temporary location, this is always relative to the test suite
363    # root, not test source root.
364    #
365    # FIXME: This should not be here?
366    sourcepath = test.getSourcePath()
367    execpath = test.getExecPath()
368    execdir,execbase = os.path.split(execpath)
369    tmpBase = os.path.join(execdir, 'Output', execbase)
370    if test.index is not None:
371        tmpBase += '_%d' % test.index
372
373    # We use #_MARKER_# to hide %% while we do the other substitutions.
374    substitutions = [('%%', '#_MARKER_#')]
375    substitutions.extend(test.config.substitutions)
376    substitutions.extend([('%s', sourcepath),
377                          ('%S', os.path.dirname(sourcepath)),
378                          ('%p', os.path.dirname(sourcepath)),
379                          ('%t', tmpBase + '.tmp'),
380                          # FIXME: Remove this once we kill DejaGNU.
381                          ('%abs_tmp', tmpBase + '.tmp'),
382                          ('#_MARKER_#', '%')])
383
384    # Collect the test lines from the script.
385    script = []
386    xfails = []
387    xtargets = []
388    ignoredAny = False
389    for ln in open(sourcepath):
390        if 'IF(' in ln:
391            # Required syntax here is IF(condition(value)):
392            index = ln.index('IF(')
393            ln = ln[index+3:]
394            index = ln.index('(')
395            if index is -1:
396                return (Test.UNRESOLVED, "ill-formed IF at '"+ln[:10]+"'")
397            condition = ln[:index]
398            ln = ln[index+1:]
399            index = ln.index(')')
400            if index is -1 or ln[index:index+3] != ')):':
401                return (Test.UNRESOLVED, "ill-formed IF at '"+ln[:10]+"'")
402            value = ln[:index]
403            ln = ln[index+3:]
404
405            # Actually test the condition.
406            if condition not in test.config.conditions:
407                return (Test.UNRESOLVED, "unknown condition '"+condition+"'")
408            if not test.config.conditions[condition](value):
409                ignoredAny = True
410                continue
411
412        if 'RUN:' in ln:
413            # Isolate the command to run.
414            index = ln.index('RUN:')
415            ln = ln[index+4:]
416
417            # Trim trailing whitespace.
418            ln = ln.rstrip()
419
420            # Collapse lines with trailing '\\'.
421            if script and script[-1][-1] == '\\':
422                script[-1] = script[-1][:-1] + ln
423            else:
424                script.append(ln)
425        elif 'XFAIL:' in ln:
426            items = ln[ln.index('XFAIL:') + 6:].split(',')
427            xfails.extend([s.strip() for s in items])
428        elif 'XTARGET:' in ln:
429            items = ln[ln.index('XTARGET:') + 8:].split(',')
430            xtargets.extend([s.strip() for s in items])
431        elif 'END.' in ln:
432            # Check for END. lines.
433            if ln[ln.index('END.'):].strip() == 'END.':
434                break
435
436    # Apply substitutions to the script.
437    def processLine(ln):
438        # Apply substitutions
439        for a,b in substitutions:
440            ln = ln.replace(a,b)
441
442        # Strip the trailing newline and any extra whitespace.
443        return ln.strip()
444    script = map(processLine, script)
445
446    # Verify the script contains a run line.
447    if not script:
448        if ignoredAny:
449            return (Test.UNSUPPORTED, "Test has only ignored run lines")
450        return (Test.UNRESOLVED, "Test has no run line!")
451
452    if script[-1][-1] == '\\':
453        return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')")
454
455    isXFail = isExpectedFail(xfails, xtargets, test.suite.config.target_triple)
456    return script,isXFail,tmpBase,execdir
457
458def formatTestOutput(status, out, err, exitCode, script):
459    output = StringIO.StringIO()
460    print >>output, "Script:"
461    print >>output, "--"
462    print >>output, '\n'.join(script)
463    print >>output, "--"
464    print >>output, "Exit Code: %r" % exitCode
465    print >>output, "Command Output (stdout):"
466    print >>output, "--"
467    output.write(out)
468    print >>output, "--"
469    print >>output, "Command Output (stderr):"
470    print >>output, "--"
471    output.write(err)
472    print >>output, "--"
473    return (status, output.getvalue())
474
475def executeTclTest(test, litConfig):
476    if test.config.unsupported:
477        return (Test.UNSUPPORTED, 'Test is unsupported')
478
479    res = parseIntegratedTestScript(test)
480    if len(res) == 2:
481        return res
482
483    script, isXFail, tmpBase, execdir = res
484
485    if litConfig.noExecute:
486        return (Test.PASS, '')
487
488    # Create the output directory if it does not already exist.
489    Util.mkdir_p(os.path.dirname(tmpBase))
490
491    res = executeTclScriptInternal(test, litConfig, tmpBase, script, execdir)
492    if len(res) == 2:
493        return res
494
495    out,err,exitCode = res
496    if isXFail:
497        ok = exitCode != 0
498        status = (Test.XPASS, Test.XFAIL)[ok]
499    else:
500        ok = exitCode == 0
501        status = (Test.FAIL, Test.PASS)[ok]
502
503    if ok:
504        return (status,'')
505
506    return formatTestOutput(status, out, err, exitCode, script)
507
508def executeShTest(test, litConfig, useExternalSh):
509    if test.config.unsupported:
510        return (Test.UNSUPPORTED, 'Test is unsupported')
511
512    res = parseIntegratedTestScript(test)
513    if len(res) == 2:
514        return res
515
516    script, isXFail, tmpBase, execdir = res
517
518    if litConfig.noExecute:
519        return (Test.PASS, '')
520
521    # Create the output directory if it does not already exist.
522    Util.mkdir_p(os.path.dirname(tmpBase))
523
524    if useExternalSh:
525        res = executeScript(test, litConfig, tmpBase, script, execdir)
526    else:
527        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
528    if len(res) == 2:
529        return res
530
531    out,err,exitCode = res
532    if isXFail:
533        ok = exitCode != 0
534        status = (Test.XPASS, Test.XFAIL)[ok]
535    else:
536        ok = exitCode == 0
537        status = (Test.FAIL, Test.PASS)[ok]
538
539    if ok:
540        return (status,'')
541
542    return formatTestOutput(status, out, err, exitCode, script)
543