TestRunner.py revision 7a0be17c087ad7723e0ec65e63f4eb80ce9b5cef 
1import os, signal, subprocess, sys
2import StringIO
3
4import ShUtil
5import Test
6import Util
7
8import platform
9import tempfile
10
11import re
12
13class InternalShellError(Exception):
14    def __init__(self, command, message):
15        self.command = command
16        self.message = message
17
18kIsWindows = platform.system() == 'Windows'
19
20# Don't use close_fds on Windows.
21kUseCloseFDs = not kIsWindows
22
23# Use temporary files to replace /dev/null on Windows.
24kAvoidDevNull = kIsWindows
25
26def executeCommand(command, cwd=None, env=None):
27    p = subprocess.Popen(command, cwd=cwd,
28                         stdin=subprocess.PIPE,
29                         stdout=subprocess.PIPE,
30                         stderr=subprocess.PIPE,
31                         env=env)
32    out,err = p.communicate()
33    exitCode = p.wait()
34
35    # Detect Ctrl-C in subprocess.
36    if exitCode == -signal.SIGINT:
37        raise KeyboardInterrupt
38
39    return out, err, exitCode
40
41def executeShCmd(cmd, cfg, cwd, results):
42    if isinstance(cmd, ShUtil.Seq):
43        if cmd.op == ';':
44            res = executeShCmd(cmd.lhs, cfg, cwd, results)
45            return executeShCmd(cmd.rhs, cfg, cwd, results)
46
47        if cmd.op == '&':
48            raise NotImplementedError,"unsupported test command: '&'"
49
50        if cmd.op == '||':
51            res = executeShCmd(cmd.lhs, cfg, cwd, results)
52            if res != 0:
53                res = executeShCmd(cmd.rhs, cfg, cwd, results)
54            return res
55        if cmd.op == '&&':
56            res = executeShCmd(cmd.lhs, cfg, cwd, results)
57            if res is None:
58                return res
59
60            if res == 0:
61                res = executeShCmd(cmd.rhs, cfg, cwd, results)
62            return res
63
64        raise ValueError,'Unknown shell command: %r' % cmd.op
65
66    assert isinstance(cmd, ShUtil.Pipeline)
67    procs = []
68    input = subprocess.PIPE
69    stderrTempFiles = []
70    opened_files = []
71    named_temp_files = []
72    # To avoid deadlock, we use a single stderr stream for piped
73    # output. This is null until we have seen some output using
74    # stderr.
75    for i,j in enumerate(cmd.commands):
76        # Apply the redirections, we use (N,) as a sentinal to indicate stdin,
77        # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
78        # from a file are represented with a list [file, mode, file-object]
79        # where file-object is initially None.
80        redirects = [(0,), (1,), (2,)]
81        for r in j.redirects:
82            if r[0] == ('>',2):
83                redirects[2] = [r[1], 'w', None]
84            elif r[0] == ('>>',2):
85                redirects[2] = [r[1], 'a', None]
86            elif r[0] == ('>&',2) and r[1] in '012':
87                redirects[2] = redirects[int(r[1])]
88            elif r[0] == ('>&',) or r[0] == ('&>',):
89                redirects[1] = redirects[2] = [r[1], 'w', None]
90            elif r[0] == ('>',):
91                redirects[1] = [r[1], 'w', None]
92            elif r[0] == ('>>',):
93                redirects[1] = [r[1], 'a', None]
94            elif r[0] == ('<',):
95                redirects[0] = [r[1], 'r', None]
96            else:
97                raise NotImplementedError,"Unsupported redirect: %r" % (r,)
98
99        # Map from the final redirections to something subprocess can handle.
100        final_redirects = []
101        for index,r in enumerate(redirects):
102            if r == (0,):
103                result = input
104            elif r == (1,):
105                if index == 0:
106                    raise NotImplementedError,"Unsupported redirect for stdin"
107                elif index == 1:
108                    result = subprocess.PIPE
109                else:
110                    result = subprocess.STDOUT
111            elif r == (2,):
112                if index != 2:
113                    raise NotImplementedError,"Unsupported redirect on stdout"
114                result = subprocess.PIPE
115            else:
116                if r[2] is None:
117                    if kAvoidDevNull and r[0] == '/dev/null':
118                        r[2] = tempfile.TemporaryFile(mode=r[1])
119                    else:
120                        r[2] = open(r[0], r[1])
121                    # Workaround a Win32 and/or subprocess bug when appending.
122                    #
123                    # FIXME: Actually, this is probably an instance of PR6753.
124                    if r[1] == 'a':
125                        r[2].seek(0, 2)
126                    opened_files.append(r[2])
127                result = r[2]
128            final_redirects.append(result)
129
130        stdin, stdout, stderr = final_redirects
131
132        # If stderr wants to come from stdout, but stdout isn't a pipe, then put
133        # stderr on a pipe and treat it as stdout.
134        if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
135            stderr = subprocess.PIPE
136            stderrIsStdout = True
137        else:
138            stderrIsStdout = False
139
140            # Don't allow stderr on a PIPE except for the last
141            # process, this could deadlock.
142            #
143            # FIXME: This is slow, but so is deadlock.
144            if stderr == subprocess.PIPE and j != cmd.commands[-1]:
145                stderr = tempfile.TemporaryFile(mode='w+b')
146                stderrTempFiles.append((i, stderr))
147
148        # Resolve the executable path ourselves.
149        args = list(j.args)
150        args[0] = Util.which(args[0], cfg.environment['PATH'])
151        if not args[0]:
152            raise InternalShellError(j, '%r: command not found' % j.args[0])
153
154        # Replace uses of /dev/null with temporary files.
155        if kAvoidDevNull:
156            for i,arg in enumerate(args):
157                if arg == "/dev/null":
158                    f = tempfile.NamedTemporaryFile(delete=False)
159                    f.close()
160                    named_temp_files.append(f.name)
161                    args[i] = f.name
162
163        procs.append(subprocess.Popen(args, cwd=cwd,
164                                      stdin = stdin,
165                                      stdout = stdout,
166                                      stderr = stderr,
167                                      env = cfg.environment,
168                                      close_fds = kUseCloseFDs))
169
170        # Immediately close stdin for any process taking stdin from us.
171        if stdin == subprocess.PIPE:
172            procs[-1].stdin.close()
173            procs[-1].stdin = None
174
175        # Update the current stdin source.
176        if stdout == subprocess.PIPE:
177            input = procs[-1].stdout
178        elif stderrIsStdout:
179            input = procs[-1].stderr
180        else:
181            input = subprocess.PIPE
182
183    # Explicitly close any redirected files. We need to do this now because we
184    # need to release any handles we may have on the temporary files (important
185    # on Win32, for example). Since we have already spawned the subprocess, our
186    # handles have already been transferred so we do not need them anymore.
187    for f in opened_files:
188        f.close()
189
190    # FIXME: There is probably still deadlock potential here. Yawn.
191    procData = [None] * len(procs)
192    procData[-1] = procs[-1].communicate()
193
194    for i in range(len(procs) - 1):
195        if procs[i].stdout is not None:
196            out = procs[i].stdout.read()
197        else:
198            out = ''
199        if procs[i].stderr is not None:
200            err = procs[i].stderr.read()
201        else:
202            err = ''
203        procData[i] = (out,err)
204
205    # Read stderr out of the temp files.
206    for i,f in stderrTempFiles:
207        f.seek(0, 0)
208        procData[i] = (procData[i][0], f.read())
209
210    exitCode = None
211    for i,(out,err) in enumerate(procData):
212        res = procs[i].wait()
213        # Detect Ctrl-C in subprocess.
214        if res == -signal.SIGINT:
215            raise KeyboardInterrupt
216
217        results.append((cmd.commands[i], out, err, res))
218        if cmd.pipe_err:
219            # Python treats the exit code as a signed char.
220            if res < 0:
221                exitCode = min(exitCode, res)
222            else:
223                exitCode = max(exitCode, res)
224        else:
225            exitCode = res
226
227    # Remove any named temporary files we created.
228    for f in named_temp_files:
229        try:
230            os.remove(f)
231        except OSError:
232            pass
233
234    if cmd.negate:
235        exitCode = not exitCode
236
237    return exitCode
238
239def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
240    ln = ' &&\n'.join(commands)
241    try:
242        cmd = ShUtil.ShParser(ln, litConfig.isWindows).parse()
243    except:
244        return (Test.FAIL, "shell parser error on: %r" % ln)
245
246    results = []
247    try:
248        exitCode = executeShCmd(cmd, test.config, cwd, results)
249    except InternalShellError,e:
250        out = ''
251        err = e.message
252        exitCode = 255
253
254    out = err = ''
255    for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
256        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
257        out += 'Command %d Result: %r\n' % (i, res)
258        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
259        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
260
261    return out, err, exitCode
262
263def executeTclScriptInternal(test, litConfig, tmpBase, commands, cwd):
264    import TclUtil
265    cmds = []
266    for ln in commands:
267        # Given the unfortunate way LLVM's test are written, the line gets
268        # backslash substitution done twice.
269        ln = TclUtil.TclLexer(ln).lex_unquoted(process_all = True)
270
271        try:
272            tokens = list(TclUtil.TclLexer(ln).lex())
273        except:
274            return (Test.FAIL, "Tcl lexer error on: %r" % ln)
275
276        # Validate there are no control tokens.
277        for t in tokens:
278            if not isinstance(t, str):
279                return (Test.FAIL,
280                        "Invalid test line: %r containing %r" % (ln, t))
281
282        try:
283            cmds.append(TclUtil.TclExecCommand(tokens).parse_pipeline())
284        except:
285            return (Test.FAIL, "Tcl 'exec' parse error on: %r" % ln)
286
287    if litConfig.useValgrind:
288        for pipeline in cmds:
289            if pipeline.commands:
290                # Only valgrind the first command in each pipeline, to avoid
291                # valgrinding things like grep, not, and FileCheck.
292                cmd = pipeline.commands[0]
293                cmd.args = litConfig.valgrindArgs + cmd.args
294
295    cmd = cmds[0]
296    for c in cmds[1:]:
297        cmd = ShUtil.Seq(cmd, '&&', c)
298
299    # FIXME: This is lame, we shouldn't need bash. See PR5240.
300    bashPath = litConfig.getBashPath()
301    if litConfig.useTclAsSh and bashPath:
302        script = tmpBase + '.script'
303
304        # Write script file
305        f = open(script,'w')
306        print >>f, 'set -o pipefail'
307        cmd.toShell(f, pipefail = True)
308        f.close()
309
310        if 0:
311            print >>sys.stdout, cmd
312            print >>sys.stdout, open(script).read()
313            print >>sys.stdout
314            return '', '', 0
315
316        command = [litConfig.getBashPath(), script]
317        out,err,exitCode = executeCommand(command, cwd=cwd,
318                                          env=test.config.environment)
319
320        return out,err,exitCode
321    else:
322        results = []
323        try:
324            exitCode = executeShCmd(cmd, test.config, cwd, results)
325        except InternalShellError,e:
326            results.append((e.command, '', e.message + '\n', 255))
327            exitCode = 255
328
329    out = err = ''
330
331    for i,(cmd, cmd_out, cmd_err, res) in enumerate(results):
332        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
333        out += 'Command %d Result: %r\n' % (i, res)
334        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
335        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
336
337    return out, err, exitCode
338
339def executeScript(test, litConfig, tmpBase, commands, cwd):
340    script = tmpBase + '.script'
341    if litConfig.isWindows:
342        script += '.bat'
343
344    # Write script file
345    f = open(script,'w')
346    if litConfig.isWindows:
347        f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
348    else:
349        f.write(' &&\n'.join(commands))
350    f.write('\n')
351    f.close()
352
353    if litConfig.isWindows:
354        command = ['cmd','/c', script]
355    else:
356        command = ['/bin/sh', script]
357        if litConfig.useValgrind:
358            # FIXME: Running valgrind on sh is overkill. We probably could just
359            # run on clang with no real loss.
360            command = litConfig.valgrindArgs + command
361
362    return executeCommand(command, cwd=cwd, env=test.config.environment)
363
364def isExpectedFail(xfails, xtargets, target_triple):
365    # Check if any xfail matches this target.
366    for item in xfails:
367        if item == '*' or item in target_triple:
368            break
369    else:
370        return False
371
372    # If so, see if it is expected to pass on this target.
373    #
374    # FIXME: Rename XTARGET to something that makes sense, like XPASS.
375    for item in xtargets:
376        if item == '*' or item in target_triple:
377            return False
378
379    return True
380
381def parseIntegratedTestScript(test, normalize_slashes=False):
382    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
383    script and extract the lines to 'RUN' as well as 'XFAIL' and 'XTARGET'
384    information. The RUN lines also will have variable substitution performed.
385    """
386
387    # Get the temporary location, this is always relative to the test suite
388    # root, not test source root.
389    #
390    # FIXME: This should not be here?
391    sourcepath = test.getSourcePath()
392    sourcedir = os.path.dirname(sourcepath)
393    execpath = test.getExecPath()
394    execdir,execbase = os.path.split(execpath)
395    tmpBase = os.path.join(execdir, 'Output', execbase)
396    if test.index is not None:
397        tmpBase += '_%d' % test.index
398
399    # Normalize slashes, if requested.
400    if normalize_slashes:
401        sourcepath = sourcepath.replace('\\', '/')
402        sourcedir = sourcedir.replace('\\', '/')
403        tmpBase = tmpBase.replace('\\', '/')
404
405    # We use #_MARKER_# to hide %% while we do the other substitutions.
406    substitutions = [('%%', '#_MARKER_#')]
407    substitutions.extend(test.config.substitutions)
408    substitutions.extend([('%s', sourcepath),
409                          ('%S', sourcedir),
410                          ('%p', sourcedir),
411                          ('%t', tmpBase + '.tmp'),
412                          # FIXME: Remove this once we kill DejaGNU.
413                          ('%abs_tmp', tmpBase + '.tmp'),
414                          ('#_MARKER_#', '%')])
415
416    # Collect the test lines from the script.
417    script = []
418    xfails = []
419    xtargets = []
420    requires = []
421    for ln in open(sourcepath):
422        if 'RUN:' in ln:
423            # Isolate the command to run.
424            index = ln.index('RUN:')
425            ln = ln[index+4:]
426
427            # Trim trailing whitespace.
428            ln = ln.rstrip()
429
430            # Collapse lines with trailing '\\'.
431            if script and script[-1][-1] == '\\':
432                script[-1] = script[-1][:-1] + ln
433            else:
434                script.append(ln)
435        elif 'XFAIL:' in ln:
436            items = ln[ln.index('XFAIL:') + 6:].split(',')
437            xfails.extend([s.strip() for s in items])
438        elif 'XTARGET:' in ln:
439            items = ln[ln.index('XTARGET:') + 8:].split(',')
440            xtargets.extend([s.strip() for s in items])
441        elif 'REQUIRES:' in ln:
442            items = ln[ln.index('REQUIRES:') + 9:].split(',')
443            requires.extend([s.strip() for s in items])
444        elif 'END.' in ln:
445            # Check for END. lines.
446            if ln[ln.index('END.'):].strip() == 'END.':
447                break
448
449    # Apply substitutions to the script.  Allow full regular
450    # expression syntax.  Replace each matching occurrence of regular
451    # expression pattern a with substitution b in line ln.
452    def processLine(ln):
453        # Apply substitutions
454        for a,b in substitutions:
455            if kIsWindows:
456                b = b.replace("\\","\\\\")
457            ln = re.sub(a, b, ln)
458
459        # Strip the trailing newline and any extra whitespace.
460        return ln.strip()
461    script = map(processLine, script)
462
463    # Verify the script contains a run line.
464    if not script:
465        return (Test.UNRESOLVED, "Test has no run line!")
466
467    # Check for unterminated run lines.
468    if script[-1][-1] == '\\':
469        return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')")
470
471    # Check that we have the required features:
472    missing_required_features = [f for f in requires
473                                 if f not in test.config.available_features]
474    if missing_required_features:
475        msg = ', '.join(missing_required_features)
476        return (Test.UNSUPPORTED,
477                "Test requires the following features: %s" % msg)
478
479    isXFail = isExpectedFail(xfails, xtargets, test.suite.config.target_triple)
480    return script,isXFail,tmpBase,execdir
481
482def formatTestOutput(status, out, err, exitCode, failDueToStderr, script):
483    output = StringIO.StringIO()
484    print >>output, "Script:"
485    print >>output, "--"
486    print >>output, '\n'.join(script)
487    print >>output, "--"
488    print >>output, "Exit Code: %r" % exitCode,
489    if failDueToStderr:
490        print >>output, "(but there was output on stderr)"
491    else:
492        print >>output
493    if out:
494        print >>output, "Command Output (stdout):"
495        print >>output, "--"
496        output.write(out)
497        print >>output, "--"
498    if err:
499        print >>output, "Command Output (stderr):"
500        print >>output, "--"
501        output.write(err)
502        print >>output, "--"
503    return (status, output.getvalue())
504
505def executeTclTest(test, litConfig):
506    if test.config.unsupported:
507        return (Test.UNSUPPORTED, 'Test is unsupported')
508
509    # Parse the test script, normalizing slashes in substitutions on Windows
510    # (since otherwise Tcl style lexing will treat them as escapes).
511    res = parseIntegratedTestScript(test, normalize_slashes=kIsWindows)
512    if len(res) == 2:
513        return res
514
515    script, isXFail, tmpBase, execdir = res
516
517    if litConfig.noExecute:
518        return (Test.PASS, '')
519
520    # Create the output directory if it does not already exist.
521    Util.mkdir_p(os.path.dirname(tmpBase))
522
523    res = executeTclScriptInternal(test, litConfig, tmpBase, script, execdir)
524    if len(res) == 2:
525        return res
526
527    # Test for failure. In addition to the exit code, Tcl commands are
528    # considered to fail if there is any standard error output.
529    out,err,exitCode = res
530    if isXFail:
531        ok = exitCode != 0 or err
532        if ok:
533            status = Test.XFAIL
534        else:
535            status = Test.XPASS
536    else:
537        ok = exitCode == 0 and not err
538        if ok:
539            status = Test.PASS
540        else:
541            status = Test.FAIL
542
543    if ok:
544        return (status,'')
545
546    # Set a flag for formatTestOutput so it can explain why the test was
547    # considered to have failed, despite having an exit code of 0.
548    failDueToStderr = exitCode == 0 and err
549
550    return formatTestOutput(status, out, err, exitCode, failDueToStderr, script)
551
552def executeShTest(test, litConfig, useExternalSh):
553    if test.config.unsupported:
554        return (Test.UNSUPPORTED, 'Test is unsupported')
555
556    res = parseIntegratedTestScript(test)
557    if len(res) == 2:
558        return res
559
560    script, isXFail, tmpBase, execdir = res
561
562    if litConfig.noExecute:
563        return (Test.PASS, '')
564
565    # Create the output directory if it does not already exist.
566    Util.mkdir_p(os.path.dirname(tmpBase))
567
568    if useExternalSh:
569        res = executeScript(test, litConfig, tmpBase, script, execdir)
570    else:
571        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
572    if len(res) == 2:
573        return res
574
575    out,err,exitCode = res
576    if isXFail:
577        ok = exitCode != 0
578        if ok:
579            status = Test.XFAIL
580        else:
581            status = Test.XPASS
582    else:
583        ok = exitCode == 0
584        if ok:
585            status = Test.PASS
586        else:
587            status = Test.FAIL
588
589    if ok:
590        return (status,'')
591
592    # Sh tests are not considered to fail just from stderr output.
593    failDueToStderr = False
594
595    return formatTestOutput(status, out, err, exitCode, failDueToStderr, script)
596