TestRunner.py revision 0f9d34c71d4f4ea83912c45f99ed286557ea189c 
1import os, signal, subprocess, sys
2import StringIO
3
4import ShUtil
5import Test
6import Util
7
8import platform
9import tempfile
10
11import re
12
13class InternalShellError(Exception):
14    def __init__(self, command, message):
15        self.command = command
16        self.message = message
17
18kIsWindows = platform.system() == 'Windows'
19
20# Don't use close_fds on Windows.
21kUseCloseFDs = not kIsWindows
22
23# Use temporary files to replace /dev/null on Windows.
24kAvoidDevNull = kIsWindows
25
26def executeCommand(command, cwd=None, env=None):
27    p = subprocess.Popen(command, cwd=cwd,
28                         stdin=subprocess.PIPE,
29                         stdout=subprocess.PIPE,
30                         stderr=subprocess.PIPE,
31                         env=env)
32    out,err = p.communicate()
33    exitCode = p.wait()
34
35    # Detect Ctrl-C in subprocess.
36    if exitCode == -signal.SIGINT:
37        raise KeyboardInterrupt
38
39    return out, err, exitCode
40
41def executeShCmd(cmd, cfg, cwd, results):
42    if isinstance(cmd, ShUtil.Seq):
43        if cmd.op == ';':
44            res = executeShCmd(cmd.lhs, cfg, cwd, results)
45            return executeShCmd(cmd.rhs, cfg, cwd, results)
46
47        if cmd.op == '&':
48            raise NotImplementedError,"unsupported test command: '&'"
49
50        if cmd.op == '||':
51            res = executeShCmd(cmd.lhs, cfg, cwd, results)
52            if res != 0:
53                res = executeShCmd(cmd.rhs, cfg, cwd, results)
54            return res
55        if cmd.op == '&&':
56            res = executeShCmd(cmd.lhs, cfg, cwd, results)
57            if res is None:
58                return res
59
60            if res == 0:
61                res = executeShCmd(cmd.rhs, cfg, cwd, results)
62            return res
63
64        raise ValueError,'Unknown shell command: %r' % cmd.op
65
66    assert isinstance(cmd, ShUtil.Pipeline)
67    procs = []
68    input = subprocess.PIPE
69    stderrTempFiles = []
70    opened_files = []
71    named_temp_files = []
72    # To avoid deadlock, we use a single stderr stream for piped
73    # output. This is null until we have seen some output using
74    # stderr.
75    for i,j in enumerate(cmd.commands):
76        # Apply the redirections, we use (N,) as a sentinal to indicate stdin,
77        # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
78        # from a file are represented with a list [file, mode, file-object]
79        # where file-object is initially None.
80        redirects = [(0,), (1,), (2,)]
81        for r in j.redirects:
82            if r[0] == ('>',2):
83                redirects[2] = [r[1], 'w', None]
84            elif r[0] == ('>>',2):
85                redirects[2] = [r[1], 'a', None]
86            elif r[0] == ('>&',2) and r[1] in '012':
87                redirects[2] = redirects[int(r[1])]
88            elif r[0] == ('>&',) or r[0] == ('&>',):
89                redirects[1] = redirects[2] = [r[1], 'w', None]
90            elif r[0] == ('>',):
91                redirects[1] = [r[1], 'w', None]
92            elif r[0] == ('>>',):
93                redirects[1] = [r[1], 'a', None]
94            elif r[0] == ('<',):
95                redirects[0] = [r[1], 'r', None]
96            else:
97                raise NotImplementedError,"Unsupported redirect: %r" % (r,)
98
99        # Map from the final redirections to something subprocess can handle.
100        final_redirects = []
101        for index,r in enumerate(redirects):
102            if r == (0,):
103                result = input
104            elif r == (1,):
105                if index == 0:
106                    raise NotImplementedError,"Unsupported redirect for stdin"
107                elif index == 1:
108                    result = subprocess.PIPE
109                else:
110                    result = subprocess.STDOUT
111            elif r == (2,):
112                if index != 2:
113                    raise NotImplementedError,"Unsupported redirect on stdout"
114                result = subprocess.PIPE
115            else:
116                if r[2] is None:
117                    if kAvoidDevNull and r[0] == '/dev/null':
118                        r[2] = tempfile.TemporaryFile(mode=r[1])
119                    else:
120                        r[2] = open(r[0], r[1])
121                    # Workaround a Win32 and/or subprocess bug when appending.
122                    #
123                    # FIXME: Actually, this is probably an instance of PR6753.
124                    if r[1] == 'a':
125                        r[2].seek(0, 2)
126                    opened_files.append(r[2])
127                result = r[2]
128            final_redirects.append(result)
129
130        stdin, stdout, stderr = final_redirects
131
132        # If stderr wants to come from stdout, but stdout isn't a pipe, then put
133        # stderr on a pipe and treat it as stdout.
134        if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
135            stderr = subprocess.PIPE
136            stderrIsStdout = True
137        else:
138            stderrIsStdout = False
139
140            # Don't allow stderr on a PIPE except for the last
141            # process, this could deadlock.
142            #
143            # FIXME: This is slow, but so is deadlock.
144            if stderr == subprocess.PIPE and j != cmd.commands[-1]:
145                stderr = tempfile.TemporaryFile(mode='w+b')
146                stderrTempFiles.append((i, stderr))
147
148        # Resolve the executable path ourselves.
149        args = list(j.args)
150        args[0] = Util.which(args[0], cfg.environment['PATH'])
151        if not args[0]:
152            raise InternalShellError(j, '%r: command not found' % j.args[0])
153
154        # Replace uses of /dev/null with temporary files.
155        if kAvoidDevNull:
156            for i,arg in enumerate(args):
157                if arg == "/dev/null":
158                    f = tempfile.NamedTemporaryFile(delete=False)
159                    f.close()
160                    named_temp_files.append(f.name)
161                    args[i] = f.name
162
163        procs.append(subprocess.Popen(args, cwd=cwd,
164                                      stdin = stdin,
165                                      stdout = stdout,
166                                      stderr = stderr,
167                                      env = cfg.environment,
168                                      close_fds = kUseCloseFDs))
169
170        # Immediately close stdin for any process taking stdin from us.
171        if stdin == subprocess.PIPE:
172            procs[-1].stdin.close()
173            procs[-1].stdin = None
174
175        # Update the current stdin source.
176        if stdout == subprocess.PIPE:
177            input = procs[-1].stdout
178        elif stderrIsStdout:
179            input = procs[-1].stderr
180        else:
181            input = subprocess.PIPE
182
183    # Explicitly close any redirected files. We need to do this now because we
184    # need to release any handles we may have on the temporary files (important
185    # on Win32, for example). Since we have already spawned the subprocess, our
186    # handles have already been transferred so we do not need them anymore.
187    for f in opened_files:
188        f.close()
189
190    # FIXME: There is probably still deadlock potential here. Yawn.
191    procData = [None] * len(procs)
192    procData[-1] = procs[-1].communicate()
193
194    for i in range(len(procs) - 1):
195        if procs[i].stdout is not None:
196            out = procs[i].stdout.read()
197        else:
198            out = ''
199        if procs[i].stderr is not None:
200            err = procs[i].stderr.read()
201        else:
202            err = ''
203        procData[i] = (out,err)
204
205    # Read stderr out of the temp files.
206    for i,f in stderrTempFiles:
207        f.seek(0, 0)
208        procData[i] = (procData[i][0], f.read())
209
210    exitCode = None
211    for i,(out,err) in enumerate(procData):
212        res = procs[i].wait()
213        # Detect Ctrl-C in subprocess.
214        if res == -signal.SIGINT:
215            raise KeyboardInterrupt
216
217        results.append((cmd.commands[i], out, err, res))
218        if cmd.pipe_err:
219            # Python treats the exit code as a signed char.
220            if res < 0:
221                exitCode = min(exitCode, res)
222            else:
223                exitCode = max(exitCode, res)
224        else:
225            exitCode = res
226
227    # Remove any named temporary files we created.
228    for f in named_temp_files:
229        try:
230            os.remove(f)
231        except OSError:
232            pass
233
234    if cmd.negate:
235        exitCode = not exitCode
236
237    return exitCode
238
239def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
240    ln = ' &&\n'.join(commands)
241    try:
242        cmd = ShUtil.ShParser(ln, litConfig.isWindows).parse()
243    except:
244        return (Test.FAIL, "shell parser error on: %r" % ln)
245
246    results = []
247    try:
248        exitCode = executeShCmd(cmd, test.config, cwd, results)
249    except InternalShellError,e:
250        out = ''
251        err = e.message
252        exitCode = 255
253
254    out = err = ''
255    for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
256        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
257        out += 'Command %d Result: %r\n' % (i, res)
258        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
259        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
260
261    return out, err, exitCode
262
263def executeTclScriptInternal(test, litConfig, tmpBase, commands, cwd):
264    import TclUtil
265    cmds = []
266    for ln in commands:
267        # Given the unfortunate way LLVM's test are written, the line gets
268        # backslash substitution done twice.
269        ln = TclUtil.TclLexer(ln).lex_unquoted(process_all = True)
270
271        try:
272            tokens = list(TclUtil.TclLexer(ln).lex())
273        except:
274            return (Test.FAIL, "Tcl lexer error on: %r" % ln)
275
276        # Validate there are no control tokens.
277        for t in tokens:
278            if not isinstance(t, str):
279                return (Test.FAIL,
280                        "Invalid test line: %r containing %r" % (ln, t))
281
282        try:
283            cmds.append(TclUtil.TclExecCommand(tokens).parse_pipeline())
284        except:
285            return (Test.FAIL, "Tcl 'exec' parse error on: %r" % ln)
286
287    if litConfig.useValgrind:
288        for pipeline in cmds:
289            if pipeline.commands:
290                # Only valgrind the first command in each pipeline, to avoid
291                # valgrinding things like grep, not, and FileCheck.
292                cmd = pipeline.commands[0]
293                cmd.args = litConfig.valgrindArgs + cmd.args
294
295    cmd = cmds[0]
296    for c in cmds[1:]:
297        cmd = ShUtil.Seq(cmd, '&&', c)
298
299    # FIXME: This is lame, we shouldn't need bash. See PR5240.
300    bashPath = litConfig.getBashPath()
301    if litConfig.useTclAsSh and bashPath:
302        script = tmpBase + '.script'
303
304        # Write script file
305        f = open(script,'w')
306        print >>f, 'set -o pipefail'
307        cmd.toShell(f, pipefail = True)
308        f.close()
309
310        if 0:
311            print >>sys.stdout, cmd
312            print >>sys.stdout, open(script).read()
313            print >>sys.stdout
314            return '', '', 0
315
316        command = [litConfig.getBashPath(), script]
317        out,err,exitCode = executeCommand(command, cwd=cwd,
318                                          env=test.config.environment)
319
320        return out,err,exitCode
321    else:
322        results = []
323        try:
324            exitCode = executeShCmd(cmd, test.config, cwd, results)
325        except InternalShellError,e:
326            results.append((e.command, '', e.message + '\n', 255))
327            exitCode = 255
328
329    out = err = ''
330
331    for i,(cmd, cmd_out, cmd_err, res) in enumerate(results):
332        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
333        out += 'Command %d Result: %r\n' % (i, res)
334        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
335        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
336
337    return out, err, exitCode
338
339def executeScript(test, litConfig, tmpBase, commands, cwd):
340    bashPath = litConfig.getBashPath();
341    isWin32CMDEXE = (litConfig.isWindows and not bashPath)
342    script = tmpBase + '.script'
343    if isWin32CMDEXE:
344        script += '.bat'
345
346    # Write script file
347    f = open(script,'w')
348    if isWin32CMDEXE:
349        f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
350    else:
351        f.write(' &&\n'.join(commands))
352    f.write('\n')
353    f.close()
354
355    if isWin32CMDEXE:
356        command = ['cmd','/c', script]
357    else:
358        if bashPath:
359            command = [bashPath, script]
360        else:
361            command = ['/bin/sh', script]
362        if litConfig.useValgrind:
363            # FIXME: Running valgrind on sh is overkill. We probably could just
364            # run on clang with no real loss.
365            command = litConfig.valgrindArgs + command
366
367    return executeCommand(command, cwd=cwd, env=test.config.environment)
368
369def isExpectedFail(xfails, xtargets, target_triple):
370    # Check if any xfail matches this target.
371    for item in xfails:
372        if item == '*' or item in target_triple:
373            break
374    else:
375        return False
376
377    # If so, see if it is expected to pass on this target.
378    #
379    # FIXME: Rename XTARGET to something that makes sense, like XPASS.
380    for item in xtargets:
381        if item == '*' or item in target_triple:
382            return False
383
384    return True
385
386def parseIntegratedTestScript(test, normalize_slashes=False):
387    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
388    script and extract the lines to 'RUN' as well as 'XFAIL' and 'XTARGET'
389    information. The RUN lines also will have variable substitution performed.
390    """
391
392    # Get the temporary location, this is always relative to the test suite
393    # root, not test source root.
394    #
395    # FIXME: This should not be here?
396    sourcepath = test.getSourcePath()
397    sourcedir = os.path.dirname(sourcepath)
398    execpath = test.getExecPath()
399    execdir,execbase = os.path.split(execpath)
400    tmpDir = os.path.join(execdir, 'Output')
401    tmpBase = os.path.join(tmpDir, execbase)
402    if test.index is not None:
403        tmpBase += '_%d' % test.index
404
405    # Normalize slashes, if requested.
406    if normalize_slashes:
407        sourcepath = sourcepath.replace('\\', '/')
408        sourcedir = sourcedir.replace('\\', '/')
409        tmpBase = tmpBase.replace('\\', '/')
410
411    # We use #_MARKER_# to hide %% while we do the other substitutions.
412    substitutions = [('%%', '#_MARKER_#')]
413    substitutions.extend(test.config.substitutions)
414    substitutions.extend([('%s', sourcepath),
415                          ('%S', sourcedir),
416                          ('%p', sourcedir),
417                          ('%t', tmpBase + '.tmp'),
418                          ('%T', tmpDir),
419                          # FIXME: Remove this once we kill DejaGNU.
420                          ('%abs_tmp', tmpBase + '.tmp'),
421                          ('#_MARKER_#', '%')])
422
423    # Collect the test lines from the script.
424    script = []
425    xfails = []
426    xtargets = []
427    requires = []
428    for ln in open(sourcepath):
429        if 'RUN:' in ln:
430            # Isolate the command to run.
431            index = ln.index('RUN:')
432            ln = ln[index+4:]
433
434            # Trim trailing whitespace.
435            ln = ln.rstrip()
436
437            # Collapse lines with trailing '\\'.
438            if script and script[-1][-1] == '\\':
439                script[-1] = script[-1][:-1] + ln
440            else:
441                script.append(ln)
442        elif 'XFAIL:' in ln:
443            items = ln[ln.index('XFAIL:') + 6:].split(',')
444            xfails.extend([s.strip() for s in items])
445        elif 'XTARGET:' in ln:
446            items = ln[ln.index('XTARGET:') + 8:].split(',')
447            xtargets.extend([s.strip() for s in items])
448        elif 'REQUIRES:' in ln:
449            items = ln[ln.index('REQUIRES:') + 9:].split(',')
450            requires.extend([s.strip() for s in items])
451        elif 'END.' in ln:
452            # Check for END. lines.
453            if ln[ln.index('END.'):].strip() == 'END.':
454                break
455
456    # Apply substitutions to the script.  Allow full regular
457    # expression syntax.  Replace each matching occurrence of regular
458    # expression pattern a with substitution b in line ln.
459    def processLine(ln):
460        # Apply substitutions
461        for a,b in substitutions:
462            if kIsWindows:
463                b = b.replace("\\","\\\\")
464            ln = re.sub(a, b, ln)
465
466        # Strip the trailing newline and any extra whitespace.
467        return ln.strip()
468    script = map(processLine, script)
469
470    # Verify the script contains a run line.
471    if not script:
472        return (Test.UNRESOLVED, "Test has no run line!")
473
474    # Check for unterminated run lines.
475    if script[-1][-1] == '\\':
476        return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')")
477
478    # Check that we have the required features:
479    missing_required_features = [f for f in requires
480                                 if f not in test.config.available_features]
481    if missing_required_features:
482        msg = ', '.join(missing_required_features)
483        return (Test.UNSUPPORTED,
484                "Test requires the following features: %s" % msg)
485
486    isXFail = isExpectedFail(xfails, xtargets, test.suite.config.target_triple)
487    return script,isXFail,tmpBase,execdir
488
489def formatTestOutput(status, out, err, exitCode, failDueToStderr, script):
490    output = StringIO.StringIO()
491    print >>output, "Script:"
492    print >>output, "--"
493    print >>output, '\n'.join(script)
494    print >>output, "--"
495    print >>output, "Exit Code: %r" % exitCode,
496    if failDueToStderr:
497        print >>output, "(but there was output on stderr)"
498    else:
499        print >>output
500    if out:
501        print >>output, "Command Output (stdout):"
502        print >>output, "--"
503        output.write(out)
504        print >>output, "--"
505    if err:
506        print >>output, "Command Output (stderr):"
507        print >>output, "--"
508        output.write(err)
509        print >>output, "--"
510    return (status, output.getvalue())
511
512def executeTclTest(test, litConfig):
513    if test.config.unsupported:
514        return (Test.UNSUPPORTED, 'Test is unsupported')
515
516    # Parse the test script, normalizing slashes in substitutions on Windows
517    # (since otherwise Tcl style lexing will treat them as escapes).
518    res = parseIntegratedTestScript(test, normalize_slashes=kIsWindows)
519    if len(res) == 2:
520        return res
521
522    script, isXFail, tmpBase, execdir = res
523
524    if litConfig.noExecute:
525        return (Test.PASS, '')
526
527    # Create the output directory if it does not already exist.
528    Util.mkdir_p(os.path.dirname(tmpBase))
529
530    res = executeTclScriptInternal(test, litConfig, tmpBase, script, execdir)
531    if len(res) == 2:
532        return res
533
534    # Test for failure. In addition to the exit code, Tcl commands are
535    # considered to fail if there is any standard error output.
536    out,err,exitCode = res
537    if isXFail:
538        ok = exitCode != 0 or err
539        if ok:
540            status = Test.XFAIL
541        else:
542            status = Test.XPASS
543    else:
544        ok = exitCode == 0 and not err
545        if ok:
546            status = Test.PASS
547        else:
548            status = Test.FAIL
549
550    if ok:
551        return (status,'')
552
553    # Set a flag for formatTestOutput so it can explain why the test was
554    # considered to have failed, despite having an exit code of 0.
555    failDueToStderr = exitCode == 0 and err
556
557    return formatTestOutput(status, out, err, exitCode, failDueToStderr, script)
558
559def executeShTest(test, litConfig, useExternalSh):
560    if test.config.unsupported:
561        return (Test.UNSUPPORTED, 'Test is unsupported')
562
563    res = parseIntegratedTestScript(test, useExternalSh)
564    if len(res) == 2:
565        return res
566
567    script, isXFail, tmpBase, execdir = res
568
569    if litConfig.noExecute:
570        return (Test.PASS, '')
571
572    # Create the output directory if it does not already exist.
573    Util.mkdir_p(os.path.dirname(tmpBase))
574
575    if useExternalSh:
576        res = executeScript(test, litConfig, tmpBase, script, execdir)
577    else:
578        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
579    if len(res) == 2:
580        return res
581
582    out,err,exitCode = res
583    if isXFail:
584        ok = exitCode != 0
585        if ok:
586            status = Test.XFAIL
587        else:
588            status = Test.XPASS
589    else:
590        ok = exitCode == 0
591        if ok:
592            status = Test.PASS
593        else:
594            status = Test.FAIL
595
596    if ok:
597        return (status,'')
598
599    # Sh tests are not considered to fail just from stderr output.
600    failDueToStderr = False
601
602    return formatTestOutput(status, out, err, exitCode, failDueToStderr, script)
603