TestRunner.py revision fc1a1870b5b4e1ef5a555426acd762f19387a57b 
1import os, signal, subprocess, sys
2import StringIO
3
4import ShUtil
5import Test
6import Util
7
8import platform
9import tempfile
10
11import re
12
13class InternalShellError(Exception):
14    def __init__(self, command, message):
15        self.command = command
16        self.message = message
17
18kIsWindows = platform.system() == 'Windows'
19
20# Don't use close_fds on Windows.
21kUseCloseFDs = not kIsWindows
22
23# Use temporary files to replace /dev/null on Windows.
24kAvoidDevNull = kIsWindows
25
26def RemoveForce(f):
27    try:
28        os.remove(f)
29    except OSError:
30        pass
31
32def WinRename(f_o, f_n):
33    import time
34    retry_cnt = 256
35    while (True):
36        try:
37            os.rename(f_o, f_n)
38            break
39        except WindowsError, (winerror, strerror):
40            retry_cnt = retry_cnt - 1
41            if retry_cnt <= 0:
42                raise
43            elif winerror == 32: # ERROR_SHARING_VIOLATION
44                time.sleep(0.01)
45            else:
46                raise
47
48def WinWaitReleased(f):
49    import random
50    t = "%s%06d" % (f, random.randint(0, 999999))
51    RemoveForce(t)
52    try:
53        WinRename(f, t) # rename
54        WinRename(t, f) # restore
55    except WindowsError, (winerror, strerror):
56        if winerror == 3: # ERROR_PATH_NOT_FOUND
57            pass
58        else:
59            raise
60
61def executeCommand(command, cwd=None, env=None):
62    p = subprocess.Popen(command, cwd=cwd,
63                         stdin=subprocess.PIPE,
64                         stdout=subprocess.PIPE,
65                         stderr=subprocess.PIPE,
66                         env=env)
67    out,err = p.communicate()
68    exitCode = p.wait()
69
70    # Detect Ctrl-C in subprocess.
71    if exitCode == -signal.SIGINT:
72        raise KeyboardInterrupt
73
74    return out, err, exitCode
75
76def executeShCmd(cmd, cfg, cwd, results):
77    if isinstance(cmd, ShUtil.Seq):
78        if cmd.op == ';':
79            res = executeShCmd(cmd.lhs, cfg, cwd, results)
80            return executeShCmd(cmd.rhs, cfg, cwd, results)
81
82        if cmd.op == '&':
83            raise NotImplementedError,"unsupported test command: '&'"
84
85        if cmd.op == '||':
86            res = executeShCmd(cmd.lhs, cfg, cwd, results)
87            if res != 0:
88                res = executeShCmd(cmd.rhs, cfg, cwd, results)
89            return res
90        if cmd.op == '&&':
91            res = executeShCmd(cmd.lhs, cfg, cwd, results)
92            if res is None:
93                return res
94
95            if res == 0:
96                res = executeShCmd(cmd.rhs, cfg, cwd, results)
97            return res
98
99        raise ValueError,'Unknown shell command: %r' % cmd.op
100
101    assert isinstance(cmd, ShUtil.Pipeline)
102    procs = []
103    input = subprocess.PIPE
104    stderrTempFiles = []
105    opened_files = []
106    written_files = []
107    named_temp_files = []
108    # To avoid deadlock, we use a single stderr stream for piped
109    # output. This is null until we have seen some output using
110    # stderr.
111    for i,j in enumerate(cmd.commands):
112        # Apply the redirections, we use (N,) as a sentinal to indicate stdin,
113        # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
114        # from a file are represented with a list [file, mode, file-object]
115        # where file-object is initially None.
116        redirects = [(0,), (1,), (2,)]
117        for r in j.redirects:
118            if r[0] == ('>',2):
119                redirects[2] = [r[1], 'w', None]
120            elif r[0] == ('>>',2):
121                redirects[2] = [r[1], 'a', None]
122            elif r[0] == ('>&',2) and r[1] in '012':
123                redirects[2] = redirects[int(r[1])]
124            elif r[0] == ('>&',) or r[0] == ('&>',):
125                redirects[1] = redirects[2] = [r[1], 'w', None]
126            elif r[0] == ('>',):
127                redirects[1] = [r[1], 'w', None]
128            elif r[0] == ('>>',):
129                redirects[1] = [r[1], 'a', None]
130            elif r[0] == ('<',):
131                redirects[0] = [r[1], 'r', None]
132            else:
133                raise NotImplementedError,"Unsupported redirect: %r" % (r,)
134
135        # Map from the final redirections to something subprocess can handle.
136        final_redirects = []
137        for index,r in enumerate(redirects):
138            if r == (0,):
139                result = input
140            elif r == (1,):
141                if index == 0:
142                    raise NotImplementedError,"Unsupported redirect for stdin"
143                elif index == 1:
144                    result = subprocess.PIPE
145                else:
146                    result = subprocess.STDOUT
147            elif r == (2,):
148                if index != 2:
149                    raise NotImplementedError,"Unsupported redirect on stdout"
150                result = subprocess.PIPE
151            else:
152                if r[2] is None:
153                    if kAvoidDevNull and r[0] == '/dev/null':
154                        r[2] = tempfile.TemporaryFile(mode=r[1])
155                    else:
156                        r[2] = open(r[0], r[1])
157                    # Workaround a Win32 and/or subprocess bug when appending.
158                    #
159                    # FIXME: Actually, this is probably an instance of PR6753.
160                    if r[1] == 'a':
161                        r[2].seek(0, 2)
162                    opened_files.append(r[2])
163                    if r[1] in 'aw':
164                        written_files.append(r[0])
165                result = r[2]
166            final_redirects.append(result)
167
168        stdin, stdout, stderr = final_redirects
169
170        # If stderr wants to come from stdout, but stdout isn't a pipe, then put
171        # stderr on a pipe and treat it as stdout.
172        if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
173            stderr = subprocess.PIPE
174            stderrIsStdout = True
175        else:
176            stderrIsStdout = False
177
178            # Don't allow stderr on a PIPE except for the last
179            # process, this could deadlock.
180            #
181            # FIXME: This is slow, but so is deadlock.
182            if stderr == subprocess.PIPE and j != cmd.commands[-1]:
183                stderr = tempfile.TemporaryFile(mode='w+b')
184                stderrTempFiles.append((i, stderr))
185
186        # Resolve the executable path ourselves.
187        args = list(j.args)
188        args[0] = Util.which(args[0], cfg.environment['PATH'])
189        if not args[0]:
190            raise InternalShellError(j, '%r: command not found' % j.args[0])
191
192        # Replace uses of /dev/null with temporary files.
193        if kAvoidDevNull:
194            for i,arg in enumerate(args):
195                if arg == "/dev/null":
196                    f = tempfile.NamedTemporaryFile(delete=False)
197                    f.close()
198                    named_temp_files.append(f.name)
199                    args[i] = f.name
200
201        procs.append(subprocess.Popen(args, cwd=cwd,
202                                      stdin = stdin,
203                                      stdout = stdout,
204                                      stderr = stderr,
205                                      env = cfg.environment,
206                                      close_fds = kUseCloseFDs))
207
208        # Immediately close stdin for any process taking stdin from us.
209        if stdin == subprocess.PIPE:
210            procs[-1].stdin.close()
211            procs[-1].stdin = None
212
213        # Update the current stdin source.
214        if stdout == subprocess.PIPE:
215            input = procs[-1].stdout
216        elif stderrIsStdout:
217            input = procs[-1].stderr
218        else:
219            input = subprocess.PIPE
220
221    # Explicitly close any redirected files. We need to do this now because we
222    # need to release any handles we may have on the temporary files (important
223    # on Win32, for example). Since we have already spawned the subprocess, our
224    # handles have already been transferred so we do not need them anymore.
225    for f in opened_files:
226        f.close()
227
228    # FIXME: There is probably still deadlock potential here. Yawn.
229    procData = [None] * len(procs)
230    procData[-1] = procs[-1].communicate()
231
232    for i in range(len(procs) - 1):
233        if procs[i].stdout is not None:
234            out = procs[i].stdout.read()
235        else:
236            out = ''
237        if procs[i].stderr is not None:
238            err = procs[i].stderr.read()
239        else:
240            err = ''
241        procData[i] = (out,err)
242
243    # Read stderr out of the temp files.
244    for i,f in stderrTempFiles:
245        f.seek(0, 0)
246        procData[i] = (procData[i][0], f.read())
247
248    exitCode = None
249    for i,(out,err) in enumerate(procData):
250        res = procs[i].wait()
251        # Detect Ctrl-C in subprocess.
252        if res == -signal.SIGINT:
253            raise KeyboardInterrupt
254
255        results.append((cmd.commands[i], out, err, res))
256        if cmd.pipe_err:
257            # Python treats the exit code as a signed char.
258            if res < 0:
259                exitCode = min(exitCode, res)
260            else:
261                exitCode = max(exitCode, res)
262        else:
263            exitCode = res
264
265    # Make sure written_files is released by other (child) processes.
266    if (kIsWindows):
267        for f in written_files:
268            WinWaitReleased(f)
269
270    # Remove any named temporary files we created.
271    for f in named_temp_files:
272        try:
273            os.remove(f)
274        except OSError:
275            pass
276
277    if cmd.negate:
278        exitCode = not exitCode
279
280    return exitCode
281
282def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
283    ln = ' &&\n'.join(commands)
284    try:
285        cmd = ShUtil.ShParser(ln, litConfig.isWindows).parse()
286    except:
287        return (Test.FAIL, "shell parser error on: %r" % ln)
288
289    results = []
290    try:
291        exitCode = executeShCmd(cmd, test.config, cwd, results)
292    except InternalShellError,e:
293        out = ''
294        err = e.message
295        exitCode = 255
296
297    out = err = ''
298    for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
299        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
300        out += 'Command %d Result: %r\n' % (i, res)
301        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
302        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
303
304    return out, err, exitCode
305
306def executeTclScriptInternal(test, litConfig, tmpBase, commands, cwd):
307    import TclUtil
308    cmds = []
309    for ln in commands:
310        # Given the unfortunate way LLVM's test are written, the line gets
311        # backslash substitution done twice.
312        ln = TclUtil.TclLexer(ln).lex_unquoted(process_all = True)
313
314        try:
315            tokens = list(TclUtil.TclLexer(ln).lex())
316        except:
317            return (Test.FAIL, "Tcl lexer error on: %r" % ln)
318
319        # Validate there are no control tokens.
320        for t in tokens:
321            if not isinstance(t, str):
322                return (Test.FAIL,
323                        "Invalid test line: %r containing %r" % (ln, t))
324
325        try:
326            cmds.append(TclUtil.TclExecCommand(tokens).parse_pipeline())
327        except:
328            return (Test.FAIL, "Tcl 'exec' parse error on: %r" % ln)
329
330    if litConfig.useValgrind:
331        for pipeline in cmds:
332            if pipeline.commands:
333                # Only valgrind the first command in each pipeline, to avoid
334                # valgrinding things like grep, not, and FileCheck.
335                cmd = pipeline.commands[0]
336                cmd.args = litConfig.valgrindArgs + cmd.args
337
338    cmd = cmds[0]
339    for c in cmds[1:]:
340        cmd = ShUtil.Seq(cmd, '&&', c)
341
342    # FIXME: This is lame, we shouldn't need bash. See PR5240.
343    bashPath = litConfig.getBashPath()
344    if litConfig.useTclAsSh and bashPath:
345        script = tmpBase + '.script'
346
347        # Write script file
348        f = open(script,'w')
349        print >>f, 'set -o pipefail'
350        cmd.toShell(f, pipefail = True)
351        f.close()
352
353        if 0:
354            print >>sys.stdout, cmd
355            print >>sys.stdout, open(script).read()
356            print >>sys.stdout
357            return '', '', 0
358
359        command = [litConfig.getBashPath(), script]
360        out,err,exitCode = executeCommand(command, cwd=cwd,
361                                          env=test.config.environment)
362
363        return out,err,exitCode
364    else:
365        results = []
366        try:
367            exitCode = executeShCmd(cmd, test.config, cwd, results)
368        except InternalShellError,e:
369            results.append((e.command, '', e.message + '\n', 255))
370            exitCode = 255
371
372    out = err = ''
373
374    for i,(cmd, cmd_out, cmd_err, res) in enumerate(results):
375        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
376        out += 'Command %d Result: %r\n' % (i, res)
377        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
378        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
379
380    return out, err, exitCode
381
382def executeScript(test, litConfig, tmpBase, commands, cwd):
383    bashPath = litConfig.getBashPath();
384    isWin32CMDEXE = (litConfig.isWindows and not bashPath)
385    script = tmpBase + '.script'
386    if isWin32CMDEXE:
387        script += '.bat'
388
389    # Write script file
390    f = open(script,'w')
391    if isWin32CMDEXE:
392        f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
393    else:
394        f.write(' &&\n'.join(commands))
395    f.write('\n')
396    f.close()
397
398    if isWin32CMDEXE:
399        command = ['cmd','/c', script]
400    else:
401        if bashPath:
402            command = [bashPath, script]
403        else:
404            command = ['/bin/sh', script]
405        if litConfig.useValgrind:
406            # FIXME: Running valgrind on sh is overkill. We probably could just
407            # run on clang with no real loss.
408            command = litConfig.valgrindArgs + command
409
410    return executeCommand(command, cwd=cwd, env=test.config.environment)
411
412def isExpectedFail(xfails, xtargets, target_triple):
413    # Check if any xfail matches this target.
414    for item in xfails:
415        if item == '*' or item in target_triple:
416            break
417    else:
418        return False
419
420    # If so, see if it is expected to pass on this target.
421    #
422    # FIXME: Rename XTARGET to something that makes sense, like XPASS.
423    for item in xtargets:
424        if item == '*' or item in target_triple:
425            return False
426
427    return True
428
429def parseIntegratedTestScript(test, normalize_slashes=False):
430    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
431    script and extract the lines to 'RUN' as well as 'XFAIL' and 'XTARGET'
432    information. The RUN lines also will have variable substitution performed.
433    """
434
435    # Get the temporary location, this is always relative to the test suite
436    # root, not test source root.
437    #
438    # FIXME: This should not be here?
439    sourcepath = test.getSourcePath()
440    sourcedir = os.path.dirname(sourcepath)
441    execpath = test.getExecPath()
442    execdir,execbase = os.path.split(execpath)
443    tmpDir = os.path.join(execdir, 'Output')
444    tmpBase = os.path.join(tmpDir, execbase)
445    if test.index is not None:
446        tmpBase += '_%d' % test.index
447
448    # Normalize slashes, if requested.
449    if normalize_slashes:
450        sourcepath = sourcepath.replace('\\', '/')
451        sourcedir = sourcedir.replace('\\', '/')
452        tmpDir = tmpDir.replace('\\', '/')
453        tmpBase = tmpBase.replace('\\', '/')
454
455    # We use #_MARKER_# to hide %% while we do the other substitutions.
456    substitutions = [('%%', '#_MARKER_#')]
457    substitutions.extend(test.config.substitutions)
458    substitutions.extend([('%s', sourcepath),
459                          ('%S', sourcedir),
460                          ('%p', sourcedir),
461                          ('%t', tmpBase + '.tmp'),
462                          ('%T', tmpDir),
463                          # FIXME: Remove this once we kill DejaGNU.
464                          ('%abs_tmp', tmpBase + '.tmp'),
465                          ('#_MARKER_#', '%')])
466
467    # Collect the test lines from the script.
468    script = []
469    xfails = []
470    xtargets = []
471    requires = []
472    for ln in open(sourcepath):
473        if 'RUN:' in ln:
474            # Isolate the command to run.
475            index = ln.index('RUN:')
476            ln = ln[index+4:]
477
478            # Trim trailing whitespace.
479            ln = ln.rstrip()
480
481            # Collapse lines with trailing '\\'.
482            if script and script[-1][-1] == '\\':
483                script[-1] = script[-1][:-1] + ln
484            else:
485                script.append(ln)
486        elif 'XFAIL:' in ln:
487            items = ln[ln.index('XFAIL:') + 6:].split(',')
488            xfails.extend([s.strip() for s in items])
489        elif 'XTARGET:' in ln:
490            items = ln[ln.index('XTARGET:') + 8:].split(',')
491            xtargets.extend([s.strip() for s in items])
492        elif 'REQUIRES:' in ln:
493            items = ln[ln.index('REQUIRES:') + 9:].split(',')
494            requires.extend([s.strip() for s in items])
495        elif 'END.' in ln:
496            # Check for END. lines.
497            if ln[ln.index('END.'):].strip() == 'END.':
498                break
499
500    # Apply substitutions to the script.  Allow full regular
501    # expression syntax.  Replace each matching occurrence of regular
502    # expression pattern a with substitution b in line ln.
503    def processLine(ln):
504        # Apply substitutions
505        for a,b in substitutions:
506            if kIsWindows:
507                b = b.replace("\\","\\\\")
508            ln = re.sub(a, b, ln)
509
510        # Strip the trailing newline and any extra whitespace.
511        return ln.strip()
512    script = map(processLine, script)
513
514    # Verify the script contains a run line.
515    if not script:
516        return (Test.UNRESOLVED, "Test has no run line!")
517
518    # Check for unterminated run lines.
519    if script[-1][-1] == '\\':
520        return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')")
521
522    # Check that we have the required features:
523    missing_required_features = [f for f in requires
524                                 if f not in test.config.available_features]
525    if missing_required_features:
526        msg = ', '.join(missing_required_features)
527        return (Test.UNSUPPORTED,
528                "Test requires the following features: %s" % msg)
529
530    isXFail = isExpectedFail(xfails, xtargets, test.suite.config.target_triple)
531    return script,isXFail,tmpBase,execdir
532
533def formatTestOutput(status, out, err, exitCode, failDueToStderr, script):
534    output = StringIO.StringIO()
535    print >>output, "Script:"
536    print >>output, "--"
537    print >>output, '\n'.join(script)
538    print >>output, "--"
539    print >>output, "Exit Code: %r" % exitCode,
540    if failDueToStderr:
541        print >>output, "(but there was output on stderr)"
542    else:
543        print >>output
544    if out:
545        print >>output, "Command Output (stdout):"
546        print >>output, "--"
547        output.write(out)
548        print >>output, "--"
549    if err:
550        print >>output, "Command Output (stderr):"
551        print >>output, "--"
552        output.write(err)
553        print >>output, "--"
554    return (status, output.getvalue())
555
556def executeTclTest(test, litConfig):
557    if test.config.unsupported:
558        return (Test.UNSUPPORTED, 'Test is unsupported')
559
560    # Parse the test script, normalizing slashes in substitutions on Windows
561    # (since otherwise Tcl style lexing will treat them as escapes).
562    res = parseIntegratedTestScript(test, normalize_slashes=kIsWindows)
563    if len(res) == 2:
564        return res
565
566    script, isXFail, tmpBase, execdir = res
567
568    if litConfig.noExecute:
569        return (Test.PASS, '')
570
571    # Create the output directory if it does not already exist.
572    Util.mkdir_p(os.path.dirname(tmpBase))
573
574    res = executeTclScriptInternal(test, litConfig, tmpBase, script, execdir)
575    if len(res) == 2:
576        return res
577
578    # Test for failure. In addition to the exit code, Tcl commands are
579    # considered to fail if there is any standard error output.
580    out,err,exitCode = res
581    if isXFail:
582        ok = exitCode != 0 or err and not litConfig.ignoreStdErr
583        if ok:
584            status = Test.XFAIL
585        else:
586            status = Test.XPASS
587    else:
588        ok = exitCode == 0 and (not err or litConfig.ignoreStdErr)
589        if ok:
590            status = Test.PASS
591        else:
592            status = Test.FAIL
593
594    if ok:
595        return (status,'')
596
597    # Set a flag for formatTestOutput so it can explain why the test was
598    # considered to have failed, despite having an exit code of 0.
599    failDueToStderr = exitCode == 0 and err and not litConfig.ignoreStdErr
600
601    return formatTestOutput(status, out, err, exitCode, failDueToStderr, script)
602
603def executeShTest(test, litConfig, useExternalSh):
604    if test.config.unsupported:
605        return (Test.UNSUPPORTED, 'Test is unsupported')
606
607    res = parseIntegratedTestScript(test, useExternalSh)
608    if len(res) == 2:
609        return res
610
611    script, isXFail, tmpBase, execdir = res
612
613    if litConfig.noExecute:
614        return (Test.PASS, '')
615
616    # Create the output directory if it does not already exist.
617    Util.mkdir_p(os.path.dirname(tmpBase))
618
619    if useExternalSh:
620        res = executeScript(test, litConfig, tmpBase, script, execdir)
621    else:
622        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
623    if len(res) == 2:
624        return res
625
626    out,err,exitCode = res
627    if isXFail:
628        ok = exitCode != 0
629        if ok:
630            status = Test.XFAIL
631        else:
632            status = Test.XPASS
633    else:
634        ok = exitCode == 0
635        if ok:
636            status = Test.PASS
637        else:
638            status = Test.FAIL
639
640    if ok:
641        return (status,'')
642
643    # Sh tests are not considered to fail just from stderr output.
644    failDueToStderr = False
645
646    return formatTestOutput(status, out, err, exitCode, failDueToStderr, script)
647