TestRunner.py revision 213a789a711724c864defdc2c3e1208d6c5b7fd4 
1import os, signal, subprocess, sys
2import StringIO
3
4import ShUtil
5import Test
6import Util
7
8import platform
9import tempfile
10
11import re
12
13class InternalShellError(Exception):
14    def __init__(self, command, message):
15        self.command = command
16        self.message = message
17
18kIsWindows = platform.system() == 'Windows'
19
20# Don't use close_fds on Windows.
21kUseCloseFDs = not kIsWindows
22
23# Use temporary files to replace /dev/null on Windows.
24kAvoidDevNull = kIsWindows
25
26def executeCommand(command, cwd=None, env=None):
27    # Close extra file handles on UNIX (on Windows this cannot be done while
28    # also redirecting input).
29    close_fds = not kIsWindows
30
31    p = subprocess.Popen(command, cwd=cwd,
32                         stdin=subprocess.PIPE,
33                         stdout=subprocess.PIPE,
34                         stderr=subprocess.PIPE,
35                         env=env, close_fds=close_fds)
36    out,err = p.communicate()
37    exitCode = p.wait()
38
39    # Detect Ctrl-C in subprocess.
40    if exitCode == -signal.SIGINT:
41        raise KeyboardInterrupt
42
43    return out, err, exitCode
44
45def executeShCmd(cmd, cfg, cwd, results):
46    if isinstance(cmd, ShUtil.Seq):
47        if cmd.op == ';':
48            res = executeShCmd(cmd.lhs, cfg, cwd, results)
49            return executeShCmd(cmd.rhs, cfg, cwd, results)
50
51        if cmd.op == '&':
52            raise NotImplementedError,"unsupported test command: '&'"
53
54        if cmd.op == '||':
55            res = executeShCmd(cmd.lhs, cfg, cwd, results)
56            if res != 0:
57                res = executeShCmd(cmd.rhs, cfg, cwd, results)
58            return res
59        if cmd.op == '&&':
60            res = executeShCmd(cmd.lhs, cfg, cwd, results)
61            if res is None:
62                return res
63
64            if res == 0:
65                res = executeShCmd(cmd.rhs, cfg, cwd, results)
66            return res
67
68        raise ValueError,'Unknown shell command: %r' % cmd.op
69
70    assert isinstance(cmd, ShUtil.Pipeline)
71    procs = []
72    input = subprocess.PIPE
73    stderrTempFiles = []
74    opened_files = []
75    named_temp_files = []
76    # To avoid deadlock, we use a single stderr stream for piped
77    # output. This is null until we have seen some output using
78    # stderr.
79    for i,j in enumerate(cmd.commands):
80        # Apply the redirections, we use (N,) as a sentinel to indicate stdin,
81        # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
82        # from a file are represented with a list [file, mode, file-object]
83        # where file-object is initially None.
84        redirects = [(0,), (1,), (2,)]
85        for r in j.redirects:
86            if r[0] == ('>',2):
87                redirects[2] = [r[1], 'w', None]
88            elif r[0] == ('>>',2):
89                redirects[2] = [r[1], 'a', None]
90            elif r[0] == ('>&',2) and r[1] in '012':
91                redirects[2] = redirects[int(r[1])]
92            elif r[0] == ('>&',) or r[0] == ('&>',):
93                redirects[1] = redirects[2] = [r[1], 'w', None]
94            elif r[0] == ('>',):
95                redirects[1] = [r[1], 'w', None]
96            elif r[0] == ('>>',):
97                redirects[1] = [r[1], 'a', None]
98            elif r[0] == ('<',):
99                redirects[0] = [r[1], 'r', None]
100            else:
101                raise NotImplementedError,"Unsupported redirect: %r" % (r,)
102
103        # Map from the final redirections to something subprocess can handle.
104        final_redirects = []
105        for index,r in enumerate(redirects):
106            if r == (0,):
107                result = input
108            elif r == (1,):
109                if index == 0:
110                    raise NotImplementedError,"Unsupported redirect for stdin"
111                elif index == 1:
112                    result = subprocess.PIPE
113                else:
114                    result = subprocess.STDOUT
115            elif r == (2,):
116                if index != 2:
117                    raise NotImplementedError,"Unsupported redirect on stdout"
118                result = subprocess.PIPE
119            else:
120                if r[2] is None:
121                    if kAvoidDevNull and r[0] == '/dev/null':
122                        r[2] = tempfile.TemporaryFile(mode=r[1])
123                    else:
124                        r[2] = open(r[0], r[1])
125                    # Workaround a Win32 and/or subprocess bug when appending.
126                    #
127                    # FIXME: Actually, this is probably an instance of PR6753.
128                    if r[1] == 'a':
129                        r[2].seek(0, 2)
130                    opened_files.append(r[2])
131                result = r[2]
132            final_redirects.append(result)
133
134        stdin, stdout, stderr = final_redirects
135
136        # If stderr wants to come from stdout, but stdout isn't a pipe, then put
137        # stderr on a pipe and treat it as stdout.
138        if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
139            stderr = subprocess.PIPE
140            stderrIsStdout = True
141        else:
142            stderrIsStdout = False
143
144            # Don't allow stderr on a PIPE except for the last
145            # process, this could deadlock.
146            #
147            # FIXME: This is slow, but so is deadlock.
148            if stderr == subprocess.PIPE and j != cmd.commands[-1]:
149                stderr = tempfile.TemporaryFile(mode='w+b')
150                stderrTempFiles.append((i, stderr))
151
152        # Resolve the executable path ourselves.
153        args = list(j.args)
154        args[0] = Util.which(args[0], cfg.environment['PATH'])
155        if not args[0]:
156            raise InternalShellError(j, '%r: command not found' % j.args[0])
157
158        # Replace uses of /dev/null with temporary files.
159        if kAvoidDevNull:
160            for i,arg in enumerate(args):
161                if arg == "/dev/null":
162                    f = tempfile.NamedTemporaryFile(delete=False)
163                    f.close()
164                    named_temp_files.append(f.name)
165                    args[i] = f.name
166
167        procs.append(subprocess.Popen(args, cwd=cwd,
168                                      stdin = stdin,
169                                      stdout = stdout,
170                                      stderr = stderr,
171                                      env = cfg.environment,
172                                      close_fds = kUseCloseFDs))
173
174        # Immediately close stdin for any process taking stdin from us.
175        if stdin == subprocess.PIPE:
176            procs[-1].stdin.close()
177            procs[-1].stdin = None
178
179        # Update the current stdin source.
180        if stdout == subprocess.PIPE:
181            input = procs[-1].stdout
182        elif stderrIsStdout:
183            input = procs[-1].stderr
184        else:
185            input = subprocess.PIPE
186
187    # Explicitly close any redirected files. We need to do this now because we
188    # need to release any handles we may have on the temporary files (important
189    # on Win32, for example). Since we have already spawned the subprocess, our
190    # handles have already been transferred so we do not need them anymore.
191    for f in opened_files:
192        f.close()
193
194    # FIXME: There is probably still deadlock potential here. Yawn.
195    procData = [None] * len(procs)
196    procData[-1] = procs[-1].communicate()
197
198    for i in range(len(procs) - 1):
199        if procs[i].stdout is not None:
200            out = procs[i].stdout.read()
201        else:
202            out = ''
203        if procs[i].stderr is not None:
204            err = procs[i].stderr.read()
205        else:
206            err = ''
207        procData[i] = (out,err)
208
209    # Read stderr out of the temp files.
210    for i,f in stderrTempFiles:
211        f.seek(0, 0)
212        procData[i] = (procData[i][0], f.read())
213
214    exitCode = None
215    for i,(out,err) in enumerate(procData):
216        res = procs[i].wait()
217        # Detect Ctrl-C in subprocess.
218        if res == -signal.SIGINT:
219            raise KeyboardInterrupt
220
221        results.append((cmd.commands[i], out, err, res))
222        if cmd.pipe_err:
223            # Python treats the exit code as a signed char.
224            if res < 0:
225                exitCode = min(exitCode, res)
226            else:
227                exitCode = max(exitCode, res)
228        else:
229            exitCode = res
230
231    # Remove any named temporary files we created.
232    for f in named_temp_files:
233        try:
234            os.remove(f)
235        except OSError:
236            pass
237
238    if cmd.negate:
239        exitCode = not exitCode
240
241    return exitCode
242
243def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
244    cmds = []
245    for ln in commands:
246        try:
247            cmds.append(ShUtil.ShParser(ln, litConfig.isWindows).parse())
248        except:
249            return (Test.FAIL, "shell parser error on: %r" % ln)
250
251    cmd = cmds[0]
252    for c in cmds[1:]:
253        cmd = ShUtil.Seq(cmd, '&&', c)
254
255    results = []
256    try:
257        exitCode = executeShCmd(cmd, test.config, cwd, results)
258    except InternalShellError,e:
259        out = ''
260        err = e.message
261        exitCode = 255
262
263    out = err = ''
264    for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
265        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
266        out += 'Command %d Result: %r\n' % (i, res)
267        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
268        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
269
270    return out, err, exitCode
271
272def executeScript(test, litConfig, tmpBase, commands, cwd):
273    bashPath = litConfig.getBashPath();
274    isWin32CMDEXE = (litConfig.isWindows and not bashPath)
275    script = tmpBase + '.script'
276    if isWin32CMDEXE:
277        script += '.bat'
278
279    # Write script file
280    f = open(script,'w')
281    if isWin32CMDEXE:
282        f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
283    else:
284        f.write('{ ' + '; } &&\n{ '.join(commands) + '; }')
285    f.write('\n')
286    f.close()
287
288    if isWin32CMDEXE:
289        command = ['cmd','/c', script]
290    else:
291        if bashPath:
292            command = [bashPath, script]
293        else:
294            command = ['/bin/sh', script]
295        if litConfig.useValgrind:
296            # FIXME: Running valgrind on sh is overkill. We probably could just
297            # run on clang with no real loss.
298            command = litConfig.valgrindArgs + command
299
300    return executeCommand(command, cwd=cwd, env=test.config.environment)
301
302def isExpectedFail(test, xfails):
303    # Check if any of the xfails match an available feature or the target.
304    for item in xfails:
305        # If this is the wildcard, it always fails.
306        if item == '*':
307            return True
308
309        # If this is an exact match for one of the features, it fails.
310        if item in test.config.available_features:
311            return True
312
313        # If this is a part of the target triple, it fails.
314        if item in test.suite.config.target_triple:
315            return True
316
317    return False
318
319def parseIntegratedTestScript(test, normalize_slashes=False,
320                              extra_substitutions=[]):
321    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
322    script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES'
323    information. The RUN lines also will have variable substitution performed.
324    """
325
326    # Get the temporary location, this is always relative to the test suite
327    # root, not test source root.
328    #
329    # FIXME: This should not be here?
330    sourcepath = test.getSourcePath()
331    sourcedir = os.path.dirname(sourcepath)
332    execpath = test.getExecPath()
333    execdir,execbase = os.path.split(execpath)
334    tmpDir = os.path.join(execdir, 'Output')
335    tmpBase = os.path.join(tmpDir, execbase)
336    if test.index is not None:
337        tmpBase += '_%d' % test.index
338
339    # Normalize slashes, if requested.
340    if normalize_slashes:
341        sourcepath = sourcepath.replace('\\', '/')
342        sourcedir = sourcedir.replace('\\', '/')
343        tmpDir = tmpDir.replace('\\', '/')
344        tmpBase = tmpBase.replace('\\', '/')
345
346    # We use #_MARKER_# to hide %% while we do the other substitutions.
347    substitutions = list(extra_substitutions)
348    substitutions.extend([('%%', '#_MARKER_#')])
349    substitutions.extend(test.config.substitutions)
350    substitutions.extend([('%s', sourcepath),
351                          ('%S', sourcedir),
352                          ('%p', sourcedir),
353                          ('%{pathsep}', os.pathsep),
354                          ('%t', tmpBase + '.tmp'),
355                          ('%T', tmpDir),
356                          ('#_MARKER_#', '%')])
357
358    # Collect the test lines from the script.
359    script = []
360    xfails = []
361    requires = []
362    line_number = 0
363    for ln in open(sourcepath):
364        line_number += 1
365        if 'RUN:' in ln:
366            # Isolate the command to run.
367            index = ln.index('RUN:')
368            ln = ln[index+4:]
369
370            # Trim trailing whitespace.
371            ln = ln.rstrip()
372
373            # Substitute line number expressions
374            ln = re.sub('%\(line\)', str(line_number), ln)
375            def replace_line_number(match):
376                if match.group(1) == '+':
377                    return str(line_number + int(match.group(2)))
378                if match.group(1) == '-':
379                    return str(line_number - int(match.group(2)))
380            ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln)
381
382            # Collapse lines with trailing '\\'.
383            if script and script[-1][-1] == '\\':
384                script[-1] = script[-1][:-1] + ln
385            else:
386                script.append(ln)
387        elif 'XFAIL:' in ln:
388            items = ln[ln.index('XFAIL:') + 6:].split(',')
389            xfails.extend([s.strip() for s in items])
390        elif 'REQUIRES:' in ln:
391            items = ln[ln.index('REQUIRES:') + 9:].split(',')
392            requires.extend([s.strip() for s in items])
393        elif 'END.' in ln:
394            # Check for END. lines.
395            if ln[ln.index('END.'):].strip() == 'END.':
396                break
397
398    # Apply substitutions to the script.  Allow full regular
399    # expression syntax.  Replace each matching occurrence of regular
400    # expression pattern a with substitution b in line ln.
401    def processLine(ln):
402        # Apply substitutions
403        for a,b in substitutions:
404            if kIsWindows:
405                b = b.replace("\\","\\\\")
406            ln = re.sub(a, b, ln)
407
408        # Strip the trailing newline and any extra whitespace.
409        return ln.strip()
410    script = map(processLine, script)
411
412    # Verify the script contains a run line.
413    if not script:
414        return (Test.UNRESOLVED, "Test has no run line!")
415
416    # Check for unterminated run lines.
417    if script[-1][-1] == '\\':
418        return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')")
419
420    # Check that we have the required features:
421    missing_required_features = [f for f in requires
422                                 if f not in test.config.available_features]
423    if missing_required_features:
424        msg = ', '.join(missing_required_features)
425        return (Test.UNSUPPORTED,
426                "Test requires the following features: %s" % msg)
427
428    isXFail = isExpectedFail(test, xfails)
429    return script,isXFail,tmpBase,execdir
430
431def formatTestOutput(status, out, err, exitCode, script):
432    output = StringIO.StringIO()
433    print >>output, "Script:"
434    print >>output, "--"
435    print >>output, '\n'.join(script)
436    print >>output, "--"
437    print >>output, "Exit Code: %r" % exitCode,
438    print >>output
439    if out:
440        print >>output, "Command Output (stdout):"
441        print >>output, "--"
442        output.write(out)
443        print >>output, "--"
444    if err:
445        print >>output, "Command Output (stderr):"
446        print >>output, "--"
447        output.write(err)
448        print >>output, "--"
449    return (status, output.getvalue())
450
451def executeShTest(test, litConfig, useExternalSh,
452                  extra_substitutions=[]):
453    if test.config.unsupported:
454        return (Test.UNSUPPORTED, 'Test is unsupported')
455
456    res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions)
457    if len(res) == 2:
458        return res
459
460    script, isXFail, tmpBase, execdir = res
461
462    if litConfig.noExecute:
463        return (Test.PASS, '')
464
465    # Create the output directory if it does not already exist.
466    Util.mkdir_p(os.path.dirname(tmpBase))
467
468    if useExternalSh:
469        res = executeScript(test, litConfig, tmpBase, script, execdir)
470    else:
471        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
472    if len(res) == 2:
473        return res
474
475    out,err,exitCode = res
476    if isXFail:
477        ok = exitCode != 0
478        if ok:
479            status = Test.XFAIL
480        else:
481            status = Test.XPASS
482    else:
483        ok = exitCode == 0
484        if ok:
485            status = Test.PASS
486        else:
487            status = Test.FAIL
488
489    if ok:
490        return (status,'')
491
492    return formatTestOutput(status, out, err, exitCode, script)
493