TestRunner.py revision 35d5e9044c580c843eb6e825d87816619e6d7f8f
1from __future__ import absolute_import 2import os, signal, subprocess, sys 3import re 4import platform 5import tempfile 6 7import lit.ShUtil as ShUtil 8import lit.Test as Test 9import lit.util 10 11class InternalShellError(Exception): 12 def __init__(self, command, message): 13 self.command = command 14 self.message = message 15 16kIsWindows = platform.system() == 'Windows' 17 18# Don't use close_fds on Windows. 19kUseCloseFDs = not kIsWindows 20 21# Use temporary files to replace /dev/null on Windows. 22kAvoidDevNull = kIsWindows 23 24def executeShCmd(cmd, cfg, cwd, results): 25 if isinstance(cmd, ShUtil.Seq): 26 if cmd.op == ';': 27 res = executeShCmd(cmd.lhs, cfg, cwd, results) 28 return executeShCmd(cmd.rhs, cfg, cwd, results) 29 30 if cmd.op == '&': 31 raise InternalShellError(cmd,"unsupported shell operator: '&'") 32 33 if cmd.op == '||': 34 res = executeShCmd(cmd.lhs, cfg, cwd, results) 35 if res != 0: 36 res = executeShCmd(cmd.rhs, cfg, cwd, results) 37 return res 38 39 if cmd.op == '&&': 40 res = executeShCmd(cmd.lhs, cfg, cwd, results) 41 if res is None: 42 return res 43 44 if res == 0: 45 res = executeShCmd(cmd.rhs, cfg, cwd, results) 46 return res 47 48 raise ValueError('Unknown shell command: %r' % cmd.op) 49 50 assert isinstance(cmd, ShUtil.Pipeline) 51 procs = [] 52 input = subprocess.PIPE 53 stderrTempFiles = [] 54 opened_files = [] 55 named_temp_files = [] 56 # To avoid deadlock, we use a single stderr stream for piped 57 # output. This is null until we have seen some output using 58 # stderr. 59 for i,j in enumerate(cmd.commands): 60 # Apply the redirections, we use (N,) as a sentinel to indicate stdin, 61 # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or 62 # from a file are represented with a list [file, mode, file-object] 63 # where file-object is initially None. 64 redirects = [(0,), (1,), (2,)] 65 for r in j.redirects: 66 if r[0] == ('>',2): 67 redirects[2] = [r[1], 'w', None] 68 elif r[0] == ('>>',2): 69 redirects[2] = [r[1], 'a', None] 70 elif r[0] == ('>&',2) and r[1] in '012': 71 redirects[2] = redirects[int(r[1])] 72 elif r[0] == ('>&',) or r[0] == ('&>',): 73 redirects[1] = redirects[2] = [r[1], 'w', None] 74 elif r[0] == ('>',): 75 redirects[1] = [r[1], 'w', None] 76 elif r[0] == ('>>',): 77 redirects[1] = [r[1], 'a', None] 78 elif r[0] == ('<',): 79 redirects[0] = [r[1], 'r', None] 80 else: 81 raise InternalShellError(j,"Unsupported redirect: %r" % (r,)) 82 83 # Map from the final redirections to something subprocess can handle. 84 final_redirects = [] 85 for index,r in enumerate(redirects): 86 if r == (0,): 87 result = input 88 elif r == (1,): 89 if index == 0: 90 raise InternalShellError(j,"Unsupported redirect for stdin") 91 elif index == 1: 92 result = subprocess.PIPE 93 else: 94 result = subprocess.STDOUT 95 elif r == (2,): 96 if index != 2: 97 raise InternalShellError(j,"Unsupported redirect on stdout") 98 result = subprocess.PIPE 99 else: 100 if r[2] is None: 101 if kAvoidDevNull and r[0] == '/dev/null': 102 r[2] = tempfile.TemporaryFile(mode=r[1]) 103 else: 104 r[2] = open(r[0], r[1]) 105 # Workaround a Win32 and/or subprocess bug when appending. 106 # 107 # FIXME: Actually, this is probably an instance of PR6753. 108 if r[1] == 'a': 109 r[2].seek(0, 2) 110 opened_files.append(r[2]) 111 result = r[2] 112 final_redirects.append(result) 113 114 stdin, stdout, stderr = final_redirects 115 116 # If stderr wants to come from stdout, but stdout isn't a pipe, then put 117 # stderr on a pipe and treat it as stdout. 118 if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE): 119 stderr = subprocess.PIPE 120 stderrIsStdout = True 121 else: 122 stderrIsStdout = False 123 124 # Don't allow stderr on a PIPE except for the last 125 # process, this could deadlock. 126 # 127 # FIXME: This is slow, but so is deadlock. 128 if stderr == subprocess.PIPE and j != cmd.commands[-1]: 129 stderr = tempfile.TemporaryFile(mode='w+b') 130 stderrTempFiles.append((i, stderr)) 131 132 # Resolve the executable path ourselves. 133 args = list(j.args) 134 args[0] = lit.util.which(args[0], cfg.environment['PATH']) 135 if not args[0]: 136 raise InternalShellError(j, '%r: command not found' % j.args[0]) 137 138 # Replace uses of /dev/null with temporary files. 139 if kAvoidDevNull: 140 for i,arg in enumerate(args): 141 if arg == "/dev/null": 142 f = tempfile.NamedTemporaryFile(delete=False) 143 f.close() 144 named_temp_files.append(f.name) 145 args[i] = f.name 146 147 procs.append(subprocess.Popen(args, cwd=cwd, 148 stdin = stdin, 149 stdout = stdout, 150 stderr = stderr, 151 env = cfg.environment, 152 close_fds = kUseCloseFDs)) 153 154 # Immediately close stdin for any process taking stdin from us. 155 if stdin == subprocess.PIPE: 156 procs[-1].stdin.close() 157 procs[-1].stdin = None 158 159 # Update the current stdin source. 160 if stdout == subprocess.PIPE: 161 input = procs[-1].stdout 162 elif stderrIsStdout: 163 input = procs[-1].stderr 164 else: 165 input = subprocess.PIPE 166 167 # Explicitly close any redirected files. We need to do this now because we 168 # need to release any handles we may have on the temporary files (important 169 # on Win32, for example). Since we have already spawned the subprocess, our 170 # handles have already been transferred so we do not need them anymore. 171 for f in opened_files: 172 f.close() 173 174 # FIXME: There is probably still deadlock potential here. Yawn. 175 procData = [None] * len(procs) 176 procData[-1] = procs[-1].communicate() 177 178 for i in range(len(procs) - 1): 179 if procs[i].stdout is not None: 180 out = procs[i].stdout.read() 181 else: 182 out = '' 183 if procs[i].stderr is not None: 184 err = procs[i].stderr.read() 185 else: 186 err = '' 187 procData[i] = (out,err) 188 189 # Read stderr out of the temp files. 190 for i,f in stderrTempFiles: 191 f.seek(0, 0) 192 procData[i] = (procData[i][0], f.read()) 193 194 exitCode = None 195 for i,(out,err) in enumerate(procData): 196 res = procs[i].wait() 197 # Detect Ctrl-C in subprocess. 198 if res == -signal.SIGINT: 199 raise KeyboardInterrupt 200 201 results.append((cmd.commands[i], out, err, res)) 202 if cmd.pipe_err: 203 # Python treats the exit code as a signed char. 204 if exitCode is None: 205 exitCode = res 206 elif res < 0: 207 exitCode = min(exitCode, res) 208 else: 209 exitCode = max(exitCode, res) 210 else: 211 exitCode = res 212 213 # Remove any named temporary files we created. 214 for f in named_temp_files: 215 try: 216 os.remove(f) 217 except OSError: 218 pass 219 220 if cmd.negate: 221 exitCode = not exitCode 222 223 return exitCode 224 225def executeScriptInternal(test, litConfig, tmpBase, commands, cwd): 226 cmds = [] 227 for ln in commands: 228 try: 229 cmds.append(ShUtil.ShParser(ln, litConfig.isWindows, 230 test.config.pipefail).parse()) 231 except: 232 return (Test.FAIL, "shell parser error on: %r" % ln) 233 234 cmd = cmds[0] 235 for c in cmds[1:]: 236 cmd = ShUtil.Seq(cmd, '&&', c) 237 238 results = [] 239 try: 240 exitCode = executeShCmd(cmd, test.config, cwd, results) 241 except InternalShellError: 242 e = sys.exc_info()[1] 243 exitCode = 127 244 results.append((e.command, '', e.message, exitCode)) 245 246 out = err = '' 247 for i,(cmd, cmd_out,cmd_err,res) in enumerate(results): 248 out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args)) 249 out += 'Command %d Result: %r\n' % (i, res) 250 out += 'Command %d Output:\n%s\n\n' % (i, cmd_out) 251 out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err) 252 253 return out, err, exitCode 254 255def executeScript(test, litConfig, tmpBase, commands, cwd): 256 bashPath = litConfig.getBashPath(); 257 isWin32CMDEXE = (litConfig.isWindows and not bashPath) 258 script = tmpBase + '.script' 259 if isWin32CMDEXE: 260 script += '.bat' 261 262 # Write script file 263 mode = 'w' 264 if litConfig.isWindows and not isWin32CMDEXE: 265 mode += 'b' # Avoid CRLFs when writing bash scripts. 266 f = open(script, mode) 267 if isWin32CMDEXE: 268 f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands)) 269 else: 270 if test.config.pipefail: 271 f.write('set -o pipefail;') 272 f.write('{ ' + '; } &&\n{ '.join(commands) + '; }') 273 f.write('\n') 274 f.close() 275 276 if isWin32CMDEXE: 277 command = ['cmd','/c', script] 278 else: 279 if bashPath: 280 command = [bashPath, script] 281 else: 282 command = ['/bin/sh', script] 283 if litConfig.useValgrind: 284 # FIXME: Running valgrind on sh is overkill. We probably could just 285 # run on clang with no real loss. 286 command = litConfig.valgrindArgs + command 287 288 return lit.util.executeCommand(command, cwd=cwd, 289 env=test.config.environment) 290 291def isExpectedFail(test, xfails): 292 # Check if any of the xfails match an available feature or the target. 293 for item in xfails: 294 # If this is the wildcard, it always fails. 295 if item == '*': 296 return True 297 298 # If this is an exact match for one of the features, it fails. 299 if item in test.config.available_features: 300 return True 301 302 # If this is a part of the target triple, it fails. 303 if item in test.suite.config.target_triple: 304 return True 305 306 return False 307 308def parseIntegratedTestScriptCommands(source_path): 309 """ 310 parseIntegratedTestScriptCommands(source_path) -> commands 311 312 Parse the commands in an integrated test script file into a list of 313 (line_number, command_type, line). 314 """ 315 316 # This code is carefully written to be dual compatible with Python 2.5+ and 317 # Python 3 without requiring input files to always have valid codings. The 318 # trick we use is to open the file in binary mode and use the regular 319 # expression library to find the commands, with it scanning strings in 320 # Python2 and bytes in Python3. 321 # 322 # Once we find a match, we do require each script line to be decodable to 323 # ascii, so we convert the outputs to ascii before returning. This way the 324 # remaining code can work with "strings" agnostic of the executing Python 325 # version. 326 327 def to_bytes(str): 328 # Encode to Latin1 to get binary data. 329 return str.encode('ISO-8859-1') 330 keywords = ('RUN:', 'XFAIL:', 'REQUIRES:', 'END.') 331 keywords_re = re.compile( 332 to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),))) 333 334 f = open(source_path, 'rb') 335 try: 336 # Read the entire file contents. 337 data = f.read() 338 339 # Iterate over the matches. 340 line_number = 1 341 last_match_position = 0 342 for match in keywords_re.finditer(data): 343 # Compute the updated line number by counting the intervening 344 # newlines. 345 match_position = match.start() 346 line_number += data.count(to_bytes('\n'), last_match_position, 347 match_position) 348 last_match_position = match_position 349 350 # Convert the keyword and line to ascii strings and yield the 351 # command. Note that we take care to return regular strings in 352 # Python 2, to avoid other code having to differentiate between the 353 # str and unicode types. 354 keyword,ln = match.groups() 355 yield (line_number, str(keyword[:-1].decode('ascii')), 356 str(ln.decode('ascii'))) 357 finally: 358 f.close() 359 360def parseIntegratedTestScript(test, normalize_slashes=False, 361 extra_substitutions=[]): 362 """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test 363 script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES' 364 information. The RUN lines also will have variable substitution performed. 365 """ 366 367 # Get the temporary location, this is always relative to the test suite 368 # root, not test source root. 369 # 370 # FIXME: This should not be here? 371 sourcepath = test.getSourcePath() 372 sourcedir = os.path.dirname(sourcepath) 373 execpath = test.getExecPath() 374 execdir,execbase = os.path.split(execpath) 375 tmpDir = os.path.join(execdir, 'Output') 376 tmpBase = os.path.join(tmpDir, execbase) 377 378 # Normalize slashes, if requested. 379 if normalize_slashes: 380 sourcepath = sourcepath.replace('\\', '/') 381 sourcedir = sourcedir.replace('\\', '/') 382 tmpDir = tmpDir.replace('\\', '/') 383 tmpBase = tmpBase.replace('\\', '/') 384 385 # We use #_MARKER_# to hide %% while we do the other substitutions. 386 substitutions = list(extra_substitutions) 387 substitutions.extend([('%%', '#_MARKER_#')]) 388 substitutions.extend(test.config.substitutions) 389 substitutions.extend([('%s', sourcepath), 390 ('%S', sourcedir), 391 ('%p', sourcedir), 392 ('%{pathsep}', os.pathsep), 393 ('%t', tmpBase + '.tmp'), 394 ('%T', tmpDir), 395 ('#_MARKER_#', '%')]) 396 397 # "%/[STpst]" should be normalized. 398 substitutions.extend([ 399 ('%/s', sourcepath.replace('\\', '/')), 400 ('%/S', sourcedir.replace('\\', '/')), 401 ('%/p', sourcedir.replace('\\', '/')), 402 ('%/t', tmpBase.replace('\\', '/') + '.tmp'), 403 ('%/T', tmpDir.replace('\\', '/')), 404 ]) 405 406 # Collect the test lines from the script. 407 script = [] 408 xfails = [] 409 requires = [] 410 for line_number, command_type, ln in \ 411 parseIntegratedTestScriptCommands(sourcepath): 412 if command_type == 'RUN': 413 # Trim trailing whitespace. 414 ln = ln.rstrip() 415 416 # Substitute line number expressions 417 ln = re.sub('%\(line\)', str(line_number), ln) 418 def replace_line_number(match): 419 if match.group(1) == '+': 420 return str(line_number + int(match.group(2))) 421 if match.group(1) == '-': 422 return str(line_number - int(match.group(2))) 423 ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln) 424 425 # Collapse lines with trailing '\\'. 426 if script and script[-1][-1] == '\\': 427 script[-1] = script[-1][:-1] + ln 428 else: 429 script.append(ln) 430 elif command_type == 'XFAIL': 431 xfails.extend([s.strip() for s in ln.split(',')]) 432 elif command_type == 'REQUIRES': 433 requires.extend([s.strip() for s in ln.split(',')]) 434 elif command_type == 'END': 435 # END commands are only honored if the rest of the line is empty. 436 if not ln.strip(): 437 break 438 else: 439 raise ValueError("unknown script command type: %r" % ( 440 command_type,)) 441 442 # Apply substitutions to the script. Allow full regular 443 # expression syntax. Replace each matching occurrence of regular 444 # expression pattern a with substitution b in line ln. 445 def processLine(ln): 446 # Apply substitutions 447 for a,b in substitutions: 448 if kIsWindows: 449 b = b.replace("\\","\\\\") 450 ln = re.sub(a, b, ln) 451 452 # Strip the trailing newline and any extra whitespace. 453 return ln.strip() 454 script = [processLine(ln) 455 for ln in script] 456 457 # Verify the script contains a run line. 458 if not script: 459 return (Test.UNRESOLVED, "Test has no run line!") 460 461 # Check for unterminated run lines. 462 if script[-1][-1] == '\\': 463 return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')") 464 465 # Check that we have the required features: 466 missing_required_features = [f for f in requires 467 if f not in test.config.available_features] 468 if missing_required_features: 469 msg = ', '.join(missing_required_features) 470 return (Test.UNSUPPORTED, 471 "Test requires the following features: %s" % msg) 472 473 isXFail = isExpectedFail(test, xfails) 474 return script,isXFail,tmpBase,execdir 475 476def formatTestOutput(status, out, err, exitCode, script): 477 output = """\ 478Script: 479-- 480%s 481-- 482Exit Code: %d 483 484""" % ('\n'.join(script), exitCode) 485 486 # Append the stdout, if present. 487 if out: 488 output += """\ 489Command Output (stdout): 490-- 491%s 492-- 493""" % (out,) 494 495 # Append the stderr, if present. 496 if err: 497 output += """\ 498Command Output (stderr): 499-- 500%s 501-- 502""" % (err,) 503 return (status, output) 504 505def executeShTest(test, litConfig, useExternalSh, 506 extra_substitutions=[]): 507 if test.config.unsupported: 508 return (Test.UNSUPPORTED, 'Test is unsupported') 509 510 res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions) 511 if len(res) == 2: 512 return res 513 514 script, isXFail, tmpBase, execdir = res 515 516 if litConfig.noExecute: 517 return (Test.PASS, '') 518 519 # Create the output directory if it does not already exist. 520 lit.util.mkdir_p(os.path.dirname(tmpBase)) 521 522 if useExternalSh: 523 res = executeScript(test, litConfig, tmpBase, script, execdir) 524 else: 525 res = executeScriptInternal(test, litConfig, tmpBase, script, execdir) 526 if len(res) == 2: 527 return res 528 529 out,err,exitCode = res 530 if isXFail: 531 ok = exitCode != 0 532 if ok: 533 status = Test.XFAIL 534 else: 535 status = Test.XPASS 536 else: 537 ok = exitCode == 0 538 if ok: 539 status = Test.PASS 540 else: 541 status = Test.FAIL 542 543 if ok: 544 return (status,'') 545 546 return formatTestOutput(status, out, err, exitCode, script) 547