cmd_helper.py revision eeb03f366d148f92337cfd7577087ade44ab9285
1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""A wrapper for subprocess to make calling shell commands easier.""" 6 7import logging 8import os 9import pipes 10import select 11import signal 12import string 13import StringIO 14import subprocess 15import time 16 17# fcntl is not available on Windows. 18try: 19 import fcntl 20except ImportError: 21 fcntl = None 22 23logger = logging.getLogger(__name__) 24 25_SafeShellChars = frozenset(string.ascii_letters + string.digits + '@%_-+=:,./') 26 27 28def SingleQuote(s): 29 """Return an shell-escaped version of the string using single quotes. 30 31 Reliably quote a string which may contain unsafe characters (e.g. space, 32 quote, or other special characters such as '$'). 33 34 The returned value can be used in a shell command line as one token that gets 35 to be interpreted literally. 36 37 Args: 38 s: The string to quote. 39 40 Return: 41 The string quoted using single quotes. 42 """ 43 return pipes.quote(s) 44 45 46def DoubleQuote(s): 47 """Return an shell-escaped version of the string using double quotes. 48 49 Reliably quote a string which may contain unsafe characters (e.g. space 50 or quote characters), while retaining some shell features such as variable 51 interpolation. 52 53 The returned value can be used in a shell command line as one token that gets 54 to be further interpreted by the shell. 55 56 The set of characters that retain their special meaning may depend on the 57 shell implementation. This set usually includes: '$', '`', '\', '!', '*', 58 and '@'. 59 60 Args: 61 s: The string to quote. 62 63 Return: 64 The string quoted using double quotes. 65 """ 66 if not s: 67 return '""' 68 elif all(c in _SafeShellChars for c in s): 69 return s 70 else: 71 return '"' + s.replace('"', '\\"') + '"' 72 73 74def ShrinkToSnippet(cmd_parts, var_name, var_value): 75 """Constructs a shell snippet for a command using a variable to shrink it. 76 77 Takes into account all quoting that needs to happen. 78 79 Args: 80 cmd_parts: A list of command arguments. 81 var_name: The variable that holds var_value. 82 var_value: The string to replace in cmd_parts with $var_name 83 84 Returns: 85 A shell snippet that does not include setting the variable. 86 """ 87 def shrink(value): 88 parts = (x and SingleQuote(x) for x in value.split(var_value)) 89 with_substitutions = ('"$%s"' % var_name).join(parts) 90 return with_substitutions or "''" 91 92 return ' '.join(shrink(part) for part in cmd_parts) 93 94 95def Popen(args, stdout=None, stderr=None, shell=None, cwd=None, env=None): 96 return subprocess.Popen( 97 args=args, cwd=cwd, stdout=stdout, stderr=stderr, 98 shell=shell, close_fds=True, env=env, 99 preexec_fn=lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL)) 100 101 102def Call(args, stdout=None, stderr=None, shell=None, cwd=None, env=None): 103 pipe = Popen(args, stdout=stdout, stderr=stderr, shell=shell, cwd=cwd, 104 env=env) 105 pipe.communicate() 106 return pipe.wait() 107 108 109def RunCmd(args, cwd=None): 110 """Opens a subprocess to execute a program and returns its return value. 111 112 Args: 113 args: A string or a sequence of program arguments. The program to execute is 114 the string or the first item in the args sequence. 115 cwd: If not None, the subprocess's current directory will be changed to 116 |cwd| before it's executed. 117 118 Returns: 119 Return code from the command execution. 120 """ 121 logger.info(str(args) + ' ' + (cwd or '')) 122 return Call(args, cwd=cwd) 123 124 125def GetCmdOutput(args, cwd=None, shell=False): 126 """Open a subprocess to execute a program and returns its output. 127 128 Args: 129 args: A string or a sequence of program arguments. The program to execute is 130 the string or the first item in the args sequence. 131 cwd: If not None, the subprocess's current directory will be changed to 132 |cwd| before it's executed. 133 shell: Whether to execute args as a shell command. 134 135 Returns: 136 Captures and returns the command's stdout. 137 Prints the command's stderr to logger (which defaults to stdout). 138 """ 139 (_, output) = GetCmdStatusAndOutput(args, cwd, shell) 140 return output 141 142 143def _ValidateAndLogCommand(args, cwd, shell): 144 if isinstance(args, basestring): 145 if not shell: 146 raise Exception('string args must be run with shell=True') 147 else: 148 if shell: 149 raise Exception('array args must be run with shell=False') 150 args = ' '.join(SingleQuote(c) for c in args) 151 if cwd is None: 152 cwd = '' 153 else: 154 cwd = ':' + cwd 155 logger.info('[host]%s> %s', cwd, args) 156 return args 157 158 159def GetCmdStatusAndOutput(args, cwd=None, shell=False): 160 """Executes a subprocess and returns its exit code and output. 161 162 Args: 163 args: A string or a sequence of program arguments. The program to execute is 164 the string or the first item in the args sequence. 165 cwd: If not None, the subprocess's current directory will be changed to 166 |cwd| before it's executed. 167 shell: Whether to execute args as a shell command. Must be True if args 168 is a string and False if args is a sequence. 169 170 Returns: 171 The 2-tuple (exit code, output). 172 """ 173 status, stdout, stderr = GetCmdStatusOutputAndError( 174 args, cwd=cwd, shell=shell) 175 176 if stderr: 177 logger.critical('STDERR: %s', stderr) 178 logger.debug('STDOUT: %s%s', stdout[:4096].rstrip(), 179 '<truncated>' if len(stdout) > 4096 else '') 180 return (status, stdout) 181 182 183def GetCmdStatusOutputAndError(args, cwd=None, shell=False): 184 """Executes a subprocess and returns its exit code, output, and errors. 185 186 Args: 187 args: A string or a sequence of program arguments. The program to execute is 188 the string or the first item in the args sequence. 189 cwd: If not None, the subprocess's current directory will be changed to 190 |cwd| before it's executed. 191 shell: Whether to execute args as a shell command. Must be True if args 192 is a string and False if args is a sequence. 193 194 Returns: 195 The 2-tuple (exit code, output). 196 """ 197 _ValidateAndLogCommand(args, cwd, shell) 198 pipe = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, 199 shell=shell, cwd=cwd) 200 stdout, stderr = pipe.communicate() 201 return (pipe.returncode, stdout, stderr) 202 203 204class TimeoutError(Exception): 205 """Module-specific timeout exception.""" 206 207 def __init__(self, output=None): 208 super(TimeoutError, self).__init__() 209 self._output = output 210 211 @property 212 def output(self): 213 return self._output 214 215 216def _IterProcessStdout(process, timeout=None, buffer_size=4096, 217 poll_interval=1): 218 assert fcntl, 'fcntl module is required' 219 try: 220 # Enable non-blocking reads from the child's stdout. 221 child_fd = process.stdout.fileno() 222 fl = fcntl.fcntl(child_fd, fcntl.F_GETFL) 223 fcntl.fcntl(child_fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) 224 225 end_time = (time.time() + timeout) if timeout else None 226 while True: 227 if end_time and time.time() > end_time: 228 raise TimeoutError() 229 read_fds, _, _ = select.select([child_fd], [], [], poll_interval) 230 if child_fd in read_fds: 231 data = os.read(child_fd, buffer_size) 232 if not data: 233 break 234 yield data 235 if process.poll() is not None: 236 break 237 finally: 238 try: 239 # Make sure the process doesn't stick around if we fail with an 240 # exception. 241 process.kill() 242 except OSError: 243 pass 244 process.wait() 245 246 247def GetCmdStatusAndOutputWithTimeout(args, timeout, cwd=None, shell=False, 248 logfile=None): 249 """Executes a subprocess with a timeout. 250 251 Args: 252 args: List of arguments to the program, the program to execute is the first 253 element. 254 timeout: the timeout in seconds or None to wait forever. 255 cwd: If not None, the subprocess's current directory will be changed to 256 |cwd| before it's executed. 257 shell: Whether to execute args as a shell command. Must be True if args 258 is a string and False if args is a sequence. 259 logfile: Optional file-like object that will receive output from the 260 command as it is running. 261 262 Returns: 263 The 2-tuple (exit code, output). 264 Raises: 265 TimeoutError on timeout. 266 """ 267 _ValidateAndLogCommand(args, cwd, shell) 268 output = StringIO.StringIO() 269 process = Popen(args, cwd=cwd, shell=shell, stdout=subprocess.PIPE, 270 stderr=subprocess.STDOUT) 271 try: 272 for data in _IterProcessStdout(process, timeout=timeout): 273 if logfile: 274 logfile.write(data) 275 output.write(data) 276 except TimeoutError: 277 raise TimeoutError(output.getvalue()) 278 279 str_output = output.getvalue() 280 logger.debug('STDOUT+STDERR: %s%s', str_output[:4096].rstrip(), 281 '<truncated>' if len(str_output) > 4096 else '') 282 return process.returncode, str_output 283 284 285def IterCmdOutputLines(args, timeout=None, cwd=None, shell=False, 286 check_status=True): 287 """Executes a subprocess and continuously yields lines from its output. 288 289 Args: 290 args: List of arguments to the program, the program to execute is the first 291 element. 292 cwd: If not None, the subprocess's current directory will be changed to 293 |cwd| before it's executed. 294 shell: Whether to execute args as a shell command. Must be True if args 295 is a string and False if args is a sequence. 296 check_status: A boolean indicating whether to check the exit status of the 297 process after all output has been read. 298 299 Yields: 300 The output of the subprocess, line by line. 301 302 Raises: 303 CalledProcessError if check_status is True and the process exited with a 304 non-zero exit status. 305 """ 306 cmd = _ValidateAndLogCommand(args, cwd, shell) 307 process = Popen(args, cwd=cwd, shell=shell, stdout=subprocess.PIPE, 308 stderr=subprocess.STDOUT) 309 buffer_output = '' 310 for data in _IterProcessStdout(process, timeout=timeout): 311 buffer_output += data 312 has_incomplete_line = buffer_output[-1] not in '\r\n' 313 lines = buffer_output.splitlines() 314 buffer_output = lines.pop() if has_incomplete_line else '' 315 for line in lines: 316 yield line 317 if buffer_output: 318 yield buffer_output 319 if check_status and process.returncode: 320 raise subprocess.CalledProcessError(process.returncode, cmd) 321