cmd_helper.py revision cedac228d2dd51db4b79ea1e72c7f249408ee061
15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Copyright (c) 2012 The Chromium Authors. All rights reserved.
25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be
35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# found in the LICENSE file.
45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)"""A wrapper for subprocess to make calling shell commands easier."""
65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import logging
8cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)import os
9c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import pipes
10cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)import select
11c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import signal
12cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)import StringIO
135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import subprocess
14cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)import time
155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
16cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# fcntl is not available on Windows.
17cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)try:
18cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  import fcntl
19cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)except ImportError:
20cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  fcntl = None
215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
22c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
23f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)def Popen(args, stdout=None, stderr=None, shell=None, cwd=None, env=None):
24f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  return subprocess.Popen(
25c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      args=args, cwd=cwd, stdout=stdout, stderr=stderr,
26424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)      shell=shell, close_fds=True, env=env,
27c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      preexec_fn=lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL))
28c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
29c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
30f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)def Call(args, stdout=None, stderr=None, shell=None, cwd=None, env=None):
31f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  pipe = Popen(args, stdout=stdout, stderr=stderr, shell=shell, cwd=cwd,
32f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)               env=env)
33f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  pipe.communicate()
34f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  return pipe.wait()
35f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
36f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def RunCmd(args, cwd=None):
385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  """Opens a subprocess to execute a program and returns its return value.
395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  Args:
415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    args: A string or a sequence of program arguments. The program to execute is
425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      the string or the first item in the args sequence.
435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    cwd: If not None, the subprocess's current directory will be changed to
445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      |cwd| before it's executed.
455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  Returns:
475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    Return code from the command execution.
485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  """
495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  logging.info(str(args) + ' ' + (cwd or ''))
50424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)  return Call(args, cwd=cwd)
515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetCmdOutput(args, cwd=None, shell=False):
545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  """Open a subprocess to execute a program and returns its output.
555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  Args:
575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    args: A string or a sequence of program arguments. The program to execute is
585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      the string or the first item in the args sequence.
595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    cwd: If not None, the subprocess's current directory will be changed to
605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      |cwd| before it's executed.
615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    shell: Whether to execute args as a shell command.
625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  Returns:
645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    Captures and returns the command's stdout.
655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    Prints the command's stderr to logger (which defaults to stdout).
665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  """
675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  (_, output) = GetCmdStatusAndOutput(args, cwd, shell)
685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  return output
695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
70c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetCmdStatusAndOutput(args, cwd=None, shell=False):
725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  """Executes a subprocess and returns its exit code and output.
735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  Args:
755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    args: A string or a sequence of program arguments. The program to execute is
765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      the string or the first item in the args sequence.
775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    cwd: If not None, the subprocess's current directory will be changed to
785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)      |cwd| before it's executed.
795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    shell: Whether to execute args as a shell command.
805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)
815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  Returns:
82f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)    The 2-tuple (exit code, output).
835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  """
84c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  if isinstance(args, basestring):
85c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    args_repr = args
86c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    if not shell:
87c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      raise Exception('string args must be run with shell=True')
88c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  elif shell:
89c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    raise Exception('array args must be run with shell=False')
90c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  else:
91c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    args_repr = ' '.join(map(pipes.quote, args))
92c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
93c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  s = '[host]'
94c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  if cwd:
95c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    s += ':' + cwd
96c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  s += '> ' + args_repr
97c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  logging.info(s)
98cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  pipe = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
99cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)               shell=shell, cwd=cwd)
100cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  stdout, stderr = pipe.communicate()
101cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)
1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)  if stderr:
1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)    logging.critical(stderr)
104c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  if len(stdout) > 4096:
105c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    logging.debug('Truncated output:')
106c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  logging.debug(stdout[:4096])
107cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  return (pipe.returncode, stdout)
108cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)
109cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)
110cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)class TimeoutError(Exception):
111cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  """Module-specific timeout exception."""
112cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  pass
113f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
114f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
115cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)def GetCmdStatusAndOutputWithTimeout(args, timeout, cwd=None, shell=False,
116cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                                     logfile=None):
117cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  """Executes a subprocess with a timeout.
118f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
119f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  Args:
120f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)    args: List of arguments to the program, the program to execute is the first
121f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)      element.
122cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    timeout: the timeout in seconds or None to wait forever.
123cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    cwd: If not None, the subprocess's current directory will be changed to
124cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      |cwd| before it's executed.
125cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    shell: Whether to execute args as a shell command.
126cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    logfile: Optional file-like object that will receive output from the
127cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      command as it is running.
128f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
129f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  Returns:
130f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)    The 2-tuple (exit code, output).
131f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  """
132cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  assert fcntl, 'fcntl module is required'
133cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  process = Popen(args, cwd=cwd, shell=shell, stdout=subprocess.PIPE,
134cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                  stderr=subprocess.STDOUT)
135cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  try:
136cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    end_time = (time.time() + timeout) if timeout else None
137cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    poll_interval = 1
138cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    buffer_size = 4096
139cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    child_fd = process.stdout.fileno()
140cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    output = StringIO.StringIO()
141cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)
142cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    # Enable non-blocking reads from the child's stdout.
143cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    fl = fcntl.fcntl(child_fd, fcntl.F_GETFL)
144cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    fcntl.fcntl(child_fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
145cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)
146cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    while True:
147cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      if end_time and time.time() > end_time:
148cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)        raise TimeoutError
149cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      read_fds, _, _ = select.select([child_fd], [], [], poll_interval)
150cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      if child_fd in read_fds:
151cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)        data = os.read(child_fd, buffer_size)
152cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)        if not data:
153cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)          break
154cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)        if logfile:
155cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)          logfile.write(data)
156cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)        output.write(data)
157cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      if process.poll() is not None:
158cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)        break
159cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  finally:
160cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    try:
161cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      # Make sure the process doesn't stick around if we fail with an
162cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      # exception.
163cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      process.kill()
164cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    except OSError:
165cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      pass
166cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    process.wait()
167cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  return process.returncode, output.getvalue()
168