15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Copyright (c) 2009, Google Inc. All rights reserved. 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Copyright (c) 2009 Apple Inc. All rights reserved. 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 45c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Redistribution and use in source and binary forms, with or without 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# modification, are permitted provided that the following conditions are 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# met: 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 85c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions of source code must retain the above copyright 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# notice, this list of conditions and the following disclaimer. 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions in binary form must reproduce the above 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# copyright notice, this list of conditions and the following disclaimer 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# in the documentation and/or other materials provided with the 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# distribution. 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Neither the name of Google Inc. nor the names of its 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# contributors may be used to endorse or promote products derived from 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# this software without specific prior written permission. 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import errno 315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import logging 325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import multiprocessing 335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import os 345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import StringIO 355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import signal 365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import subprocess 375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import sys 385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import time 395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 40926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.common.system.outputtee import Tee 415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.common.system.filesystem import FileSystem 425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)_log = logging.getLogger(__name__) 455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class ScriptError(Exception): 485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __init__(self, 505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) message=None, 515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) script_args=None, 525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) exit_code=None, 535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) output=None, 54e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch cwd=None, 55e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch output_limit=500): 56e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch shortened_output = output 57e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch if output and output_limit and len(output) > output_limit: 58e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch shortened_output = "Last %s characters of output:\n%s" % (output_limit, output[-output_limit:]) 59e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not message: 61926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) message = 'Failed to run "%s"' % repr(script_args) 625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if exit_code: 635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) message += " exit_code: %d" % exit_code 645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if cwd: 655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) message += " cwd: %s" % cwd 665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 67e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch if shortened_output: 68e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch message += "\n\noutput: %s" % shortened_output 69e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Exception.__init__(self, message) 715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.script_args = script_args # 'args' is already used by Exception 725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.exit_code = exit_code 735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.output = output 745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.cwd = cwd 755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 76e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def message_with_output(self): 775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return unicode(self) 785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def command_name(self): 805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) command_path = self.script_args 815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if type(command_path) is list: 825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) command_path = command_path[0] 835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return os.path.basename(command_path) 845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class Executive(object): 875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) PIPE = subprocess.PIPE 885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) STDOUT = subprocess.STDOUT 895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _should_close_fds(self): 915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # We need to pass close_fds=True to work around Python bug #2320 925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # (otherwise we can hang when we kill DumpRenderTree when we are running 935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # multiple threads). See http://bugs.python.org/issue2320 . 945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Note that close_fds isn't supported on Windows, but this bug only 955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # shows up on Mac and Linux. 965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return sys.platform not in ('win32', 'cygwin') 975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _run_command_with_teed_output(self, args, teed_output, **kwargs): 995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) child_process = self.popen(args, 1005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) stdout=self.PIPE, 1015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) stderr=self.STDOUT, 1025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) close_fds=self._should_close_fds(), 1035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) **kwargs) 1045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Use our own custom wait loop because Popen ignores a tee'd 1065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # stderr/stdout. 1075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: This could be improved not to flatten output to stdout. 1085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while True: 1095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) output_line = child_process.stdout.readline() 1105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if output_line == "" and child_process.poll() != None: 1115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # poll() is not threadsafe and can throw OSError due to: 1125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # http://bugs.python.org/issue1731717 1135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return child_process.poll() 1145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # We assume that the child process wrote to us in utf-8, 1155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # so no re-encoding is necessary before writing here. 1165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) teed_output.write(output_line) 1175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: Remove this deprecated method and move callers to run_command. 1195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: This method is a hack to allow running command which both 1205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # capture their output and print out to stdin. Useful for things 1215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # like "build-webkit" where we want to display to the user that we're building 1225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # but still have the output to stuff into a log file. 1235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def run_and_throw_if_fail(self, args, quiet=False, decode_output=True, **kwargs): 1245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Cache the child's output locally so it can be used for error reports. 1255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) child_out_file = StringIO.StringIO() 1265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tee_stdout = sys.stdout 1275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if quiet: 1285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) dev_null = open(os.devnull, "w") # FIXME: Does this need an encoding? 1295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tee_stdout = dev_null 130926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) child_stdout = Tee(child_out_file, tee_stdout) 1315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) exit_code = self._run_command_with_teed_output(args, child_stdout, **kwargs) 1325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if quiet: 1335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) dev_null.close() 1345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) child_output = child_out_file.getvalue() 1365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) child_out_file.close() 1375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if decode_output: 1395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) child_output = child_output.decode(self._child_process_encoding()) 1405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if exit_code: 1425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) raise ScriptError(script_args=args, 1435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) exit_code=exit_code, 1445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) output=child_output) 1455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return child_output 1465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def cpu_count(self): 1485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return multiprocessing.cpu_count() 1495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) @staticmethod 1515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def interpreter_for_script(script_path, fs=None): 1525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) fs = fs or FileSystem() 1535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) lines = fs.read_text_file(script_path).splitlines() 1545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not len(lines): 1555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return None 1565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) first_line = lines[0] 1575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not first_line.startswith('#!'): 1585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return None 1595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if first_line.find('python') > -1: 1605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return sys.executable 1615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if first_line.find('perl') > -1: 1625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 'perl' 1635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if first_line.find('ruby') > -1: 1645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 'ruby' 1655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return None 1665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) @staticmethod 1685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def shell_command_for_script(script_path, fs=None): 1695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) fs = fs or FileSystem() 1705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Win32 does not support shebang. We need to detect the interpreter ourself. 1715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if sys.platform == 'win32': 1725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) interpreter = Executive.interpreter_for_script(script_path, fs) 1735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if interpreter: 1745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return [interpreter, script_path] 1755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return [script_path] 1765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def kill_process(self, pid): 1785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Attempts to kill the given pid. 1795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Will fail silently if pid does not exist or insufficient permisssions.""" 1805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if sys.platform == "win32": 1815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # We only use taskkill.exe on windows (not cygwin) because subprocess.pid 1825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # is a CYGWIN pid and taskkill.exe expects a windows pid. 1835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Thankfully os.kill on CYGWIN handles either pid type. 1845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) command = ["taskkill.exe", "/f", "/pid", pid] 1855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # taskkill will exit 128 if the process is not found. We should log. 1865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.run_command(command, error_handler=self.ignore_error) 1875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 1885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # According to http://docs.python.org/library/os.html 1905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # os.kill isn't available on Windows. python 2.5.5 os.kill appears 1915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # to work in cygwin, however it occasionally raises EAGAIN. 1925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) retries_left = 10 if sys.platform == "cygwin" else 1 1935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while retries_left > 0: 1945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) try: 1955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) retries_left -= 1 1965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) os.kill(pid, signal.SIGKILL) 1975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) _ = os.waitpid(pid, os.WNOHANG) 1985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) except OSError, e: 1995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if e.errno == errno.EAGAIN: 2005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if retries_left <= 0: 2015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) _log.warn("Failed to kill pid %s. Too many EAGAIN errors." % pid) 2025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) continue 2035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if e.errno == errno.ESRCH: # The process does not exist. 2045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 2055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if e.errno == errno.EPIPE: # The process has exited already on cygwin 2065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 2075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if e.errno == errno.ECHILD: 2085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Can't wait on a non-child process, but the kill worked. 2095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 2105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if e.errno == errno.EACCES and sys.platform == 'cygwin': 2115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Cygwin python sometimes can't kill native processes. 2125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 2135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) raise 2145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _win32_check_running_pid(self, pid): 2165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # importing ctypes at the top-level seems to cause weird crashes at 2175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # exit under cygwin on apple's win port. Only win32 needs cygwin, so 2185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # we import it here instead. See https://bugs.webkit.org/show_bug.cgi?id=91682 2195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) import ctypes 2205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) class PROCESSENTRY32(ctypes.Structure): 2225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) _fields_ = [("dwSize", ctypes.c_ulong), 2235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ("cntUsage", ctypes.c_ulong), 2245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ("th32ProcessID", ctypes.c_ulong), 2255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ("th32DefaultHeapID", ctypes.POINTER(ctypes.c_ulong)), 2265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ("th32ModuleID", ctypes.c_ulong), 2275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ("cntThreads", ctypes.c_ulong), 2285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ("th32ParentProcessID", ctypes.c_ulong), 2295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ("pcPriClassBase", ctypes.c_ulong), 2305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ("dwFlags", ctypes.c_ulong), 2315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ("szExeFile", ctypes.c_char * 260)] 2325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot 2345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Process32First = ctypes.windll.kernel32.Process32First 2355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Process32Next = ctypes.windll.kernel32.Process32Next 2365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CloseHandle = ctypes.windll.kernel32.CloseHandle 2375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) TH32CS_SNAPPROCESS = 0x00000002 # win32 magic number 2385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) 2395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pe32 = PROCESSENTRY32() 2405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pe32.dwSize = ctypes.sizeof(PROCESSENTRY32) 2415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) result = False 2425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not Process32First(hProcessSnap, ctypes.byref(pe32)): 2435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) _log.debug("Failed getting first process.") 2445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CloseHandle(hProcessSnap) 2455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result 2465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while True: 2475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if pe32.th32ProcessID == pid: 2485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) result = True 2495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) break 2505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not Process32Next(hProcessSnap, ctypes.byref(pe32)): 2515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) break 2525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) CloseHandle(hProcessSnap) 2535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return result 2545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def check_running_pid(self, pid): 2565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Return True if pid is alive, otherwise return False.""" 2575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if sys.platform == 'win32': 2585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self._win32_check_running_pid(pid) 2595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) try: 2615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) os.kill(pid, 0) 2625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return True 2635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) except OSError: 2645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return False 2655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def running_pids(self, process_name_filter=None): 2675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not process_name_filter: 2685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) process_name_filter = lambda process_name: True 2695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) running_pids = [] 2715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if sys.platform in ("win32", "cygwin"): 2735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: running_pids isn't implemented on Windows yet... 2745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return [] 2755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ps_process = self.popen(['ps', '-eo', 'pid,comm'], stdout=self.PIPE, stderr=self.PIPE) 2775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) stdout, _ = ps_process.communicate() 2785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for line in stdout.splitlines(): 2795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) try: 2805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # In some cases the line can contain one or more 2815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # leading white-spaces, so strip it before split. 2825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pid, process_name = line.strip().split(' ', 1) 2835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if process_name_filter(process_name): 2845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) running_pids.append(int(pid)) 2855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) except ValueError, e: 2865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 2875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return sorted(running_pids) 2895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def wait_newest(self, process_name_filter=None): 2915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not process_name_filter: 2925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) process_name_filter = lambda process_name: True 2935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) running_pids = self.running_pids(process_name_filter) 2955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not running_pids: 2965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 2975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pid = running_pids[-1] 2985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) while self.check_running_pid(pid): 3005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) time.sleep(0.25) 3015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 302926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def wait_limited(self, pid, limit_in_seconds=None, check_frequency_in_seconds=None): 303926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) seconds_left = limit_in_seconds or 10 304926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) sleep_length = check_frequency_in_seconds or 1 305926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) while seconds_left > 0 and self.check_running_pid(pid): 306926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) seconds_left -= sleep_length 307926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) time.sleep(sleep_length) 308926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 3095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _windows_image_name(self, process_name): 3105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) name, extension = os.path.splitext(process_name) 3115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not extension: 3125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # taskkill expects processes to end in .exe 3135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # If necessary we could add a flag to disable appending .exe. 3145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) process_name = "%s.exe" % name 3155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return process_name 3165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 317926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def interrupt(self, pid): 318926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) interrupt_signal = signal.SIGINT 319926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # FIXME: The python docs seem to imply that platform == 'win32' may need to use signal.CTRL_C_EVENT 320926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # http://docs.python.org/2/library/signal.html 321926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) try: 322926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) os.kill(pid, interrupt_signal) 323926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) except OSError: 324926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # Silently ignore when the pid doesn't exist. 325926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # It's impossible for callers to avoid race conditions with process shutdown. 326926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) pass 327926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 3285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def kill_all(self, process_name): 3295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Attempts to kill processes matching process_name. 3305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Will fail silently if no process are found.""" 3315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if sys.platform in ("win32", "cygwin"): 3325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) image_name = self._windows_image_name(process_name) 3335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) command = ["taskkill.exe", "/f", "/im", image_name] 3345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # taskkill will exit 128 if the process is not found. We should log. 3355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.run_command(command, error_handler=self.ignore_error) 3365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 3375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: This is inconsistent that kill_all uses TERM and kill_process 3395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # uses KILL. Windows is always using /f (which seems like -KILL). 3405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # We should pick one mode, or add support for switching between them. 3415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Note: Mac OS X 10.6 requires -SIGNALNAME before -u USER 3425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) command = ["killall", "-TERM", "-u", os.getenv("USER"), process_name] 3435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # killall returns 1 if no process can be found and 2 on command error. 3445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: We should pass a custom error_handler to allow only exit_code 1. 3455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # We should log in exit_code == 1 3465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.run_command(command, error_handler=self.ignore_error) 3475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Error handlers do not need to be static methods once all callers are 3495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # updated to use an Executive object. 3505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) @staticmethod 3525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def default_error_handler(error): 3535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) raise error 3545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) @staticmethod 3565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def ignore_error(error): 3575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 3585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _compute_stdin(self, input): 3605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Returns (stdin, string_to_communicate)""" 3615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: We should be returning /dev/null for stdin 3625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # or closing stdin after process creation to prevent 3635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # child processes from getting input from the user. 3645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not input: 3655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return (None, None) 3665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if hasattr(input, "read"): # Check if the input is a file. 3675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return (input, None) # Assume the file is in the right encoding. 3685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Popen in Python 2.5 and before does not automatically encode unicode objects. 3705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # http://bugs.python.org/issue5290 3715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # See https://bugs.webkit.org/show_bug.cgi?id=37528 3725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # for an example of a regresion caused by passing a unicode string directly. 3735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: We may need to encode differently on different platforms. 3745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if isinstance(input, unicode): 3755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) input = input.encode(self._child_process_encoding()) 3765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return (self.PIPE, input) 3775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 378926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def command_for_printing(self, args): 3795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Returns a print-ready string representing command args. 3805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) The string should be copy/paste ready for execution in a shell.""" 381926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) args = self._stringify_args(args) 3825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) escaped_args = [] 3835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for arg in args: 3845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if isinstance(arg, unicode): 3855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Escape any non-ascii characters for easy copy/paste 3865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) arg = arg.encode("unicode_escape") 3875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: Do we need to fix quotes here? 3885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) escaped_args.append(arg) 3895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return " ".join(escaped_args) 3905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: run_and_throw_if_fail should be merged into this method. 3925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def run_command(self, 3935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) args, 3945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) cwd=None, 3955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) env=None, 3965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) input=None, 3975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) error_handler=None, 3985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return_exit_code=False, 3995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return_stderr=True, 400e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) decode_output=True, debug_logging=True): 4015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Popen wrapper for convenience and to work around python bugs.""" 4025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) assert(isinstance(args, list) or isinstance(args, tuple)) 4035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) start_time = time.time() 4045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) stdin, string_to_communicate = self._compute_stdin(input) 4065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) stderr = self.STDOUT if return_stderr else None 4075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) process = self.popen(args, 4095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) stdin=stdin, 4105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) stdout=self.PIPE, 4115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) stderr=stderr, 4125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) cwd=cwd, 4135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) env=env, 4145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) close_fds=self._should_close_fds()) 4155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) output = process.communicate(string_to_communicate)[0] 4165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # run_command automatically decodes to unicode() unless explicitly told not to. 4185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if decode_output: 4195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) output = output.decode(self._child_process_encoding()) 4205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # wait() is not threadsafe and can throw OSError due to: 4225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # http://bugs.python.org/issue1731717 4235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) exit_code = process.wait() 4245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 425e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) if debug_logging: 426e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) _log.debug('"%s" took %.2fs' % (self.command_for_printing(args), time.time() - start_time)) 4275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if return_exit_code: 4295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return exit_code 4305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if exit_code: 4325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) script_error = ScriptError(script_args=args, 4335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) exit_code=exit_code, 4345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) output=output, 4355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) cwd=cwd) 4365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) (error_handler or self.default_error_handler)(script_error) 4375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return output 4385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _child_process_encoding(self): 4405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW 4415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # to launch subprocesses, so we have to encode arguments using the 4425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # current code page. 4435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if sys.platform == 'win32' and sys.version < '3': 4445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 'mbcs' 4455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # All other platforms use UTF-8. 4465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands 4475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # which will expect arguments to be encoded using the current code 4485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # page. 4495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 'utf-8' 4505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _should_encode_child_process_arguments(self): 4525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Cygwin's Python's os.execv doesn't support unicode command 4535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # arguments, and neither does Cygwin's execv itself. 4545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if sys.platform == 'cygwin': 4555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return True 4565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW 4585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # to launch subprocesses, so we have to encode arguments using the 4595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # current code page. 4605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if sys.platform == 'win32' and sys.version < '3': 4615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return True 4625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return False 4645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _encode_argument_if_needed(self, argument): 4665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not self._should_encode_child_process_arguments(): 4675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return argument 4685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return argument.encode(self._child_process_encoding()) 4695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 470926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def _stringify_args(self, args): 471926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # Popen will throw an exception if args are non-strings (like int()) 472926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) string_args = map(unicode, args) 473926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # The Windows implementation of Popen cannot handle unicode strings. :( 474926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return map(self._encode_argument_if_needed, string_args) 475926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 476926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # The only required arugment to popen is named "args", the rest are optional keyword arguments. 477926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def popen(self, args, **kwargs): 478926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # FIXME: We should always be stringifying the args, but callers who pass shell=True 479926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # expect that the exact bytes passed will get passed to the shell (even if they're wrongly encoded). 480926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # shell=True is wrong for many other reasons, and we should remove this 481926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # hack as soon as we can fix all callers to not use shell=True. 482926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if kwargs.get('shell') == True: 483926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) string_args = args 484926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) else: 485926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) string_args = self._stringify_args(args) 486926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return subprocess.Popen(string_args, **kwargs) 4875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 48883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch def call(self, args, **kwargs): 48983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch return subprocess.call(self._stringify_args(args), **kwargs) 49083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 4915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def run_in_parallel(self, command_lines_and_cwds, processes=None): 4925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Runs a list of (cmd_line list, cwd string) tuples in parallel and returns a list of (retcode, stdout, stderr) tuples.""" 4935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) assert len(command_lines_and_cwds) 4945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if sys.platform in ('cygwin', 'win32'): 4965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return map(_run_command_thunk, command_lines_and_cwds) 4975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pool = multiprocessing.Pool(processes=processes) 4985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) results = pool.map(_run_command_thunk, command_lines_and_cwds) 4995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pool.close() 5005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pool.join() 5015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return results 5025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)def _run_command_thunk(cmd_line_and_cwd): 5055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Note that this needs to be a bare module (and hence Picklable) method to work with multiprocessing.Pool. 5065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) (cmd_line, cwd) = cmd_line_and_cwd 5075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) proc = subprocess.Popen(cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 5085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) stdout, stderr = proc.communicate() 5095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return (proc.returncode, stdout, stderr) 510