1# Copyright (c) 2009, Google Inc. All rights reserved.
2# Copyright (c) 2009 Apple Inc. All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#     * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30try:
31    # This API exists only in Python 2.6 and higher.  :(
32    import multiprocessing
33except ImportError:
34    multiprocessing = None
35
36import ctypes
37import errno
38import logging
39import os
40import platform
41import StringIO
42import signal
43import subprocess
44import sys
45import time
46
47from webkitpy.common.system.deprecated_logging import tee
48from webkitpy.common.system.filesystem import FileSystem
49from webkitpy.python24 import versioning
50
51
52_log = logging.getLogger("webkitpy.common.system")
53
54
55class ScriptError(Exception):
56
57    # This is a custom List.__str__ implementation to allow size limiting.
58    def _string_from_args(self, args, limit=100):
59        args_string = unicode(args)
60        # We could make this much fancier, but for now this is OK.
61        if len(args_string) > limit:
62            return args_string[:limit - 3] + "..."
63        return args_string
64
65    def __init__(self,
66                 message=None,
67                 script_args=None,
68                 exit_code=None,
69                 output=None,
70                 cwd=None):
71        if not message:
72            message = 'Failed to run "%s"' % self._string_from_args(script_args)
73            if exit_code:
74                message += " exit_code: %d" % exit_code
75            if cwd:
76                message += " cwd: %s" % cwd
77
78        Exception.__init__(self, message)
79        self.script_args = script_args # 'args' is already used by Exception
80        self.exit_code = exit_code
81        self.output = output
82        self.cwd = cwd
83
84    def message_with_output(self, output_limit=500):
85        if self.output:
86            if output_limit and len(self.output) > output_limit:
87                return u"%s\n\nLast %s characters of output:\n%s" % \
88                    (self, output_limit, self.output[-output_limit:])
89            return u"%s\n\n%s" % (self, self.output)
90        return unicode(self)
91
92    def command_name(self):
93        command_path = self.script_args
94        if type(command_path) is list:
95            command_path = command_path[0]
96        return os.path.basename(command_path)
97
98
99def run_command(*args, **kwargs):
100    # FIXME: This should not be a global static.
101    # New code should use Executive.run_command directly instead
102    return Executive().run_command(*args, **kwargs)
103
104
105class Executive(object):
106
107    def _should_close_fds(self):
108        # We need to pass close_fds=True to work around Python bug #2320
109        # (otherwise we can hang when we kill DumpRenderTree when we are running
110        # multiple threads). See http://bugs.python.org/issue2320 .
111        # Note that close_fds isn't supported on Windows, but this bug only
112        # shows up on Mac and Linux.
113        return sys.platform not in ('win32', 'cygwin')
114
115    def _run_command_with_teed_output(self, args, teed_output):
116        args = map(unicode, args)  # Popen will throw an exception if args are non-strings (like int())
117        args = map(self._encode_argument_if_needed, args)
118
119        child_process = subprocess.Popen(args,
120                                         stdout=subprocess.PIPE,
121                                         stderr=subprocess.STDOUT,
122                                         close_fds=self._should_close_fds())
123
124        # Use our own custom wait loop because Popen ignores a tee'd
125        # stderr/stdout.
126        # FIXME: This could be improved not to flatten output to stdout.
127        while True:
128            output_line = child_process.stdout.readline()
129            if output_line == "" and child_process.poll() != None:
130                # poll() is not threadsafe and can throw OSError due to:
131                # http://bugs.python.org/issue1731717
132                return child_process.poll()
133            # We assume that the child process wrote to us in utf-8,
134            # so no re-encoding is necessary before writing here.
135            teed_output.write(output_line)
136
137    # FIXME: Remove this deprecated method and move callers to run_command.
138    # FIXME: This method is a hack to allow running command which both
139    # capture their output and print out to stdin.  Useful for things
140    # like "build-webkit" where we want to display to the user that we're building
141    # but still have the output to stuff into a log file.
142    def run_and_throw_if_fail(self, args, quiet=False, decode_output=True):
143        # Cache the child's output locally so it can be used for error reports.
144        child_out_file = StringIO.StringIO()
145        tee_stdout = sys.stdout
146        if quiet:
147            dev_null = open(os.devnull, "w")  # FIXME: Does this need an encoding?
148            tee_stdout = dev_null
149        child_stdout = tee(child_out_file, tee_stdout)
150        exit_code = self._run_command_with_teed_output(args, child_stdout)
151        if quiet:
152            dev_null.close()
153
154        child_output = child_out_file.getvalue()
155        child_out_file.close()
156
157        if decode_output:
158            child_output = child_output.decode(self._child_process_encoding())
159
160        if exit_code:
161            raise ScriptError(script_args=args,
162                              exit_code=exit_code,
163                              output=child_output)
164        return child_output
165
166    def cpu_count(self):
167        if multiprocessing:
168            return multiprocessing.cpu_count()
169        # Darn.  We don't have the multiprocessing package.
170        system_name = platform.system()
171        if system_name == "Darwin":
172            return int(self.run_command(["sysctl", "-n", "hw.ncpu"]))
173        elif system_name == "Windows":
174            return int(os.environ.get('NUMBER_OF_PROCESSORS', 1))
175        elif system_name == "Linux":
176            num_cores = os.sysconf("SC_NPROCESSORS_ONLN")
177            if isinstance(num_cores, int) and num_cores > 0:
178                return num_cores
179        # This quantity is a lie but probably a reasonable guess for modern
180        # machines.
181        return 2
182
183    @staticmethod
184    def interpreter_for_script(script_path, fs=FileSystem()):
185        lines = fs.read_text_file(script_path).splitlines()
186        if not len(lines):
187            return None
188        first_line = lines[0]
189        if not first_line.startswith('#!'):
190            return None
191        if first_line.find('python') > -1:
192            return sys.executable
193        if first_line.find('perl') > -1:
194            return 'perl'
195        if first_line.find('ruby') > -1:
196            return 'ruby'
197        return None
198
199    def kill_process(self, pid):
200        """Attempts to kill the given pid.
201        Will fail silently if pid does not exist or insufficient permisssions."""
202        if sys.platform == "win32":
203            # We only use taskkill.exe on windows (not cygwin) because subprocess.pid
204            # is a CYGWIN pid and taskkill.exe expects a windows pid.
205            # Thankfully os.kill on CYGWIN handles either pid type.
206            command = ["taskkill.exe", "/f", "/pid", pid]
207            # taskkill will exit 128 if the process is not found.  We should log.
208            self.run_command(command, error_handler=self.ignore_error)
209            return
210
211        # According to http://docs.python.org/library/os.html
212        # os.kill isn't available on Windows. python 2.5.5 os.kill appears
213        # to work in cygwin, however it occasionally raises EAGAIN.
214        retries_left = 10 if sys.platform == "cygwin" else 1
215        while retries_left > 0:
216            try:
217                retries_left -= 1
218                os.kill(pid, signal.SIGKILL)
219            except OSError, e:
220                if e.errno == errno.EAGAIN:
221                    if retries_left <= 0:
222                        _log.warn("Failed to kill pid %s.  Too many EAGAIN errors." % pid)
223                    continue
224                if e.errno == errno.ESRCH:  # The process does not exist.
225                    _log.warn("Called kill_process with a non-existant pid %s" % pid)
226                    return
227                raise
228
229    def _win32_check_running_pid(self, pid):
230
231        class PROCESSENTRY32(ctypes.Structure):
232            _fields_ = [("dwSize", ctypes.c_ulong),
233                        ("cntUsage", ctypes.c_ulong),
234                        ("th32ProcessID", ctypes.c_ulong),
235                        ("th32DefaultHeapID", ctypes.c_ulong),
236                        ("th32ModuleID", ctypes.c_ulong),
237                        ("cntThreads", ctypes.c_ulong),
238                        ("th32ParentProcessID", ctypes.c_ulong),
239                        ("pcPriClassBase", ctypes.c_ulong),
240                        ("dwFlags", ctypes.c_ulong),
241                        ("szExeFile", ctypes.c_char * 260)]
242
243        CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot
244        Process32First = ctypes.windll.kernel32.Process32First
245        Process32Next = ctypes.windll.kernel32.Process32Next
246        CloseHandle = ctypes.windll.kernel32.CloseHandle
247        TH32CS_SNAPPROCESS = 0x00000002  # win32 magic number
248        hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0)
249        pe32 = PROCESSENTRY32()
250        pe32.dwSize = ctypes.sizeof(PROCESSENTRY32)
251        result = False
252        if not Process32First(hProcessSnap, ctypes.byref(pe32)):
253            _log.debug("Failed getting first process.")
254            CloseHandle(hProcessSnap)
255            return result
256        while True:
257            if pe32.th32ProcessID == pid:
258                result = True
259                break
260            if not Process32Next(hProcessSnap, ctypes.byref(pe32)):
261                break
262        CloseHandle(hProcessSnap)
263        return result
264
265    def check_running_pid(self, pid):
266        """Return True if pid is alive, otherwise return False."""
267        if sys.platform in ('darwin', 'linux2', 'cygwin'):
268            try:
269                os.kill(pid, 0)
270                return True
271            except OSError:
272                return False
273        elif sys.platform == 'win32':
274            return self._win32_check_running_pid(pid)
275
276        assert(False)
277
278    def _windows_image_name(self, process_name):
279        name, extension = os.path.splitext(process_name)
280        if not extension:
281            # taskkill expects processes to end in .exe
282            # If necessary we could add a flag to disable appending .exe.
283            process_name = "%s.exe" % name
284        return process_name
285
286    def kill_all(self, process_name):
287        """Attempts to kill processes matching process_name.
288        Will fail silently if no process are found."""
289        if sys.platform in ("win32", "cygwin"):
290            image_name = self._windows_image_name(process_name)
291            command = ["taskkill.exe", "/f", "/im", image_name]
292            # taskkill will exit 128 if the process is not found.  We should log.
293            self.run_command(command, error_handler=self.ignore_error)
294            return
295
296        # FIXME: This is inconsistent that kill_all uses TERM and kill_process
297        # uses KILL.  Windows is always using /f (which seems like -KILL).
298        # We should pick one mode, or add support for switching between them.
299        # Note: Mac OS X 10.6 requires -SIGNALNAME before -u USER
300        command = ["killall", "-TERM", "-u", os.getenv("USER"), process_name]
301        # killall returns 1 if no process can be found and 2 on command error.
302        # FIXME: We should pass a custom error_handler to allow only exit_code 1.
303        # We should log in exit_code == 1
304        self.run_command(command, error_handler=self.ignore_error)
305
306    # Error handlers do not need to be static methods once all callers are
307    # updated to use an Executive object.
308
309    @staticmethod
310    def default_error_handler(error):
311        raise error
312
313    @staticmethod
314    def ignore_error(error):
315        pass
316
317    def _compute_stdin(self, input):
318        """Returns (stdin, string_to_communicate)"""
319        # FIXME: We should be returning /dev/null for stdin
320        # or closing stdin after process creation to prevent
321        # child processes from getting input from the user.
322        if not input:
323            return (None, None)
324        if hasattr(input, "read"):  # Check if the input is a file.
325            return (input, None)  # Assume the file is in the right encoding.
326
327        # Popen in Python 2.5 and before does not automatically encode unicode objects.
328        # http://bugs.python.org/issue5290
329        # See https://bugs.webkit.org/show_bug.cgi?id=37528
330        # for an example of a regresion caused by passing a unicode string directly.
331        # FIXME: We may need to encode differently on different platforms.
332        if isinstance(input, unicode):
333            input = input.encode(self._child_process_encoding())
334        return (subprocess.PIPE, input)
335
336    def _command_for_printing(self, args):
337        """Returns a print-ready string representing command args.
338        The string should be copy/paste ready for execution in a shell."""
339        escaped_args = []
340        for arg in args:
341            if isinstance(arg, unicode):
342                # Escape any non-ascii characters for easy copy/paste
343                arg = arg.encode("unicode_escape")
344            # FIXME: Do we need to fix quotes here?
345            escaped_args.append(arg)
346        return " ".join(escaped_args)
347
348    # FIXME: run_and_throw_if_fail should be merged into this method.
349    def run_command(self,
350                    args,
351                    cwd=None,
352                    input=None,
353                    error_handler=None,
354                    return_exit_code=False,
355                    return_stderr=True,
356                    decode_output=True):
357        """Popen wrapper for convenience and to work around python bugs."""
358        assert(isinstance(args, list) or isinstance(args, tuple))
359        start_time = time.time()
360        args = map(unicode, args)  # Popen will throw an exception if args are non-strings (like int())
361        args = map(self._encode_argument_if_needed, args)
362
363        stdin, string_to_communicate = self._compute_stdin(input)
364        stderr = subprocess.STDOUT if return_stderr else None
365
366        process = subprocess.Popen(args,
367                                   stdin=stdin,
368                                   stdout=subprocess.PIPE,
369                                   stderr=stderr,
370                                   cwd=cwd,
371                                   close_fds=self._should_close_fds())
372        output = process.communicate(string_to_communicate)[0]
373
374        # run_command automatically decodes to unicode() unless explicitly told not to.
375        if decode_output:
376            output = output.decode(self._child_process_encoding())
377
378        # wait() is not threadsafe and can throw OSError due to:
379        # http://bugs.python.org/issue1731717
380        exit_code = process.wait()
381
382        _log.debug('"%s" took %.2fs' % (self._command_for_printing(args), time.time() - start_time))
383
384        if return_exit_code:
385            return exit_code
386
387        if exit_code:
388            script_error = ScriptError(script_args=args,
389                                       exit_code=exit_code,
390                                       output=output,
391                                       cwd=cwd)
392            (error_handler or self.default_error_handler)(script_error)
393        return output
394
395    def _child_process_encoding(self):
396        # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW
397        # to launch subprocesses, so we have to encode arguments using the
398        # current code page.
399        if sys.platform == 'win32' and versioning.compare_version(sys, '3.0')[0] < 0:
400            return 'mbcs'
401        # All other platforms use UTF-8.
402        # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands
403        # which will expect arguments to be encoded using the current code
404        # page.
405        return 'utf-8'
406
407    def _should_encode_child_process_arguments(self):
408        # Cygwin's Python's os.execv doesn't support unicode command
409        # arguments, and neither does Cygwin's execv itself.
410        if sys.platform == 'cygwin':
411            return True
412
413        # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW
414        # to launch subprocesses, so we have to encode arguments using the
415        # current code page.
416        if sys.platform == 'win32' and versioning.compare_version(sys, '3.0')[0] < 0:
417            return True
418
419        return False
420
421    def _encode_argument_if_needed(self, argument):
422        if not self._should_encode_child_process_arguments():
423            return argument
424        return argument.encode(self._child_process_encoding())
425