1# Copyright (c) 2009, Google Inc. All rights reserved.
2# Copyright (c) 2009 Apple Inc. All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#     * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30import StringIO
31import errno
32import logging
33import multiprocessing
34import os
35import signal
36import subprocess
37import sys
38import time
39
40from webkitpy.common.system.outputtee import Tee
41from webkitpy.common.system.filesystem import FileSystem
42
43
44_log = logging.getLogger(__name__)
45
46
47class ScriptError(Exception):
48
49    def __init__(self,
50                 message=None,
51                 script_args=None,
52                 exit_code=None,
53                 output=None,
54                 cwd=None,
55                 output_limit=500):
56        shortened_output = output
57        if output and output_limit and len(output) > output_limit:
58            shortened_output = "Last %s characters of output:\n%s" % (output_limit, output[-output_limit:])
59
60        if not message:
61            message = 'Failed to run "%s"' % repr(script_args)
62            if exit_code:
63                message += " exit_code: %d" % exit_code
64            if cwd:
65                message += " cwd: %s" % cwd
66
67        if shortened_output:
68            message += "\n\noutput: %s" % shortened_output
69
70        Exception.__init__(self, message)
71        self.script_args = script_args # 'args' is already used by Exception
72        self.exit_code = exit_code
73        self.output = output
74        self.cwd = cwd
75
76    def message_with_output(self):
77        return unicode(self)
78
79    def command_name(self):
80        command_path = self.script_args
81        if type(command_path) is list:
82            command_path = command_path[0]
83        return os.path.basename(command_path)
84
85
86class Executive(object):
87    PIPE = subprocess.PIPE
88    STDOUT = subprocess.STDOUT
89
90    def _should_close_fds(self):
91        # We need to pass close_fds=True to work around Python bug #2320
92        # (otherwise we can hang when we kill DumpRenderTree when we are running
93        # multiple threads). See http://bugs.python.org/issue2320 .
94        # Note that close_fds isn't supported on Windows, but this bug only
95        # shows up on Mac and Linux.
96        return sys.platform not in ('win32', 'cygwin')
97
98    def _run_command_with_teed_output(self, args, teed_output, **kwargs):
99        child_process = self.popen(args,
100                                   stdout=self.PIPE,
101                                   stderr=self.STDOUT,
102                                   close_fds=self._should_close_fds(),
103                                   **kwargs)
104
105        # Use our own custom wait loop because Popen ignores a tee'd
106        # stderr/stdout.
107        # FIXME: This could be improved not to flatten output to stdout.
108        while True:
109            output_line = child_process.stdout.readline()
110            if output_line == "" and child_process.poll() != None:
111                # poll() is not threadsafe and can throw OSError due to:
112                # http://bugs.python.org/issue1731717
113                return child_process.poll()
114            # We assume that the child process wrote to us in utf-8,
115            # so no re-encoding is necessary before writing here.
116            teed_output.write(output_line)
117
118    # FIXME: Remove this deprecated method and move callers to run_command.
119    # FIXME: This method is a hack to allow running command which both
120    # capture their output and print out to stdin.  Useful for things
121    # like "build-webkit" where we want to display to the user that we're building
122    # but still have the output to stuff into a log file.
123    def run_and_throw_if_fail(self, args, quiet=False, decode_output=True, **kwargs):
124        # Cache the child's output locally so it can be used for error reports.
125        child_out_file = StringIO.StringIO()
126        tee_stdout = sys.stdout
127        if quiet:
128            dev_null = open(os.devnull, "w")  # FIXME: Does this need an encoding?
129            tee_stdout = dev_null
130        child_stdout = Tee(child_out_file, tee_stdout)
131        exit_code = self._run_command_with_teed_output(args, child_stdout, **kwargs)
132        if quiet:
133            dev_null.close()
134
135        child_output = child_out_file.getvalue()
136        child_out_file.close()
137
138        if decode_output:
139            child_output = child_output.decode(self._child_process_encoding())
140
141        if exit_code:
142            raise ScriptError(script_args=args,
143                              exit_code=exit_code,
144                              output=child_output)
145        return child_output
146
147    def cpu_count(self):
148        return multiprocessing.cpu_count()
149
150    @staticmethod
151    def interpreter_for_script(script_path, fs=None):
152        fs = fs or FileSystem()
153        lines = fs.read_text_file(script_path).splitlines()
154        if not len(lines):
155            return None
156        first_line = lines[0]
157        if not first_line.startswith('#!'):
158            return None
159        if first_line.find('python') > -1:
160            return sys.executable
161        if first_line.find('perl') > -1:
162            return 'perl'
163        if first_line.find('ruby') > -1:
164            return 'ruby'
165        return None
166
167    @staticmethod
168    def shell_command_for_script(script_path, fs=None):
169        fs = fs or FileSystem()
170        # Win32 does not support shebang. We need to detect the interpreter ourself.
171        if sys.platform == 'win32':
172            interpreter = Executive.interpreter_for_script(script_path, fs)
173            if interpreter:
174                return [interpreter, script_path]
175        return [script_path]
176
177    def kill_process(self, pid):
178        """Attempts to kill the given pid.
179        Will fail silently if pid does not exist or insufficient permisssions."""
180        if sys.platform == "win32":
181            # We only use taskkill.exe on windows (not cygwin) because subprocess.pid
182            # is a CYGWIN pid and taskkill.exe expects a windows pid.
183            # Thankfully os.kill on CYGWIN handles either pid type.
184            command = ["taskkill.exe", "/f", "/t", "/pid", pid]
185            # taskkill will exit 128 if the process is not found.  We should log.
186            self.run_command(command, error_handler=self.ignore_error)
187            return
188
189        # According to http://docs.python.org/library/os.html
190        # os.kill isn't available on Windows. python 2.5.5 os.kill appears
191        # to work in cygwin, however it occasionally raises EAGAIN.
192        retries_left = 10 if sys.platform == "cygwin" else 1
193        while retries_left > 0:
194            try:
195                retries_left -= 1
196                os.kill(pid, signal.SIGKILL)
197                _ = os.waitpid(pid, os.WNOHANG)
198            except OSError, e:
199                if e.errno == errno.EAGAIN:
200                    if retries_left <= 0:
201                        _log.warn("Failed to kill pid %s.  Too many EAGAIN errors." % pid)
202                    continue
203                if e.errno == errno.ESRCH:  # The process does not exist.
204                    return
205                if e.errno == errno.EPIPE:  # The process has exited already on cygwin
206                    return
207                if e.errno == errno.ECHILD:
208                    # Can't wait on a non-child process, but the kill worked.
209                    return
210                if e.errno == errno.EACCES and sys.platform == 'cygwin':
211                    # Cygwin python sometimes can't kill native processes.
212                    return
213                raise
214
215    def _win32_check_running_pid(self, pid):
216        # importing ctypes at the top-level seems to cause weird crashes at
217        # exit under cygwin on apple's win port. Only win32 needs cygwin, so
218        # we import it here instead. See https://bugs.webkit.org/show_bug.cgi?id=91682
219        import ctypes
220
221        class PROCESSENTRY32(ctypes.Structure):
222            _fields_ = [("dwSize", ctypes.c_ulong),
223                        ("cntUsage", ctypes.c_ulong),
224                        ("th32ProcessID", ctypes.c_ulong),
225                        ("th32DefaultHeapID", ctypes.POINTER(ctypes.c_ulong)),
226                        ("th32ModuleID", ctypes.c_ulong),
227                        ("cntThreads", ctypes.c_ulong),
228                        ("th32ParentProcessID", ctypes.c_ulong),
229                        ("pcPriClassBase", ctypes.c_ulong),
230                        ("dwFlags", ctypes.c_ulong),
231                        ("szExeFile", ctypes.c_char * 260)]
232
233        CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot
234        Process32First = ctypes.windll.kernel32.Process32First
235        Process32Next = ctypes.windll.kernel32.Process32Next
236        CloseHandle = ctypes.windll.kernel32.CloseHandle
237        TH32CS_SNAPPROCESS = 0x00000002  # win32 magic number
238        hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0)
239        pe32 = PROCESSENTRY32()
240        pe32.dwSize = ctypes.sizeof(PROCESSENTRY32)
241        result = False
242        if not Process32First(hProcessSnap, ctypes.byref(pe32)):
243            _log.debug("Failed getting first process.")
244            CloseHandle(hProcessSnap)
245            return result
246        while True:
247            if pe32.th32ProcessID == pid:
248                result = True
249                break
250            if not Process32Next(hProcessSnap, ctypes.byref(pe32)):
251                break
252        CloseHandle(hProcessSnap)
253        return result
254
255    def check_running_pid(self, pid):
256        """Return True if pid is alive, otherwise return False."""
257        if sys.platform == 'win32':
258            return self._win32_check_running_pid(pid)
259
260        try:
261            os.kill(pid, 0)
262            return True
263        except OSError:
264            return False
265
266    def running_pids(self, process_name_filter=None):
267        if not process_name_filter:
268            process_name_filter = lambda process_name: True
269
270        running_pids = []
271
272        if sys.platform in ("win32", "cygwin"):
273            # FIXME: running_pids isn't implemented on Windows yet...
274            return []
275
276        ps_process = self.popen(['ps', '-eo', 'pid,comm'], stdout=self.PIPE, stderr=self.PIPE)
277        stdout, _ = ps_process.communicate()
278        for line in stdout.splitlines():
279            try:
280                # In some cases the line can contain one or more
281                # leading white-spaces, so strip it before split.
282                pid, process_name = line.strip().split(' ', 1)
283                if process_name_filter(process_name):
284                    running_pids.append(int(pid))
285            except ValueError, e:
286                pass
287
288        return sorted(running_pids)
289
290    def wait_newest(self, process_name_filter=None):
291        if not process_name_filter:
292            process_name_filter = lambda process_name: True
293
294        running_pids = self.running_pids(process_name_filter)
295        if not running_pids:
296            return
297        pid = running_pids[-1]
298
299        while self.check_running_pid(pid):
300            time.sleep(0.25)
301
302    def wait_limited(self, pid, limit_in_seconds=None, check_frequency_in_seconds=None):
303        seconds_left = limit_in_seconds or 10
304        sleep_length = check_frequency_in_seconds or 1
305        while seconds_left > 0 and self.check_running_pid(pid):
306            seconds_left -= sleep_length
307            time.sleep(sleep_length)
308
309    def _windows_image_name(self, process_name):
310        name, extension = os.path.splitext(process_name)
311        if not extension:
312            # taskkill expects processes to end in .exe
313            # If necessary we could add a flag to disable appending .exe.
314            process_name = "%s.exe" % name
315        return process_name
316
317    def interrupt(self, pid):
318        interrupt_signal = signal.SIGINT
319        # FIXME: The python docs seem to imply that platform == 'win32' may need to use signal.CTRL_C_EVENT
320        # http://docs.python.org/2/library/signal.html
321        try:
322            os.kill(pid, interrupt_signal)
323        except OSError:
324            # Silently ignore when the pid doesn't exist.
325            # It's impossible for callers to avoid race conditions with process shutdown.
326            pass
327
328    # Error handlers do not need to be static methods once all callers are
329    # updated to use an Executive object.
330
331    @staticmethod
332    def default_error_handler(error):
333        raise error
334
335    @staticmethod
336    def ignore_error(error):
337        pass
338
339    def _compute_stdin(self, input):
340        """Returns (stdin, string_to_communicate)"""
341        # FIXME: We should be returning /dev/null for stdin
342        # or closing stdin after process creation to prevent
343        # child processes from getting input from the user.
344        if not input:
345            return (None, None)
346        if hasattr(input, "read"):  # Check if the input is a file.
347            return (input, None)  # Assume the file is in the right encoding.
348
349        # Popen in Python 2.5 and before does not automatically encode unicode objects.
350        # http://bugs.python.org/issue5290
351        # See https://bugs.webkit.org/show_bug.cgi?id=37528
352        # for an example of a regresion caused by passing a unicode string directly.
353        # FIXME: We may need to encode differently on different platforms.
354        if isinstance(input, unicode):
355            input = input.encode(self._child_process_encoding())
356        return (self.PIPE, input)
357
358    def command_for_printing(self, args):
359        """Returns a print-ready string representing command args.
360        The string should be copy/paste ready for execution in a shell."""
361        args = self._stringify_args(args)
362        escaped_args = []
363        for arg in args:
364            if isinstance(arg, unicode):
365                # Escape any non-ascii characters for easy copy/paste
366                arg = arg.encode("unicode_escape")
367            # FIXME: Do we need to fix quotes here?
368            escaped_args.append(arg)
369        return " ".join(escaped_args)
370
371    # FIXME: run_and_throw_if_fail should be merged into this method.
372    def run_command(self,
373                    args,
374                    cwd=None,
375                    env=None,
376                    input=None,
377                    error_handler=None,
378                    return_exit_code=False,
379                    return_stderr=True,
380                    decode_output=True, debug_logging=True):
381        """Popen wrapper for convenience and to work around python bugs."""
382        assert(isinstance(args, list) or isinstance(args, tuple))
383        start_time = time.time()
384
385        stdin, string_to_communicate = self._compute_stdin(input)
386        stderr = self.STDOUT if return_stderr else None
387
388        process = self.popen(args,
389                             stdin=stdin,
390                             stdout=self.PIPE,
391                             stderr=stderr,
392                             cwd=cwd,
393                             env=env,
394                             close_fds=self._should_close_fds())
395        output = process.communicate(string_to_communicate)[0]
396
397        # run_command automatically decodes to unicode() unless explicitly told not to.
398        if decode_output:
399            output = output.decode(self._child_process_encoding())
400
401        # wait() is not threadsafe and can throw OSError due to:
402        # http://bugs.python.org/issue1731717
403        exit_code = process.wait()
404
405        if debug_logging:
406            _log.debug('"%s" took %.2fs' % (self.command_for_printing(args), time.time() - start_time))
407
408        if return_exit_code:
409            return exit_code
410
411        if exit_code:
412            script_error = ScriptError(script_args=args,
413                                       exit_code=exit_code,
414                                       output=output,
415                                       cwd=cwd)
416            (error_handler or self.default_error_handler)(script_error)
417        return output
418
419    def _child_process_encoding(self):
420        # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW
421        # to launch subprocesses, so we have to encode arguments using the
422        # current code page.
423        if sys.platform == 'win32' and sys.version < '3':
424            return 'mbcs'
425        # All other platforms use UTF-8.
426        # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands
427        # which will expect arguments to be encoded using the current code
428        # page.
429        return 'utf-8'
430
431    def _should_encode_child_process_arguments(self):
432        # Cygwin's Python's os.execv doesn't support unicode command
433        # arguments, and neither does Cygwin's execv itself.
434        if sys.platform == 'cygwin':
435            return True
436
437        # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW
438        # to launch subprocesses, so we have to encode arguments using the
439        # current code page.
440        if sys.platform == 'win32' and sys.version < '3':
441            return True
442
443        return False
444
445    def _encode_argument_if_needed(self, argument):
446        if not self._should_encode_child_process_arguments():
447            return argument
448        return argument.encode(self._child_process_encoding())
449
450    def _stringify_args(self, args):
451        # Popen will throw an exception if args are non-strings (like int())
452        string_args = map(unicode, args)
453        # The Windows implementation of Popen cannot handle unicode strings. :(
454        return map(self._encode_argument_if_needed, string_args)
455
456    # The only required arugment to popen is named "args", the rest are optional keyword arguments.
457    def popen(self, args, **kwargs):
458        # FIXME: We should always be stringifying the args, but callers who pass shell=True
459        # expect that the exact bytes passed will get passed to the shell (even if they're wrongly encoded).
460        # shell=True is wrong for many other reasons, and we should remove this
461        # hack as soon as we can fix all callers to not use shell=True.
462        if kwargs.get('shell') == True:
463            string_args = args
464        else:
465            string_args = self._stringify_args(args)
466        return subprocess.Popen(string_args, **kwargs)
467
468    def call(self, args, **kwargs):
469        return subprocess.call(self._stringify_args(args), **kwargs)
470
471    def run_in_parallel(self, command_lines_and_cwds, processes=None):
472        """Runs a list of (cmd_line list, cwd string) tuples in parallel and returns a list of (retcode, stdout, stderr) tuples."""
473        assert len(command_lines_and_cwds)
474        return self.map(_run_command_thunk, command_lines_and_cwds, processes)
475
476    def map(self, thunk, arglist, processes=None):
477        if sys.platform in ('cygwin', 'win32') or len(arglist) == 1:
478            return map(thunk, arglist)
479        pool = multiprocessing.Pool(processes=(processes or multiprocessing.cpu_count()))
480        try:
481            return pool.map(thunk, arglist)
482        finally:
483            pool.close()
484            pool.join()
485
486
487def _run_command_thunk(cmd_line_and_cwd):
488    # Note that this needs to be a bare module (and hence Picklable) method to work with multiprocessing.Pool.
489    (cmd_line, cwd) = cmd_line_and_cwd
490    proc = subprocess.Popen(cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
491    stdout, stderr = proc.communicate()
492    return (proc.returncode, stdout, stderr)
493