14adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 24adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# Module providing various facilities to other parts of the package 34adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 44adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# multiprocessing/util.py 54adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 64adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# Copyright (c) 2006-2008, R Oudkerk 74adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# All rights reserved. 84adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 94adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# Redistribution and use in source and binary forms, with or without 104adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# modification, are permitted provided that the following conditions 114adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# are met: 124adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 134adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 1. Redistributions of source code must retain the above copyright 144adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# notice, this list of conditions and the following disclaimer. 154adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 2. Redistributions in binary form must reproduce the above copyright 164adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# notice, this list of conditions and the following disclaimer in the 174adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# documentation and/or other materials provided with the distribution. 184adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 3. Neither the name of author nor the names of any contributors may be 194adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# used to endorse or promote products derived from this software 204adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# without specific prior written permission. 214adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 224adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND 234adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 244adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 254adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 264adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 274adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 284adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 294adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 304adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 314adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 324adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# SUCH DAMAGE. 334adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 344adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 354adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaoimport itertools 364adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaoimport weakref 374adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaoimport atexit 384adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaoimport threading # we want threading to install it's 394adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # cleanup function before multiprocessing does 404adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaofrom subprocess import _args_from_interpreter_flags 414adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 424adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaofrom multiprocessing.process import current_process, active_children 434adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 444adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao__all__ = [ 454adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger', 464adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 'log_to_stderr', 'get_temp_dir', 'register_after_fork', 474adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal', 484adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 'SUBDEBUG', 'SUBWARNING', 494adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ] 504adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 514adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 524adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# Logging 534adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 544adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 554adfde8bc82dd39f59e0445588c3e599ada477dJosh GaoNOTSET = 0 564adfde8bc82dd39f59e0445588c3e599ada477dJosh GaoSUBDEBUG = 5 574adfde8bc82dd39f59e0445588c3e599ada477dJosh GaoDEBUG = 10 584adfde8bc82dd39f59e0445588c3e599ada477dJosh GaoINFO = 20 594adfde8bc82dd39f59e0445588c3e599ada477dJosh GaoSUBWARNING = 25 604adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 614adfde8bc82dd39f59e0445588c3e599ada477dJosh GaoLOGGER_NAME = 'multiprocessing' 624adfde8bc82dd39f59e0445588c3e599ada477dJosh GaoDEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s' 634adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 644adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao_logger = None 654adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao_log_to_stderr = False 664adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 674adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaodef sub_debug(msg, *args): 684adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if _logger: 694adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao _logger.log(SUBDEBUG, msg, *args) 704adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 714adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaodef debug(msg, *args): 724adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if _logger: 734adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao _logger.log(DEBUG, msg, *args) 744adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 754adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaodef info(msg, *args): 764adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if _logger: 774adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao _logger.log(INFO, msg, *args) 784adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 794adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaodef sub_warning(msg, *args): 804adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if _logger: 814adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao _logger.log(SUBWARNING, msg, *args) 824adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 834adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaodef get_logger(): 844adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 854adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao Returns logger used by multiprocessing 864adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 874adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao global _logger 884adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao import logging, atexit 894adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 904adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao logging._acquireLock() 914adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao try: 924adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if not _logger: 934adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 944adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao _logger = logging.getLogger(LOGGER_NAME) 954adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao _logger.propagate = 0 964adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao logging.addLevelName(SUBDEBUG, 'SUBDEBUG') 974adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao logging.addLevelName(SUBWARNING, 'SUBWARNING') 984adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 994adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # XXX multiprocessing should cleanup before logging 1004adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if hasattr(atexit, 'unregister'): 1014adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao atexit.unregister(_exit_function) 1024adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao atexit.register(_exit_function) 1034adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao else: 1044adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao atexit._exithandlers.remove((_exit_function, (), {})) 1054adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao atexit._exithandlers.append((_exit_function, (), {})) 1064adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1074adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao finally: 1084adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao logging._releaseLock() 1094adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1104adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao return _logger 1114adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1124adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaodef log_to_stderr(level=None): 1134adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 1144adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao Turn on logging and add a handler which prints to stderr 1154adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 1164adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao global _log_to_stderr 1174adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao import logging 1184adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1194adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao logger = get_logger() 1204adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) 1214adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao handler = logging.StreamHandler() 1224adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao handler.setFormatter(formatter) 1234adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao logger.addHandler(handler) 1244adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1254adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if level: 1264adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao logger.setLevel(level) 1274adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao _log_to_stderr = True 1284adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao return _logger 1294adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1304adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 1314adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# Function returning a temp directory which will be removed on exit 1324adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 1334adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1344adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaodef get_temp_dir(): 1354adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # get name of a temp directory which will be automatically cleaned up 1364adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if current_process()._tempdir is None: 1374adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao import shutil, tempfile 1384adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao tempdir = tempfile.mkdtemp(prefix='pymp-') 1394adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao info('created temp directory %s', tempdir) 1404adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100) 1414adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao current_process()._tempdir = tempdir 1424adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao return current_process()._tempdir 1434adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1444adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 1454adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# Support for reinitialization of objects when bootstrapping a child process 1464adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 1474adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1484adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao_afterfork_registry = weakref.WeakValueDictionary() 1494adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao_afterfork_counter = itertools.count() 1504adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1514adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaodef _run_after_forkers(): 1524adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao items = list(_afterfork_registry.items()) 1534adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao items.sort() 1544adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao for (index, ident, func), obj in items: 1554adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao try: 1564adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao func(obj) 1574adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao except Exception, e: 1584adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao info('after forker raised exception %s', e) 1594adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1604adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaodef register_after_fork(obj, func): 1614adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao _afterfork_registry[(_afterfork_counter.next(), id(obj), func)] = obj 1624adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1634adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 1644adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# Finalization using weakrefs 1654adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 1664adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1674adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao_finalizer_registry = {} 1684adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao_finalizer_counter = itertools.count() 1694adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1704adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1714adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaoclass Finalize(object): 1724adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 1734adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao Class which supports object finalization using weakrefs 1744adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 1754adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None): 1764adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao assert exitpriority is None or type(exitpriority) is int 1774adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1784adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if obj is not None: 1794adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self._weakref = weakref.ref(obj, self) 1804adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao else: 1814adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao assert exitpriority is not None 1824adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1834adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self._callback = callback 1844adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self._args = args 1854adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self._kwargs = kwargs or {} 1864adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self._key = (exitpriority, _finalizer_counter.next()) 1874adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1884adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao _finalizer_registry[self._key] = self 1894adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 1904adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao def __call__(self, wr=None): 1914adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 1924adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao Run the callback unless it has already been called or cancelled 1934adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 1944adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao try: 1954adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao del _finalizer_registry[self._key] 1964adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao except KeyError: 1974adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao sub_debug('finalizer no longer registered') 1984adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao else: 1994adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao sub_debug('finalizer calling %s with args %s and kwargs %s', 2004adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self._callback, self._args, self._kwargs) 2014adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao res = self._callback(*self._args, **self._kwargs) 2024adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self._weakref = self._callback = self._args = \ 2034adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self._kwargs = self._key = None 2044adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao return res 2054adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2064adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao def cancel(self): 2074adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 2084adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao Cancel finalization of the object 2094adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 2104adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao try: 2114adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao del _finalizer_registry[self._key] 2124adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao except KeyError: 2134adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao pass 2144adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao else: 2154adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self._weakref = self._callback = self._args = \ 2164adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self._kwargs = self._key = None 2174adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2184adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao def still_active(self): 2194adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 2204adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao Return whether this finalizer is still waiting to invoke callback 2214adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 2224adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao return self._key in _finalizer_registry 2234adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2244adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao def __repr__(self): 2254adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao try: 2264adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao obj = self._weakref() 2274adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao except (AttributeError, TypeError): 2284adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao obj = None 2294adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2304adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if obj is None: 2314adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao return '<Finalize object, dead>' 2324adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2334adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao x = '<Finalize object, callback=%s' % \ 2344adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao getattr(self._callback, '__name__', self._callback) 2354adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if self._args: 2364adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao x += ', args=' + str(self._args) 2374adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if self._kwargs: 2384adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao x += ', kwargs=' + str(self._kwargs) 2394adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if self._key[0] is not None: 2404adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao x += ', exitprority=' + str(self._key[0]) 2414adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao return x + '>' 2424adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2434adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2444adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaodef _run_finalizers(minpriority=None): 2454adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 2464adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao Run all finalizers whose exit priority is not None and at least minpriority 2474adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2484adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao Finalizers with highest priority are called first; finalizers with 2494adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao the same priority will be called in reverse order of creation. 2504adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 2514adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if _finalizer_registry is None: 2524adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # This function may be called after this module's globals are 2534adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # destroyed. See the _exit_function function in this module for more 2544adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # notes. 2554adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao return 2564adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2574adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if minpriority is None: 2584adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao f = lambda p : p[0][0] is not None 2594adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao else: 2604adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao f = lambda p : p[0][0] is not None and p[0][0] >= minpriority 2614adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2624adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao items = [x for x in _finalizer_registry.items() if f(x)] 2634adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao items.sort(reverse=True) 2644adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2654adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao for key, finalizer in items: 2664adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao sub_debug('calling %s', finalizer) 2674adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao try: 2684adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao finalizer() 2694adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao except Exception: 2704adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao import traceback 2714adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao traceback.print_exc() 2724adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2734adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if minpriority is None: 2744adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao _finalizer_registry.clear() 2754adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2764adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 2774adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# Clean up on exit 2784adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 2794adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2804adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaodef is_exiting(): 2814adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 2824adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao Returns true if the process is shutting down 2834adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao ''' 2844adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao return _exiting or _exiting is None 2854adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2864adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao_exiting = False 2874adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2884adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaodef _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers, 2894adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao active_children=active_children, 2904adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao current_process=current_process): 2914adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # NB: we hold on to references to functions in the arglist due to the 2924adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # situation described below, where this function is called after this 2934adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # module's globals are destroyed. 2944adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2954adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao global _exiting 2964adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 2974adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao info('process shutting down') 2984adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao debug('running all "atexit" finalizers with priority >= 0') 2994adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao _run_finalizers(0) 3004adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 3014adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if current_process() is not None: 3024adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # NB: we check if the current process is None here because if 3034adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # it's None, any call to ``active_children()`` will throw an 3044adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # AttributeError (active_children winds up trying to get 3054adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # attributes from util._current_process). This happens in a 3064adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # variety of shutdown circumstances that are not well-understood 3074adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # because module-scope variables are not apparently supposed to 3084adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # be destroyed until after this function is called. However, 3094adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # they are indeed destroyed before this function is called. See 3104adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao # issues 9775 and 15881. Also related: 4106, 9205, and 9207. 3114adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 3124adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao for p in active_children(): 3134adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao if p._daemonic: 3144adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao info('calling terminate() for daemon %s', p.name) 3154adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao p._popen.terminate() 3164adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 3174adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao for p in active_children(): 3184adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao info('calling join() for process %s', p.name) 3194adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao p.join() 3204adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 3214adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao debug('running the remaining "atexit" finalizers') 3224adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao _run_finalizers() 3234adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 3244adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaoatexit.register(_exit_function) 3254adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 3264adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 3274adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# Some fork aware types 3284adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao# 3294adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 3304adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaoclass ForkAwareThreadLock(object): 3314adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao def __init__(self): 3324adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self._reset() 3334adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao register_after_fork(self, ForkAwareThreadLock._reset) 3344adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 3354adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao def _reset(self): 3364adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self._lock = threading.Lock() 3374adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self.acquire = self._lock.acquire 3384adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao self.release = self._lock.release 3394adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao 3404adfde8bc82dd39f59e0445588c3e599ada477dJosh Gaoclass ForkAwareLocal(threading.local): 3414adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao def __init__(self): 3424adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao register_after_fork(self, lambda obj : obj.__dict__.clear()) 3434adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao def __reduce__(self): 3444adfde8bc82dd39f59e0445588c3e599ada477dJosh Gao return type(self), () 345