10a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 20a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Module providing various facilities to other parts of the package 30a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 40a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# multiprocessing/util.py 50a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 60a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Copyright (c) 2006-2008, R Oudkerk 70a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# All rights reserved. 80a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 90a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Redistribution and use in source and binary forms, with or without 100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# modification, are permitted provided that the following conditions 110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# are met: 120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 1. Redistributions of source code must retain the above copyright 140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# notice, this list of conditions and the following disclaimer. 150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 2. Redistributions in binary form must reproduce the above copyright 160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# notice, this list of conditions and the following disclaimer in the 170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# documentation and/or other materials provided with the distribution. 180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 3. Neither the name of author nor the names of any contributors may be 190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# used to endorse or promote products derived from this software 200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# without specific prior written permission. 210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND 230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# SUCH DAMAGE. 330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 350a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoimport itertools 360a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoimport weakref 370a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoimport atexit 380a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoimport threading # we want threading to install it's 390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # cleanup function before multiprocessing does 400a8c90248264a8b26970b4473770bcc3df8515fJosh Gaofrom subprocess import _args_from_interpreter_flags 410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 420a8c90248264a8b26970b4473770bcc3df8515fJosh Gaofrom multiprocessing.process import current_process, active_children 430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao__all__ = [ 450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger', 460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 'log_to_stderr', 'get_temp_dir', 'register_after_fork', 470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal', 480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 'SUBDEBUG', 'SUBWARNING', 490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ] 500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Logging 530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 550a8c90248264a8b26970b4473770bcc3df8515fJosh GaoNOTSET = 0 560a8c90248264a8b26970b4473770bcc3df8515fJosh GaoSUBDEBUG = 5 570a8c90248264a8b26970b4473770bcc3df8515fJosh GaoDEBUG = 10 580a8c90248264a8b26970b4473770bcc3df8515fJosh GaoINFO = 20 590a8c90248264a8b26970b4473770bcc3df8515fJosh GaoSUBWARNING = 25 600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 610a8c90248264a8b26970b4473770bcc3df8515fJosh GaoLOGGER_NAME = 'multiprocessing' 620a8c90248264a8b26970b4473770bcc3df8515fJosh GaoDEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s' 630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao_logger = None 650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao_log_to_stderr = False 660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 670a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef sub_debug(msg, *args): 680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if _logger: 690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _logger.log(SUBDEBUG, msg, *args) 700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 710a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef debug(msg, *args): 720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if _logger: 730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _logger.log(DEBUG, msg, *args) 740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 750a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef info(msg, *args): 760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if _logger: 770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _logger.log(INFO, msg, *args) 780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 790a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef sub_warning(msg, *args): 800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if _logger: 810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _logger.log(SUBWARNING, msg, *args) 820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 830a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef get_logger(): 840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Returns logger used by multiprocessing 860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao global _logger 880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao import logging, atexit 890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao logging._acquireLock() 910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if not _logger: 930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _logger = logging.getLogger(LOGGER_NAME) 950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _logger.propagate = 0 960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao logging.addLevelName(SUBDEBUG, 'SUBDEBUG') 970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao logging.addLevelName(SUBWARNING, 'SUBWARNING') 980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # XXX multiprocessing should cleanup before logging 1000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if hasattr(atexit, 'unregister'): 1010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao atexit.unregister(_exit_function) 1020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao atexit.register(_exit_function) 1030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 1040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao atexit._exithandlers.remove((_exit_function, (), {})) 1050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao atexit._exithandlers.append((_exit_function, (), {})) 1060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao finally: 1080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao logging._releaseLock() 1090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return _logger 1110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1120a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef log_to_stderr(level=None): 1130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 1140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Turn on logging and add a handler which prints to stderr 1150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 1160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao global _log_to_stderr 1170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao import logging 1180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao logger = get_logger() 1200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) 1210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao handler = logging.StreamHandler() 1220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao handler.setFormatter(formatter) 1230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao logger.addHandler(handler) 1240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if level: 1260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao logger.setLevel(level) 1270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _log_to_stderr = True 1280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return _logger 1290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 1310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Function returning a temp directory which will be removed on exit 1320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 1330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1340a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef get_temp_dir(): 1350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # get name of a temp directory which will be automatically cleaned up 1360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if current_process()._tempdir is None: 1370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao import shutil, tempfile 1380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao tempdir = tempfile.mkdtemp(prefix='pymp-') 1390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao info('created temp directory %s', tempdir) 1400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100) 1410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao current_process()._tempdir = tempdir 1420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return current_process()._tempdir 1430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 1450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Support for reinitialization of objects when bootstrapping a child process 1460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 1470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao_afterfork_registry = weakref.WeakValueDictionary() 1490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao_afterfork_counter = itertools.count() 1500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1510a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef _run_after_forkers(): 1520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao items = list(_afterfork_registry.items()) 1530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao items.sort() 1540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for (index, ident, func), obj in items: 1550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 1560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao func(obj) 1570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except Exception, e: 1580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao info('after forker raised exception %s', e) 1590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1600a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef register_after_fork(obj, func): 1610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _afterfork_registry[(_afterfork_counter.next(), id(obj), func)] = obj 1620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 1640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Finalization using weakrefs 1650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 1660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao_finalizer_registry = {} 1680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao_finalizer_counter = itertools.count() 1690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1710a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoclass Finalize(object): 1720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 1730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Class which supports object finalization using weakrefs 1740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 1750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None): 1760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao assert exitpriority is None or type(exitpriority) is int 1770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if obj is not None: 1790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._weakref = weakref.ref(obj, self) 1800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 1810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao assert exitpriority is not None 1820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._callback = callback 1840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._args = args 1850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._kwargs = kwargs or {} 1860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._key = (exitpriority, _finalizer_counter.next()) 1870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _finalizer_registry[self._key] = self 1890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def __call__(self, wr=None): 1910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 1920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Run the callback unless it has already been called or cancelled 1930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 1940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 1950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao del _finalizer_registry[self._key] 1960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except KeyError: 1970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao sub_debug('finalizer no longer registered') 1980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 1990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao sub_debug('finalizer calling %s with args %s and kwargs %s', 2000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._callback, self._args, self._kwargs) 2010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao res = self._callback(*self._args, **self._kwargs) 2020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._weakref = self._callback = self._args = \ 2030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._kwargs = self._key = None 2040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return res 2050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def cancel(self): 2070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 2080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Cancel finalization of the object 2090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 2100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 2110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao del _finalizer_registry[self._key] 2120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except KeyError: 2130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao pass 2140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 2150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._weakref = self._callback = self._args = \ 2160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._kwargs = self._key = None 2170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def still_active(self): 2190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 2200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Return whether this finalizer is still waiting to invoke callback 2210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 2220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return self._key in _finalizer_registry 2230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def __repr__(self): 2250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 2260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao obj = self._weakref() 2270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except (AttributeError, TypeError): 2280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao obj = None 2290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if obj is None: 2310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return '<Finalize object, dead>' 2320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao x = '<Finalize object, callback=%s' % \ 2340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao getattr(self._callback, '__name__', self._callback) 2350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self._args: 2360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao x += ', args=' + str(self._args) 2370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self._kwargs: 2380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao x += ', kwargs=' + str(self._kwargs) 2390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self._key[0] is not None: 2400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao x += ', exitprority=' + str(self._key[0]) 2410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return x + '>' 2420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2440a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef _run_finalizers(minpriority=None): 2450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 2460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Run all finalizers whose exit priority is not None and at least minpriority 2470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Finalizers with highest priority are called first; finalizers with 2490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao the same priority will be called in reverse order of creation. 2500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 2510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if _finalizer_registry is None: 2520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # This function may be called after this module's globals are 2530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # destroyed. See the _exit_function function in this module for more 2540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # notes. 2550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 2560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if minpriority is None: 2580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao f = lambda p : p[0][0] is not None 2590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 2600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao f = lambda p : p[0][0] is not None and p[0][0] >= minpriority 2610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao items = [x for x in _finalizer_registry.items() if f(x)] 2630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao items.sort(reverse=True) 2640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for key, finalizer in items: 2660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao sub_debug('calling %s', finalizer) 2670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 2680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao finalizer() 2690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except Exception: 2700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao import traceback 2710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao traceback.print_exc() 2720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if minpriority is None: 2740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _finalizer_registry.clear() 2750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 2770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Clean up on exit 2780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 2790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2800a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef is_exiting(): 2810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 2820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Returns true if the process is shutting down 2830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ''' 2840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return _exiting or _exiting is None 2850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao_exiting = False 2870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2880a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers, 2890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao active_children=active_children, 2900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao current_process=current_process): 2910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # NB: we hold on to references to functions in the arglist due to the 2920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # situation described below, where this function is called after this 2930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # module's globals are destroyed. 2940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao global _exiting 2960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao info('process shutting down') 2980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao debug('running all "atexit" finalizers with priority >= 0') 2990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _run_finalizers(0) 3000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if current_process() is not None: 3020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # NB: we check if the current process is None here because if 3030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # it's None, any call to ``active_children()`` will throw an 3040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # AttributeError (active_children winds up trying to get 3050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # attributes from util._current_process). This happens in a 3060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # variety of shutdown circumstances that are not well-understood 3070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # because module-scope variables are not apparently supposed to 3080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # be destroyed until after this function is called. However, 3090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # they are indeed destroyed before this function is called. See 3100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # issues 9775 and 15881. Also related: 4106, 9205, and 9207. 3110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for p in active_children(): 3130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if p._daemonic: 3140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao info('calling terminate() for daemon %s', p.name) 3150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao p._popen.terminate() 3160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for p in active_children(): 3180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao info('calling join() for process %s', p.name) 3190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao p.join() 3200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao debug('running the remaining "atexit" finalizers') 3220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _run_finalizers() 3230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3240a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoatexit.register(_exit_function) 3250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 3270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Some fork aware types 3280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# 3290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3300a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoclass ForkAwareThreadLock(object): 3310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def __init__(self): 3320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._reset() 3330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao register_after_fork(self, ForkAwareThreadLock._reset) 3340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def _reset(self): 3360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._lock = threading.Lock() 3370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.acquire = self._lock.acquire 3380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.release = self._lock.release 3390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3400a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoclass ForkAwareLocal(threading.local): 3410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def __init__(self): 3420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao register_after_fork(self, lambda obj : obj.__dict__.clear()) 3430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def __reduce__(self): 3440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return type(self), () 345