1e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 2e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# Module providing various facilities to other parts of the package 3e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 4e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# multiprocessing/util.py 5e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 63fc969a4a26f0bd94f1412b45da5e8cf3c6eaf1eR. David Murray# Copyright (c) 2006-2008, R Oudkerk 73e268aac3b1b3c89925b59dae7a213b94bc114b8Richard Oudkerk# Licensed to PSF under a Contributor Agreement. 8e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 9e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 10739ae5692ebc7daaf1297d8e775e552bdb05f3feRichard Oudkerkimport os 11e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersonimport itertools 12a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinnerimport sys 13e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersonimport weakref 14e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersonimport atexit 15e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersonimport threading # we want threading to install it's 16e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson # cleanup function before multiprocessing does 17ebdcd859e59ed16a79dea94291c0be3a87640a08Antoine Pitroufrom subprocess import _args_from_interpreter_flags 18e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 1984ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerkfrom . import process 20e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 21e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson__all__ = [ 22e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger', 23e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 'log_to_stderr', 'get_temp_dir', 'register_after_fork', 2441faa543b66431ee5b8c79f70a40ef267615050bJesse Noller 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal', 2584ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk 'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING', 26e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ] 27e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 28e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 29e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# Logging 30e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 31e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 32e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin PetersonNOTSET = 0 33e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin PetersonSUBDEBUG = 5 34e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin PetersonDEBUG = 10 35e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin PetersonINFO = 20 36e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin PetersonSUBWARNING = 25 37e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 38e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin PetersonLOGGER_NAME = 'multiprocessing' 39e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin PetersonDEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s' 40e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 41e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson_logger = None 42e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson_log_to_stderr = False 43e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 44e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersondef sub_debug(msg, *args): 45e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson if _logger: 46e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson _logger.log(SUBDEBUG, msg, *args) 47e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 48e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersondef debug(msg, *args): 49e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson if _logger: 50e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson _logger.log(DEBUG, msg, *args) 51e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 52e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersondef info(msg, *args): 53e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson if _logger: 54e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson _logger.log(INFO, msg, *args) 55e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 56e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersondef sub_warning(msg, *args): 57e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson if _logger: 58e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson _logger.log(SUBWARNING, msg, *args) 59e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 60e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersondef get_logger(): 61e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 62e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson Returns logger used by multiprocessing 63e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 64e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson global _logger 6504842a84c31d82d9c2255e6575344412c58e518eFlorent Xicluna import logging 66e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 6741faa543b66431ee5b8c79f70a40ef267615050bJesse Noller logging._acquireLock() 6841faa543b66431ee5b8c79f70a40ef267615050bJesse Noller try: 6941faa543b66431ee5b8c79f70a40ef267615050bJesse Noller if not _logger: 70e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 7141faa543b66431ee5b8c79f70a40ef267615050bJesse Noller _logger = logging.getLogger(LOGGER_NAME) 7241faa543b66431ee5b8c79f70a40ef267615050bJesse Noller _logger.propagate = 0 73e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 7441faa543b66431ee5b8c79f70a40ef267615050bJesse Noller # XXX multiprocessing should cleanup before logging 7541faa543b66431ee5b8c79f70a40ef267615050bJesse Noller if hasattr(atexit, 'unregister'): 7641faa543b66431ee5b8c79f70a40ef267615050bJesse Noller atexit.unregister(_exit_function) 7741faa543b66431ee5b8c79f70a40ef267615050bJesse Noller atexit.register(_exit_function) 7841faa543b66431ee5b8c79f70a40ef267615050bJesse Noller else: 7941faa543b66431ee5b8c79f70a40ef267615050bJesse Noller atexit._exithandlers.remove((_exit_function, (), {})) 8041faa543b66431ee5b8c79f70a40ef267615050bJesse Noller atexit._exithandlers.append((_exit_function, (), {})) 8141faa543b66431ee5b8c79f70a40ef267615050bJesse Noller 8241faa543b66431ee5b8c79f70a40ef267615050bJesse Noller finally: 8341faa543b66431ee5b8c79f70a40ef267615050bJesse Noller logging._releaseLock() 84e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 85e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson return _logger 86e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 87e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersondef log_to_stderr(level=None): 88e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 89e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson Turn on logging and add a handler which prints to stderr 90e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 91e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson global _log_to_stderr 92e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson import logging 9341faa543b66431ee5b8c79f70a40ef267615050bJesse Noller 94e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson logger = get_logger() 95e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) 96e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson handler = logging.StreamHandler() 97e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson handler.setFormatter(formatter) 98e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson logger.addHandler(handler) 9941faa543b66431ee5b8c79f70a40ef267615050bJesse Noller 10041faa543b66431ee5b8c79f70a40ef267615050bJesse Noller if level: 101e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson logger.setLevel(level) 102e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson _log_to_stderr = True 10341faa543b66431ee5b8c79f70a40ef267615050bJesse Noller return _logger 104e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 105e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 106e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# Function returning a temp directory which will be removed on exit 107e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 108e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 109e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersondef get_temp_dir(): 110e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson # get name of a temp directory which will be automatically cleaned up 11184ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk tempdir = process.current_process()._config.get('tempdir') 11284ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk if tempdir is None: 113e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson import shutil, tempfile 114e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson tempdir = tempfile.mkdtemp(prefix='pymp-') 115e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson info('created temp directory %s', tempdir) 116e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100) 11784ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk process.current_process()._config['tempdir'] = tempdir 11884ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk return tempdir 119e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 120e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 121e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# Support for reinitialization of objects when bootstrapping a child process 122e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 123e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 124e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson_afterfork_registry = weakref.WeakValueDictionary() 125e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson_afterfork_counter = itertools.count() 126e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 127e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersondef _run_after_forkers(): 128e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson items = list(_afterfork_registry.items()) 129e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson items.sort() 130e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson for (index, ident, func), obj in items: 131e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson try: 132e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson func(obj) 133e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson except Exception as e: 134e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson info('after forker raised exception %s', e) 135e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 136e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersondef register_after_fork(obj, func): 137e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson _afterfork_registry[(next(_afterfork_counter), id(obj), func)] = obj 138e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 139e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 140e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# Finalization using weakrefs 141e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 142e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 143e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson_finalizer_registry = {} 144e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson_finalizer_counter = itertools.count() 145e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 146e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 147e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersonclass Finalize(object): 148e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 149e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson Class which supports object finalization using weakrefs 150e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 151e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None): 152e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson assert exitpriority is None or type(exitpriority) is int 153e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 154e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson if obj is not None: 155e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson self._weakref = weakref.ref(obj, self) 156e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson else: 157e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson assert exitpriority is not None 158e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 159e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson self._callback = callback 160e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson self._args = args 161e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson self._kwargs = kwargs or {} 162e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson self._key = (exitpriority, next(_finalizer_counter)) 163739ae5692ebc7daaf1297d8e775e552bdb05f3feRichard Oudkerk self._pid = os.getpid() 164e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 165e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson _finalizer_registry[self._key] = self 166e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 16771a28a91744b5cf6f881769300818232750de6d2Antoine Pitrou def __call__(self, wr=None, 16871a28a91744b5cf6f881769300818232750de6d2Antoine Pitrou # Need to bind these locally because the globals can have 16971a28a91744b5cf6f881769300818232750de6d2Antoine Pitrou # been cleared at shutdown 17071a28a91744b5cf6f881769300818232750de6d2Antoine Pitrou _finalizer_registry=_finalizer_registry, 171ad06444d6c52e614a65ee126bc4c916da0c2c57eRichard Oudkerk sub_debug=sub_debug, getpid=os.getpid): 172e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 173e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson Run the callback unless it has already been called or cancelled 174e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 175e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson try: 176e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson del _finalizer_registry[self._key] 177e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson except KeyError: 178e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson sub_debug('finalizer no longer registered') 179e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson else: 180ad06444d6c52e614a65ee126bc4c916da0c2c57eRichard Oudkerk if self._pid != getpid(): 181739ae5692ebc7daaf1297d8e775e552bdb05f3feRichard Oudkerk sub_debug('finalizer ignored because different process') 182739ae5692ebc7daaf1297d8e775e552bdb05f3feRichard Oudkerk res = None 183739ae5692ebc7daaf1297d8e775e552bdb05f3feRichard Oudkerk else: 184739ae5692ebc7daaf1297d8e775e552bdb05f3feRichard Oudkerk sub_debug('finalizer calling %s with args %s and kwargs %s', 185739ae5692ebc7daaf1297d8e775e552bdb05f3feRichard Oudkerk self._callback, self._args, self._kwargs) 186739ae5692ebc7daaf1297d8e775e552bdb05f3feRichard Oudkerk res = self._callback(*self._args, **self._kwargs) 187e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson self._weakref = self._callback = self._args = \ 188e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson self._kwargs = self._key = None 189e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson return res 190e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 191e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson def cancel(self): 192e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 193e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson Cancel finalization of the object 194e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 195e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson try: 196e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson del _finalizer_registry[self._key] 197e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson except KeyError: 198e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson pass 199e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson else: 200e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson self._weakref = self._callback = self._args = \ 201e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson self._kwargs = self._key = None 202e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 203e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson def still_active(self): 204e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 205e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson Return whether this finalizer is still waiting to invoke callback 206e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 207e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson return self._key in _finalizer_registry 208e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 209e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson def __repr__(self): 210e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson try: 211e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson obj = self._weakref() 212e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson except (AttributeError, TypeError): 213e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson obj = None 214e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 215e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson if obj is None: 216465e60e654732b3a0b726d1fd82950fc982fcba2Serhiy Storchaka return '<%s object, dead>' % self.__class__.__name__ 217e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 218465e60e654732b3a0b726d1fd82950fc982fcba2Serhiy Storchaka x = '<%s object, callback=%s' % ( 219465e60e654732b3a0b726d1fd82950fc982fcba2Serhiy Storchaka self.__class__.__name__, 220465e60e654732b3a0b726d1fd82950fc982fcba2Serhiy Storchaka getattr(self._callback, '__name__', self._callback)) 221e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson if self._args: 222e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson x += ', args=' + str(self._args) 223e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson if self._kwargs: 224e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson x += ', kwargs=' + str(self._kwargs) 225e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson if self._key[0] is not None: 226e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson x += ', exitprority=' + str(self._key[0]) 227e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson return x + '>' 228e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 229e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 230e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersondef _run_finalizers(minpriority=None): 231e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 232e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson Run all finalizers whose exit priority is not None and at least minpriority 233e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 234e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson Finalizers with highest priority are called first; finalizers with 235e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson the same priority will be called in reverse order of creation. 236e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 237f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky if _finalizer_registry is None: 238f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky # This function may be called after this module's globals are 239f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky # destroyed. See the _exit_function function in this module for more 240f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky # notes. 241f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky return 2427f704c11dbf6c766e11ed52c38fb732f7beda904Alexander Belopolsky 243e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson if minpriority is None: 244e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson f = lambda p : p[0][0] is not None 245e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson else: 246e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson f = lambda p : p[0][0] is not None and p[0][0] >= minpriority 247e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 248e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson items = [x for x in list(_finalizer_registry.items()) if f(x)] 249e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson items.sort(reverse=True) 250e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 251e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson for key, finalizer in items: 252e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson sub_debug('calling %s', finalizer) 253e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson try: 254e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson finalizer() 255e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson except Exception: 256e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson import traceback 257e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson traceback.print_exc() 258e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 259e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson if minpriority is None: 260e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson _finalizer_registry.clear() 261e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 262e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 263e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# Clean up on exit 264e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 265e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 266e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersondef is_exiting(): 267e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 268e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson Returns true if the process is shutting down 269e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson ''' 270e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson return _exiting or _exiting is None 271e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 272e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson_exiting = False 273e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 274f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolskydef _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers, 27584ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk active_children=process.active_children, 27684ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk current_process=process.current_process): 277f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky # We hold on to references to functions in the arglist due to the 278f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky # situation described below, where this function is called after this 279f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky # module's globals are destroyed. 280f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky 281e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson global _exiting 282e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 28373d9a292aeed5aa3414c911c1f34afeea17f0ddaRichard Oudkerk if not _exiting: 28473d9a292aeed5aa3414c911c1f34afeea17f0ddaRichard Oudkerk _exiting = True 285e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 28673d9a292aeed5aa3414c911c1f34afeea17f0ddaRichard Oudkerk info('process shutting down') 28773d9a292aeed5aa3414c911c1f34afeea17f0ddaRichard Oudkerk debug('running all "atexit" finalizers with priority >= 0') 28873d9a292aeed5aa3414c911c1f34afeea17f0ddaRichard Oudkerk _run_finalizers(0) 289e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 290f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky if current_process() is not None: 291f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky # We check if the current process is None here because if 2925b89840d9cf11014a4b865d79497649f74bf2866Andrew Svetlov # it's None, any call to ``active_children()`` will raise 293e8cd6bb127698306ae25ed18c8cfe4fbfe7b6f15Richard Oudkerk # an AttributeError (active_children winds up trying to 294e8cd6bb127698306ae25ed18c8cfe4fbfe7b6f15Richard Oudkerk # get attributes from util._current_process). One 295e8cd6bb127698306ae25ed18c8cfe4fbfe7b6f15Richard Oudkerk # situation where this can happen is if someone has 296e8cd6bb127698306ae25ed18c8cfe4fbfe7b6f15Richard Oudkerk # manipulated sys.modules, causing this module to be 297e8cd6bb127698306ae25ed18c8cfe4fbfe7b6f15Richard Oudkerk # garbage collected. The destructor for the module type 298e8cd6bb127698306ae25ed18c8cfe4fbfe7b6f15Richard Oudkerk # then replaces all values in the module dict with None. 299e8cd6bb127698306ae25ed18c8cfe4fbfe7b6f15Richard Oudkerk # For instance, after setuptools runs a test it replaces 300e8cd6bb127698306ae25ed18c8cfe4fbfe7b6f15Richard Oudkerk # sys.modules with a copy created earlier. See issues 301e8cd6bb127698306ae25ed18c8cfe4fbfe7b6f15Richard Oudkerk # #9775 and #15881. Also related: #4106, #9205, and 302e8cd6bb127698306ae25ed18c8cfe4fbfe7b6f15Richard Oudkerk # #9207. 303f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky 304f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky for p in active_children(): 30584ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk if p.daemon: 306f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky info('calling terminate() for daemon %s', p.name) 307f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky p._popen.terminate() 308f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky 309f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky for p in active_children(): 310f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky info('calling join() for process %s', p.name) 311f36c49d124af3cbe97ff5e1dd2a4e1a85f2575d2Alexander Belopolsky p.join() 31273d9a292aeed5aa3414c911c1f34afeea17f0ddaRichard Oudkerk 31373d9a292aeed5aa3414c911c1f34afeea17f0ddaRichard Oudkerk debug('running the remaining "atexit" finalizers') 31473d9a292aeed5aa3414c911c1f34afeea17f0ddaRichard Oudkerk _run_finalizers() 315e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 316e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersonatexit.register(_exit_function) 317e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 318e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 319e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# Some fork aware types 320e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson# 321e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 322e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersonclass ForkAwareThreadLock(object): 323e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson def __init__(self): 324409c31390f839debb2f05cef691e8b59e52bd524Richard Oudkerk self._reset() 325409c31390f839debb2f05cef691e8b59e52bd524Richard Oudkerk register_after_fork(self, ForkAwareThreadLock._reset) 326409c31390f839debb2f05cef691e8b59e52bd524Richard Oudkerk 327409c31390f839debb2f05cef691e8b59e52bd524Richard Oudkerk def _reset(self): 328e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson self._lock = threading.Lock() 329e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson self.acquire = self._lock.acquire 330e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson self.release = self._lock.release 331e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson 332a924fc7abc2d8788a4a9fa2cbef2caa5c5992ebdCharles-François Natali def __enter__(self): 333a924fc7abc2d8788a4a9fa2cbef2caa5c5992ebdCharles-François Natali return self._lock.__enter__() 334a924fc7abc2d8788a4a9fa2cbef2caa5c5992ebdCharles-François Natali 335a924fc7abc2d8788a4a9fa2cbef2caa5c5992ebdCharles-François Natali def __exit__(self, *args): 336a924fc7abc2d8788a4a9fa2cbef2caa5c5992ebdCharles-François Natali return self._lock.__exit__(*args) 337a924fc7abc2d8788a4a9fa2cbef2caa5c5992ebdCharles-François Natali 338a924fc7abc2d8788a4a9fa2cbef2caa5c5992ebdCharles-François Natali 339e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Petersonclass ForkAwareLocal(threading.local): 340e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson def __init__(self): 341e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson register_after_fork(self, lambda obj : obj.__dict__.clear()) 342e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson def __reduce__(self): 343e711cafab13efc9c1fe6c5cd75826401445eb585Benjamin Peterson return type(self), () 34484ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk 34584ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk# 34684ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk# Close fds except those specified 34784ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk# 34884ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk 34984ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerktry: 35084ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk MAXFD = os.sysconf("SC_OPEN_MAX") 35184ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerkexcept Exception: 35284ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk MAXFD = 256 35384ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk 35484ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerkdef close_all_fds_except(fds): 35584ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk fds = list(fds) + [-1, MAXFD] 35684ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk fds.sort() 35784ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk assert fds[-1] == MAXFD, 'fd too large' 35884ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk for i in range(len(fds) - 1): 35984ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk os.closerange(fds[i]+1, fds[i+1]) 360a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner# 361a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner# Close sys.stdin and replace stdin with os.devnull 362a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner# 363a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner 364a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinnerdef _close_stdin(): 365a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner if sys.stdin is None: 366a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner return 367a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner 368a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner try: 369a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner sys.stdin.close() 370a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner except (OSError, ValueError): 371a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner pass 372a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner 373a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner try: 374a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner fd = os.open(os.devnull, os.O_RDONLY) 375a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner try: 376a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner sys.stdin = open(fd, closefd=False) 377a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner except: 378a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner os.close(fd) 379a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner raise 380a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner except (OSError, ValueError): 381a6d865c128dd46a067358e94c29ca2d84205ae89Victor Stinner pass 38284ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk 38384ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk# 38484ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk# Start a program with only specified fds kept open 38584ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk# 38684ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk 38784ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerkdef spawnv_passfds(path, args, passfds): 38867973c02790471b23a66b798533ea7ceb8d1dc9dVictor Stinner import _posixsubprocess 38984ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk passfds = sorted(passfds) 390daf455554bc21b6b5df0a016ab5fa639d36cc595Victor Stinner errpipe_read, errpipe_write = os.pipe() 39184ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk try: 39284ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk return _posixsubprocess.fork_exec( 39384ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk args, [os.fsencode(path)], True, passfds, None, None, 39484ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, 39584ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk False, False, None) 39684ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk finally: 39784ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk os.close(errpipe_read) 39884ed9a68bd9a13252b376b21a9167dabae254325Richard Oudkerk os.close(errpipe_write) 399