1ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#define JEMALLOC_MUTEX_C_ 2ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#include "jemalloc/internal/jemalloc_internal.h" 3ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 4ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) 5ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#include <dlfcn.h> 6ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#endif 7ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 8ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#ifndef _CRT_SPINCOUNT 9ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#define _CRT_SPINCOUNT 4000 10ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#endif 11ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 12ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang/******************************************************************************/ 13ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang/* Data. */ 14ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 15ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#ifdef JEMALLOC_LAZY_LOCK 16ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangbool isthreaded = false; 17ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#endif 18ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#ifdef JEMALLOC_MUTEX_INIT_CB 19ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangstatic bool postpone_init = true; 20ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangstatic malloc_mutex_t *postponed_mutexes = NULL; 21ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#endif 22ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 23ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) 24ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangstatic void pthread_create_once(void); 25ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#endif 26ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 27ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang/******************************************************************************/ 28ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang/* 29ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang * We intercept pthread_create() calls in order to toggle isthreaded if the 30ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang * process goes multi-threaded. 31ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang */ 32ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 33ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) 34ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangstatic int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, 35ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang void *(*)(void *), void *__restrict); 36ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 37ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangstatic void 38ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangpthread_create_once(void) 39ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang{ 40ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 41ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); 42ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang if (pthread_create_fptr == NULL) { 43ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, " 44ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang "\"pthread_create\")\n"); 45ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang abort(); 46ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 47ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 48ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang isthreaded = true; 49ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang} 50ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 5191037db265ecdd914a26e056cf69207b4f50924ehkuangJEMALLOC_EXPORT int 52ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangpthread_create(pthread_t *__restrict thread, 53ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const pthread_attr_t *__restrict attr, void *(*start_routine)(void *), 54ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang void *__restrict arg) 55ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang{ 56ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang static pthread_once_t once_control = PTHREAD_ONCE_INIT; 57ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 58ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang pthread_once(&once_control, pthread_create_once); 59ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 60ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang return (pthread_create_fptr(thread, attr, start_routine, arg)); 61ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang} 62ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#endif 63ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 64ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang/******************************************************************************/ 65ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 66ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#ifdef JEMALLOC_MUTEX_INIT_CB 6791037db265ecdd914a26e056cf69207b4f50924ehkuangJEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 68ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang void *(calloc_cb)(size_t, size_t)); 69ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#endif 70ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 71ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangbool 72ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangmalloc_mutex_init(malloc_mutex_t *mutex) 73ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang{ 74ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 75ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#ifdef _WIN32 76ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang if (!InitializeCriticalSectionAndSpinCount(&mutex->lock, 77ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang _CRT_SPINCOUNT)) 78ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang return (true); 79ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#elif (defined(JEMALLOC_OSSPIN)) 80ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang mutex->lock = 0; 81ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#elif (defined(JEMALLOC_MUTEX_INIT_CB)) 82ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang if (postpone_init) { 83ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang mutex->postponed_next = postponed_mutexes; 84ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang postponed_mutexes = mutex; 85ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } else { 86ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang if (_pthread_mutex_init_calloc_cb(&mutex->lock, base_calloc) != 87ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 0) 88ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang return (true); 89ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 90ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#else 91ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang pthread_mutexattr_t attr; 92ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 93ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang if (pthread_mutexattr_init(&attr) != 0) 94ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang return (true); 95ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE); 96ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang if (pthread_mutex_init(&mutex->lock, &attr) != 0) { 97ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang pthread_mutexattr_destroy(&attr); 98ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang return (true); 9991037db265ecdd914a26e056cf69207b4f50924ehkuang } 100ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang pthread_mutexattr_destroy(&attr); 101ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#endif 102ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang return (false); 103ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang} 104ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 105ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangvoid 106ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangmalloc_mutex_prefork(malloc_mutex_t *mutex) 107ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang{ 108ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 109ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang malloc_mutex_lock(mutex); 110ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang} 111ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 112ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangvoid 113ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangmalloc_mutex_postfork_parent(malloc_mutex_t *mutex) 114ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang{ 115ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 11691037db265ecdd914a26e056cf69207b4f50924ehkuang malloc_mutex_unlock(mutex); 117ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang} 118ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 119ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangvoid 120ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangmalloc_mutex_postfork_child(malloc_mutex_t *mutex) 121ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang{ 122ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 123ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#ifdef JEMALLOC_MUTEX_INIT_CB 124ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang malloc_mutex_unlock(mutex); 125ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#else 126ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang if (malloc_mutex_init(mutex)) { 127ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang malloc_printf("<jemalloc>: Error re-initializing mutex in " 128ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang "child\n"); 129ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang if (opt_abort) 130ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang abort(); 131ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 132ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#endif 133ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang} 134ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 135ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangbool 13691037db265ecdd914a26e056cf69207b4f50924ehkuangmutex_boot(void) 137ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang{ 138ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 139ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#ifdef JEMALLOC_MUTEX_INIT_CB 140ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang postpone_init = false; 141ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang while (postponed_mutexes != NULL) { 14291037db265ecdd914a26e056cf69207b4f50924ehkuang if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock, 14391037db265ecdd914a26e056cf69207b4f50924ehkuang base_calloc) != 0) 14491037db265ecdd914a26e056cf69207b4f50924ehkuang return (true); 14591037db265ecdd914a26e056cf69207b4f50924ehkuang postponed_mutexes = postponed_mutexes->postponed_next; 14691037db265ecdd914a26e056cf69207b4f50924ehkuang } 14791037db265ecdd914a26e056cf69207b4f50924ehkuang#endif 148ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang return (false); 149ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang} 150ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang