tcache.c revision dafde14e08ddfda747aabb2045b350848b601b2e
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_TCACHE_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 73fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evansbool opt_tcache = true; 8dafde14e08ddfda747aabb2045b350848b601b2eJason Evansssize_t opt_lg_tcache_maxclass = LG_TCACHE_MAXCLASS_DEFAULT; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evansssize_t opt_lg_tcache_gc_sweep = LG_TCACHE_GC_SWEEP_DEFAULT; 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 11e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Map of thread-specific caches. */ 12e476f8a161d445211fd6e54fe370275196e66bcbJason Evans__thread tcache_t *tcache_tls JEMALLOC_ATTR(tls_model("initial-exec")); 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Same contents as tcache, but initialized such that the TSD destructor is 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * called when a thread exits, so that the cache can be cleaned up. 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic pthread_key_t tcache_tsd; 19e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 20dafde14e08ddfda747aabb2045b350848b601b2eJason Evanssize_t nhbins; 21dafde14e08ddfda747aabb2045b350848b601b2eJason Evanssize_t tcache_maxclass; 22e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned tcache_gc_incr; 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 24e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Function prototypes for non-inline static functions. */ 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void tcache_thread_cleanup(void *arg); 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 32dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind) 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 34e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 35e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 36dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_tcache_fill_small(tcache->arena, tbin, binind 37d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 38d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , tcache->prof_accumbytes 39d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 40d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ); 41d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 42d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans tcache->prof_accumbytes = 0; 43d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 44dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ret = tcache_alloc_easy(tbin); 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 46e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 49e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 50dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem 5186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 52d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , tcache_t *tcache 53d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 54d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ) 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 563fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans void *flush, *deferred, *ptr; 573fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans unsigned i, nflush, ndeferred; 58e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 59dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(binind < nbins); 6086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans assert(rem <= tbin->ncached); 6186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 623fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans for (flush = tbin->avail, nflush = tbin->ncached - rem; flush != NULL; 633fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans flush = deferred, nflush = ndeferred) { 6486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* Lock the arena bin associated with the first object. */ 653fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush); 663fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_t *arena = chunk->arena; 6786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin = &arena->bins[binind]; 6886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 69e69bee01de62b56d3e585042d341743239568043Jason Evans#ifdef JEMALLOC_PROF 70d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans if (arena == tcache->arena) { 7186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 72d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, tcache->prof_accumbytes); 7386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 74d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans tcache->prof_accumbytes = 0; 75e69bee01de62b56d3e585042d341743239568043Jason Evans } 76e69bee01de62b56d3e585042d341743239568043Jason Evans#endif 77e69bee01de62b56d3e585042d341743239568043Jason Evans 78e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 79e69bee01de62b56d3e585042d341743239568043Jason Evans#ifdef JEMALLOC_STATS 80e69bee01de62b56d3e585042d341743239568043Jason Evans if (arena == tcache->arena) { 8186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nflushes++; 8286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 8386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans tbin->tstats.nrequests = 0; 84d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans } 85d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 863fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans deferred = NULL; 873fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans ndeferred = 0; 883fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans for (i = 0; i < nflush; i++) { 893fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans ptr = flush; 903fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans assert(ptr != NULL); 913fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans flush = *(void **)ptr; 92e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 93e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->arena == arena) { 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = (((uintptr_t)ptr - 95e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)chunk) >> PAGE_SHIFT); 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm = 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[pageind]; 98e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_bin(arena, chunk, ptr, mapelm); 99e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * This object was allocated via a different 10286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * arena bin than the one that is currently 10386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. Stash the object, so that it can be 10486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * handled in a future pass. 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1063fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans *(void **)ptr = deferred; 1073fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans deferred = ptr; 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ndeferred++; 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 11186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1133fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (flush != NULL) { 1143fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans /* 1153fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * This was the first pass, and rem cached objects 1163fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * remain. 1173fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans */ 1183fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->avail = flush; 1193fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans } 120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1223fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->ncached = rem; 12386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (tbin->ncached < tbin->low_water) 12486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans tbin->low_water = tbin->ncached; 125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 127dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid 128dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem 129dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 130dafde14e08ddfda747aabb2045b350848b601b2eJason Evans , tcache_t *tcache 131dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 132dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ) 133dafde14e08ddfda747aabb2045b350848b601b2eJason Evans{ 134dafde14e08ddfda747aabb2045b350848b601b2eJason Evans void *flush, *deferred, *ptr; 135dafde14e08ddfda747aabb2045b350848b601b2eJason Evans unsigned i, nflush, ndeferred; 136dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 137dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(binind < nhbins); 138dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(rem <= tbin->ncached); 139dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 140dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (flush = tbin->avail, nflush = tbin->ncached - rem; flush != NULL; 141dafde14e08ddfda747aabb2045b350848b601b2eJason Evans flush = deferred, nflush = ndeferred) { 142dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* Lock the arena associated with the first object. */ 143dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush); 144dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_t *arena = chunk->arena; 145dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 146dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 147dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 148dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (arena == tcache->arena) { 149dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 150dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_PROF 151dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_prof_accum(arena, tcache->prof_accumbytes); 152dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache->prof_accumbytes = 0; 153dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 154dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_STATS 155dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 156dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.lstats[binind - nbins].nrequests += 157dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests; 158dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests = 0; 159dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 160dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 161dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 162dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 163dafde14e08ddfda747aabb2045b350848b601b2eJason Evans deferred = NULL; 164dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ndeferred = 0; 165dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (i = 0; i < nflush; i++) { 166dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ptr = flush; 167dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(ptr != NULL); 168dafde14e08ddfda747aabb2045b350848b601b2eJason Evans flush = *(void **)ptr; 169dafde14e08ddfda747aabb2045b350848b601b2eJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 170dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (chunk->arena == arena) 171dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_dalloc_large(arena, chunk, ptr); 172dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else { 173dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* 174dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * This object was allocated via a different 175dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * arena than the one that is currently locked. 176dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Stash the object, so that it can be handled 177dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * in a future pass. 178dafde14e08ddfda747aabb2045b350848b601b2eJason Evans */ 179dafde14e08ddfda747aabb2045b350848b601b2eJason Evans *(void **)ptr = deferred; 180dafde14e08ddfda747aabb2045b350848b601b2eJason Evans deferred = ptr; 181dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ndeferred++; 182dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 183dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 184dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 185dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 186dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (flush != NULL) { 187dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* 188dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * This was the first pass, and rem cached objects 189dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * remain. 190dafde14e08ddfda747aabb2045b350848b601b2eJason Evans */ 191dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->avail = flush; 192dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 193dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 194dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 195dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->ncached = rem; 196dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (tbin->ncached < tbin->low_water) 197dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->low_water = tbin->ncached; 198dafde14e08ddfda747aabb2045b350848b601b2eJason Evans} 199dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 200e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_t * 201e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_create(arena_t *arena) 202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache; 2043fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans size_t size; 2053fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans unsigned i; 206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 207dafde14e08ddfda747aabb2045b350848b601b2eJason Evans size = sizeof(tcache_t) + (sizeof(tcache_bin_t) * (nhbins - 1)); 2083fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans /* 2093fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * Round up to the nearest multiple of the cacheline size, in order to 2103fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * avoid the possibility of false cacheline sharing. 2113fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * 2123fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * That this works relies on the same logic as in ipalloc(). 2133fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans */ 2143fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans size = (size + CACHELINE_MASK) & (-CACHELINE); 2153fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans 2163fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (size <= small_maxclass) 2173fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache = (tcache_t *)arena_malloc_small(arena, size, true); 2183fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans else 2193fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache = (tcache_t *)icalloc(size); 220e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tcache == NULL) 222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 224e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Link into list of extant tcaches. */ 226e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_elm_new(tcache, link); 228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_tail_insert(&arena->tcache_ql, tcache, link); 229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache->arena = arena; 233dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); 2343fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans for (i = 0; i < nbins; i++) { 235dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if ((arena->bins[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MAX) { 2363fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache->tbins[i].ncached_max = (arena->bins[i].nregs << 2373fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans 1); 2383fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans } else 239dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache->tbins[i].ncached_max = TCACHE_NSLOTS_SMALL_MAX; 2403fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans } 241dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (; i < nhbins; i++) 242dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache->tbins[i].ncached_max = TCACHE_NSLOTS_LARGE; 243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_tls = tcache; 245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pthread_setspecific(tcache_tsd, tcache); 246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (tcache); 248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 250e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 251e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_destroy(tcache_t *tcache) 252e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 253e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 255e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 256e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Unlink from list of extant tcaches. */ 257e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&tcache->arena->lock); 258e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_remove(&tcache->arena->tcache_ql, tcache, link); 259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&tcache->arena->lock); 26086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans tcache_stats_merge(tcache, tcache->arena); 261e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 262e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 263e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < nbins; i++) { 2643fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 265dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_flush_small(tbin, i, 0 26686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 2673fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans , tcache 268d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 2693fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans ); 2703fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans 2713fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans#ifdef JEMALLOC_STATS 2723fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (tbin->tstats.nrequests != 0) { 2733fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_t *arena = tcache->arena; 2743fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_bin_t *bin = &arena->bins[i]; 27586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 2763fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans bin->stats.nrequests += tbin->tstats.nrequests; 27786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2793fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans#endif 280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 282dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (; i < nhbins; i++) { 283dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 284dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_flush_large(tbin, i, 0 285dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 286dafde14e08ddfda747aabb2045b350848b601b2eJason Evans , tcache 287dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 288dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ); 289dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 290dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_STATS 291dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (tbin->tstats.nrequests != 0) { 292dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_t *arena = tcache->arena; 293dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 294dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 295dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.lstats[i - nbins].nrequests += 296dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests; 297dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 298dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 299dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 300dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 301dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 302d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 303d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans if (tcache->prof_accumbytes > 0) { 304d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans malloc_mutex_lock(&tcache->arena->lock); 305d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(tcache->arena, tcache->prof_accumbytes); 306d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans malloc_mutex_unlock(&tcache->arena->lock); 307d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans } 308d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 309d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 310dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (arena_salloc(tcache) <= small_maxclass) { 311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); 312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena = chunk->arena; 313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = (((uintptr_t)tcache - (uintptr_t)chunk) >> 314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT); 315e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm = &chunk->map[pageind]; 31686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 31786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans (uintptr_t)((pageind - ((mapelm->bits & CHUNK_MAP_PG_MASK) 31886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans >> CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT)); 31986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin = run->bin; 320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 32186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_bin(arena, chunk, tcache, mapelm); 32386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans idalloc(tcache); 326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 328e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 329e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_thread_cleanup(void *arg) 330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache = (tcache_t *)arg; 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tcache == tcache_tls); 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tcache != NULL) { 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tcache != (void *)(uintptr_t)1); 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_destroy(tcache); 337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_tls = (void *)(uintptr_t)1; 338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 343e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_stats_merge(tcache_t *tcache, arena_t *arena) 344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Merge and reset tcache stats. */ 34886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans for (i = 0; i < nbins; i++) { 349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin = &arena->bins[i]; 3503fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 35186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 3523fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans bin->stats.nrequests += tbin->tstats.nrequests; 35386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 3543fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->tstats.nrequests = 0; 355e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 356dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 357dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (; i < nhbins; i++) { 358dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_large_stats_t *lstats = &arena->stats.lstats[i - nbins]; 359dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 360dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 361dafde14e08ddfda747aabb2045b350848b601b2eJason Evans lstats->nrequests += tbin->tstats.nrequests; 362dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests = 0; 363dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 367e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 368e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_boot(void) 369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 3713fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (opt_tcache) { 372dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* 373dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * If necessary, clamp opt_lg_tcache_maxclass, now that 374dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * small_maxclass and arena_maxclass are known. 375dafde14e08ddfda747aabb2045b350848b601b2eJason Evans */ 376dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (opt_lg_tcache_maxclass < 0 || (1U << 377dafde14e08ddfda747aabb2045b350848b601b2eJason Evans opt_lg_tcache_maxclass) < small_maxclass) 378dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_maxclass = small_maxclass; 379dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else if ((1U << opt_lg_tcache_maxclass) > arena_maxclass) 380dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_maxclass = arena_maxclass; 381dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else 382dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_maxclass = (1U << opt_lg_tcache_maxclass); 383dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 384dafde14e08ddfda747aabb2045b350848b601b2eJason Evans nhbins = nbins + (tcache_maxclass >> PAGE_SHIFT); 385dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Compute incremental GC event threshold. */ 387e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_tcache_gc_sweep >= 0) { 388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_gc_incr = ((1U << opt_lg_tcache_gc_sweep) / 389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nbins) + (((1U << opt_lg_tcache_gc_sweep) % nbins == 390e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 0) ? 0 : 1); 391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 392e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_gc_incr = 0; 393e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 394e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) != 395e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 0) { 396698805c5252465ddbdcf7048e5ccf16d02533877Jason Evans malloc_write( 397698805c5252465ddbdcf7048e5ccf16d02533877Jason Evans "<jemalloc>: Error in pthread_key_create()\n"); 398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans abort(); 399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 403e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_TCACHE */ 404