tcache.c revision a8118233ec0369e00629fe853a5655c0dabf83d2
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_TCACHE_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 73fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evansbool opt_tcache = true; 8e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evansssize_t opt_lg_tcache_gc_sweep = LG_TCACHE_GC_SWEEP_DEFAULT; 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 11e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Map of thread-specific caches. */ 122dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#ifndef NO_TLS 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans__thread tcache_t *tcache_tls JEMALLOC_ATTR(tls_model("initial-exec")); 142dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#endif 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Same contents as tcache, but initialized such that the TSD destructor is 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * called when a thread exits, so that the cache can be cleaned up. 19e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 202dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evanspthread_key_t tcache_tsd; 21e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 22dafde14e08ddfda747aabb2045b350848b601b2eJason Evanssize_t nhbins; 23dafde14e08ddfda747aabb2045b350848b601b2eJason Evanssize_t tcache_maxclass; 24e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned tcache_gc_incr; 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Function prototypes for non-inline static functions. */ 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void tcache_thread_cleanup(void *arg); 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 32e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 34dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind) 35e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 36e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 37e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 38dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_tcache_fill_small(tcache->arena, tbin, binind 39d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 40d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , tcache->prof_accumbytes 41d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 42d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ); 43d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 44d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans tcache->prof_accumbytes = 0; 45d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 46dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ret = tcache_alloc_easy(tbin); 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 49e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 52dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem 5386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 54d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , tcache_t *tcache 55d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 56d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ) 57e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 583fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans void *flush, *deferred, *ptr; 593fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans unsigned i, nflush, ndeferred; 605055f4516c8852e67668b0e746863a7d6a1c148eJason Evans bool first_pass; 61a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans#ifdef JEMALLOC_STATS 62a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans bool merged_stats = false; 63a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans#endif 64e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 65dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(binind < nbins); 6686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans assert(rem <= tbin->ncached); 675055f4516c8852e67668b0e746863a7d6a1c148eJason Evans assert(tbin->ncached > 0 || tbin->avail == NULL); 6886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 695055f4516c8852e67668b0e746863a7d6a1c148eJason Evans for (flush = tbin->avail, nflush = tbin->ncached - rem, first_pass = 705055f4516c8852e67668b0e746863a7d6a1c148eJason Evans true; flush != NULL; flush = deferred, nflush = ndeferred) { 7186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* Lock the arena bin associated with the first object. */ 723fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush); 733fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_t *arena = chunk->arena; 7486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin = &arena->bins[binind]; 7586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 76e69bee01de62b56d3e585042d341743239568043Jason Evans#ifdef JEMALLOC_PROF 77d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans if (arena == tcache->arena) { 7886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 79d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, tcache->prof_accumbytes); 8086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 81d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans tcache->prof_accumbytes = 0; 82e69bee01de62b56d3e585042d341743239568043Jason Evans } 83e69bee01de62b56d3e585042d341743239568043Jason Evans#endif 84e69bee01de62b56d3e585042d341743239568043Jason Evans 85e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 86e69bee01de62b56d3e585042d341743239568043Jason Evans#ifdef JEMALLOC_STATS 87e69bee01de62b56d3e585042d341743239568043Jason Evans if (arena == tcache->arena) { 88a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans assert(merged_stats == false); 89a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans merged_stats = true; 9086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nflushes++; 9186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 9286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans tbin->tstats.nrequests = 0; 93d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans } 94d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 953fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans deferred = NULL; 963fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans ndeferred = 0; 973fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans for (i = 0; i < nflush; i++) { 983fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans ptr = flush; 993fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans assert(ptr != NULL); 1003fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans flush = *(void **)ptr; 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->arena == arena) { 1037393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = ((uintptr_t)ptr - 1047393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)chunk) >> PAGE_SHIFT; 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm = 1067393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[pageind-map_bias]; 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_bin(arena, chunk, ptr, mapelm); 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * This object was allocated via a different 11186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * arena bin than the one that is currently 11286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. Stash the object, so that it can be 11386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * handled in a future pass. 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1153fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans *(void **)ptr = deferred; 1163fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans deferred = ptr; 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ndeferred++; 118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 12086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1225055f4516c8852e67668b0e746863a7d6a1c148eJason Evans if (first_pass) { 1233fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->avail = flush; 1245055f4516c8852e67668b0e746863a7d6a1c148eJason Evans first_pass = false; 1253fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans } 126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 127a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans#ifdef JEMALLOC_STATS 128a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans if (merged_stats == false) { 129a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans /* 130a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans * The flush loop didn't happen to flush to this thread's 131a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans * arena, so the stats didn't get merged. Manually do so now. 132a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans */ 133a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans arena_bin_t *bin = &tcache->arena->bins[binind]; 134a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans malloc_mutex_lock(&bin->lock); 135a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans bin->stats.nflushes++; 136a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans bin->stats.nrequests += tbin->tstats.nrequests; 137a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans tbin->tstats.nrequests = 0; 138a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans malloc_mutex_unlock(&bin->lock); 139a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans } 140a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans#endif 141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1423fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->ncached = rem; 14386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (tbin->ncached < tbin->low_water) 14486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans tbin->low_water = tbin->ncached; 145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 147dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid 148dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem 149dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 150dafde14e08ddfda747aabb2045b350848b601b2eJason Evans , tcache_t *tcache 151dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 152dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ) 153dafde14e08ddfda747aabb2045b350848b601b2eJason Evans{ 154dafde14e08ddfda747aabb2045b350848b601b2eJason Evans void *flush, *deferred, *ptr; 155dafde14e08ddfda747aabb2045b350848b601b2eJason Evans unsigned i, nflush, ndeferred; 1565055f4516c8852e67668b0e746863a7d6a1c148eJason Evans bool first_pass; 157dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 158dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(binind < nhbins); 159dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(rem <= tbin->ncached); 1605055f4516c8852e67668b0e746863a7d6a1c148eJason Evans assert(tbin->ncached > 0 || tbin->avail == NULL); 161dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 1625055f4516c8852e67668b0e746863a7d6a1c148eJason Evans for (flush = tbin->avail, nflush = tbin->ncached - rem, first_pass = 1635055f4516c8852e67668b0e746863a7d6a1c148eJason Evans true; flush != NULL; flush = deferred, nflush = ndeferred) { 164dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* Lock the arena associated with the first object. */ 165dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush); 166dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_t *arena = chunk->arena; 167dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 168dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 169dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 170dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (arena == tcache->arena) { 171dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 172dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_PROF 173dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_prof_accum(arena, tcache->prof_accumbytes); 174dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache->prof_accumbytes = 0; 175dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 176dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_STATS 177dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 178dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.lstats[binind - nbins].nrequests += 179dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests; 180dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests = 0; 181dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 182dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 183dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 184dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 185dafde14e08ddfda747aabb2045b350848b601b2eJason Evans deferred = NULL; 186dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ndeferred = 0; 187dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (i = 0; i < nflush; i++) { 188dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ptr = flush; 189dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(ptr != NULL); 190dafde14e08ddfda747aabb2045b350848b601b2eJason Evans flush = *(void **)ptr; 191dafde14e08ddfda747aabb2045b350848b601b2eJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 192dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (chunk->arena == arena) 193dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_dalloc_large(arena, chunk, ptr); 194dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else { 195dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* 196dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * This object was allocated via a different 197dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * arena than the one that is currently locked. 198dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Stash the object, so that it can be handled 199dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * in a future pass. 200dafde14e08ddfda747aabb2045b350848b601b2eJason Evans */ 201dafde14e08ddfda747aabb2045b350848b601b2eJason Evans *(void **)ptr = deferred; 202dafde14e08ddfda747aabb2045b350848b601b2eJason Evans deferred = ptr; 203dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ndeferred++; 204dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 205dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 206dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 207dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 2085055f4516c8852e67668b0e746863a7d6a1c148eJason Evans if (first_pass) { 209dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->avail = flush; 2105055f4516c8852e67668b0e746863a7d6a1c148eJason Evans first_pass = false; 211dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 212dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 213dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 214dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->ncached = rem; 215dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (tbin->ncached < tbin->low_water) 216dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->low_water = tbin->ncached; 217dafde14e08ddfda747aabb2045b350848b601b2eJason Evans} 218dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 219e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_t * 220e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_create(arena_t *arena) 221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache; 2233fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans size_t size; 2243fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans unsigned i; 225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 226c2fc8c8b3afbd15ec3e8ed4ca38667ec0a01ade8Jason Evans size = offsetof(tcache_t, tbins) + (sizeof(tcache_bin_t) * nhbins); 2273fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans /* 2283fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * Round up to the nearest multiple of the cacheline size, in order to 2293fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * avoid the possibility of false cacheline sharing. 2303fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * 2318e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * That this works relies on the same logic as in ipalloc(), but we 2328e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * cannot directly call ipalloc() here due to tcache bootstrapping 2338e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * issues. 2343fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans */ 2353fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans size = (size + CACHELINE_MASK) & (-CACHELINE); 2363fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans 2373fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (size <= small_maxclass) 2383fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache = (tcache_t *)arena_malloc_small(arena, size, true); 2393fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans else 2403fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache = (tcache_t *)icalloc(size); 241e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tcache == NULL) 243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Link into list of extant tcaches. */ 247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_elm_new(tcache, link); 249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_tail_insert(&arena->tcache_ql, tcache, link); 250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 251e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 252e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 253e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache->arena = arena; 254dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); 2553fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans for (i = 0; i < nbins; i++) { 256dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if ((arena->bins[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MAX) { 2573fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache->tbins[i].ncached_max = (arena->bins[i].nregs << 2583fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans 1); 2593fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans } else 260dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache->tbins[i].ncached_max = TCACHE_NSLOTS_SMALL_MAX; 2613fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans } 262dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (; i < nhbins; i++) 263dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache->tbins[i].ncached_max = TCACHE_NSLOTS_LARGE; 264e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2652dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans TCACHE_SET(tcache); 266e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 267e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (tcache); 268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 270e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 271e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_destroy(tcache_t *tcache) 272e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 276e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Unlink from list of extant tcaches. */ 277e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&tcache->arena->lock); 278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_remove(&tcache->arena->tcache_ql, tcache, link); 279e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&tcache->arena->lock); 28086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans tcache_stats_merge(tcache, tcache->arena); 281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < nbins; i++) { 2843fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 285dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_flush_small(tbin, i, 0 28686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 2873fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans , tcache 288d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 2893fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans ); 2903fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans 2913fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans#ifdef JEMALLOC_STATS 2923fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (tbin->tstats.nrequests != 0) { 2933fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_t *arena = tcache->arena; 2943fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_bin_t *bin = &arena->bins[i]; 29586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 2963fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans bin->stats.nrequests += tbin->tstats.nrequests; 29786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2993fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans#endif 300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 302dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (; i < nhbins; i++) { 303dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 304dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_flush_large(tbin, i, 0 305dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 306dafde14e08ddfda747aabb2045b350848b601b2eJason Evans , tcache 307dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 308dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ); 309dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 310dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_STATS 311dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (tbin->tstats.nrequests != 0) { 312dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_t *arena = tcache->arena; 313dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 314dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 315dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.lstats[i - nbins].nrequests += 316dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests; 317dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 318dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 319dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 320dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 321dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 322d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 323d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans if (tcache->prof_accumbytes > 0) { 324d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans malloc_mutex_lock(&tcache->arena->lock); 325d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(tcache->arena, tcache->prof_accumbytes); 326d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans malloc_mutex_unlock(&tcache->arena->lock); 327d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans } 328d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 329d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 330dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (arena_salloc(tcache) <= small_maxclass) { 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena = chunk->arena; 3337393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >> 3347393f44ff025ca67716fc53b68003fd65122fd97Jason Evans PAGE_SHIFT; 3357393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; 33686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 33719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans (uintptr_t)((pageind - (mapelm->bits >> PAGE_SHIFT)) << 33819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)); 33986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin = run->bin; 340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 34186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_bin(arena, chunk, tcache, mapelm); 34386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans idalloc(tcache); 346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 348e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 349e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_thread_cleanup(void *arg) 350e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache = (tcache_t *)arg; 352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 3532dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans if (tcache == (void *)(uintptr_t)1) { 3542dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans /* 3552dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * The previous time this destructor was called, we set the key 3562dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * to 1 so that other destructors wouldn't cause re-creation of 3572dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * the tcache. This time, do nothing, so that the destructor 3582dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * will not be called again. 3592dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans */ 3602dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans } else if (tcache == (void *)(uintptr_t)2) { 3612dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans /* 3622dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * Another destructor called an allocator function after this 3632dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * destructor was called. Reset tcache to 1 in order to 3642dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * receive another callback. 3652dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans */ 3662dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans TCACHE_SET((uintptr_t)1); 3672dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans } else if (tcache != NULL) { 368e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tcache != (void *)(uintptr_t)1); 369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_destroy(tcache); 3702dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans TCACHE_SET((uintptr_t)1); 371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 375e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 376e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_stats_merge(tcache_t *tcache, arena_t *arena) 377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 379e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 380e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Merge and reset tcache stats. */ 38186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans for (i = 0; i < nbins; i++) { 382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin = &arena->bins[i]; 3833fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 38486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 3853fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans bin->stats.nrequests += tbin->tstats.nrequests; 38686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 3873fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->tstats.nrequests = 0; 388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 389dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 390dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (; i < nhbins; i++) { 391dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_large_stats_t *lstats = &arena->stats.lstats[i - nbins]; 392dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 393dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 394dafde14e08ddfda747aabb2045b350848b601b2eJason Evans lstats->nrequests += tbin->tstats.nrequests; 395dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests = 0; 396dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 400e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 401e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_boot(void) 402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 403e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4043fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (opt_tcache) { 405dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* 406e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evans * If necessary, clamp opt_lg_tcache_max, now that 407dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * small_maxclass and arena_maxclass are known. 408dafde14e08ddfda747aabb2045b350848b601b2eJason Evans */ 409e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evans if (opt_lg_tcache_max < 0 || (1U << 410e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evans opt_lg_tcache_max) < small_maxclass) 411dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_maxclass = small_maxclass; 412e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evans else if ((1U << opt_lg_tcache_max) > arena_maxclass) 413dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_maxclass = arena_maxclass; 414dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else 415e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evans tcache_maxclass = (1U << opt_lg_tcache_max); 416dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 417dafde14e08ddfda747aabb2045b350848b601b2eJason Evans nhbins = nbins + (tcache_maxclass >> PAGE_SHIFT); 418dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 419e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Compute incremental GC event threshold. */ 420e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_tcache_gc_sweep >= 0) { 421e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_gc_incr = ((1U << opt_lg_tcache_gc_sweep) / 422e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nbins) + (((1U << opt_lg_tcache_gc_sweep) % nbins == 423e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 0) ? 0 : 1); 424e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 425e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_gc_incr = 0; 426e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 427e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) != 428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 0) { 429698805c5252465ddbdcf7048e5ccf16d02533877Jason Evans malloc_write( 430698805c5252465ddbdcf7048e5ccf16d02533877Jason Evans "<jemalloc>: Error in pthread_key_create()\n"); 431e476f8a161d445211fd6e54fe370275196e66bcbJason Evans abort(); 432e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 433e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 434e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 435e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 436e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_TCACHE */ 437