tcache.c revision 1dcb4f86b23a5760f5a717ace716360b63b33fad
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_TCACHE_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 73fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evansbool opt_tcache = true; 8e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evansssize_t opt_lg_tcache_gc_sweep = LG_TCACHE_GC_SWEEP_DEFAULT; 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evanstcache_bin_info_t *tcache_bin_info; 1284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evansstatic unsigned stack_nelms; /* Total stack elms per tcache. */ 1384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Map of thread-specific caches. */ 152dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#ifndef NO_TLS 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evans__thread tcache_t *tcache_tls JEMALLOC_ATTR(tls_model("initial-exec")); 172dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#endif 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Same contents as tcache, but initialized such that the TSD destructor is 21e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * called when a thread exits, so that the cache can be cleaned up. 22e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 232dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evanspthread_key_t tcache_tsd; 24e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 25dafde14e08ddfda747aabb2045b350848b601b2eJason Evanssize_t nhbins; 26dafde14e08ddfda747aabb2045b350848b601b2eJason Evanssize_t tcache_maxclass; 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned tcache_gc_incr; 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Function prototypes for non-inline static functions. */ 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 32e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void tcache_thread_cleanup(void *arg); 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 34e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 35e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 36e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 37dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind) 38e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 41dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_tcache_fill_small(tcache->arena, tbin, binind 42d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 43d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , tcache->prof_accumbytes 44d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 45d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ); 46d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 47d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans tcache->prof_accumbytes = 0; 48d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 49dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ret = tcache_alloc_easy(tbin); 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 55dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem 5686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 57d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , tcache_t *tcache 58d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 59d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ) 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 6184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans void *ptr; 623fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans unsigned i, nflush, ndeferred; 63a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans#ifdef JEMALLOC_STATS 64a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans bool merged_stats = false; 65a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans#endif 66e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 67dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(binind < nbins); 6886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans assert(rem <= tbin->ncached); 6986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 7084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { 7186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* Lock the arena bin associated with the first object. */ 7284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( 7384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tbin->avail[0]); 743fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_t *arena = chunk->arena; 7586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin = &arena->bins[binind]; 7686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 77e69bee01de62b56d3e585042d341743239568043Jason Evans#ifdef JEMALLOC_PROF 78d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans if (arena == tcache->arena) { 7986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 80d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, tcache->prof_accumbytes); 8186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 82d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans tcache->prof_accumbytes = 0; 83e69bee01de62b56d3e585042d341743239568043Jason Evans } 84e69bee01de62b56d3e585042d341743239568043Jason Evans#endif 85e69bee01de62b56d3e585042d341743239568043Jason Evans 86e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 87e69bee01de62b56d3e585042d341743239568043Jason Evans#ifdef JEMALLOC_STATS 88e69bee01de62b56d3e585042d341743239568043Jason Evans if (arena == tcache->arena) { 89a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans assert(merged_stats == false); 90a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans merged_stats = true; 9186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nflushes++; 9286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 9386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans tbin->tstats.nrequests = 0; 94d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans } 95d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 963fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans ndeferred = 0; 973fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans for (i = 0; i < nflush; i++) { 9884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans ptr = tbin->avail[i]; 993fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans assert(ptr != NULL); 100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->arena == arena) { 1027393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = ((uintptr_t)ptr - 1037393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)chunk) >> PAGE_SHIFT; 104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm = 1057393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[pageind-map_bias]; 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_bin(arena, chunk, ptr, mapelm); 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * This object was allocated via a different 11086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * arena bin than the one that is currently 11186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. Stash the object, so that it can be 11286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * handled in a future pass. 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 11484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tbin->avail[ndeferred] = ptr; 115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ndeferred++; 116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 11886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 120a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans#ifdef JEMALLOC_STATS 121a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans if (merged_stats == false) { 122a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans /* 123a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans * The flush loop didn't happen to flush to this thread's 124a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans * arena, so the stats didn't get merged. Manually do so now. 125a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans */ 126a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans arena_bin_t *bin = &tcache->arena->bins[binind]; 127a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans malloc_mutex_lock(&bin->lock); 128a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans bin->stats.nflushes++; 129a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans bin->stats.nrequests += tbin->tstats.nrequests; 130a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans tbin->tstats.nrequests = 0; 131a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans malloc_mutex_unlock(&bin->lock); 132a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans } 133a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans#endif 134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 13584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans memmove(tbin->avail, &tbin->avail[tbin->ncached - rem], 13684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans rem * sizeof(void *)); 1373fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->ncached = rem; 1381dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans if ((int)tbin->ncached < tbin->low_water) 13986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans tbin->low_water = tbin->ncached; 140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 142dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid 143dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem 144dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 145dafde14e08ddfda747aabb2045b350848b601b2eJason Evans , tcache_t *tcache 146dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 147dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ) 148dafde14e08ddfda747aabb2045b350848b601b2eJason Evans{ 14984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans void *ptr; 150dafde14e08ddfda747aabb2045b350848b601b2eJason Evans unsigned i, nflush, ndeferred; 15184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans#ifdef JEMALLOC_STATS 15284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bool merged_stats = false; 15384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans#endif 154dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 155dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(binind < nhbins); 156dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(rem <= tbin->ncached); 157dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 15884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { 159dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* Lock the arena associated with the first object. */ 16084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( 16184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tbin->avail[0]); 162dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_t *arena = chunk->arena; 163dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 164dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 165dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 166dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (arena == tcache->arena) { 167dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 168dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_PROF 169dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_prof_accum(arena, tcache->prof_accumbytes); 170dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache->prof_accumbytes = 0; 171dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 172dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_STATS 17384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans merged_stats = true; 174dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 175dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.lstats[binind - nbins].nrequests += 176dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests; 177dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests = 0; 178dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 179dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) 180dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 181dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 182dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ndeferred = 0; 183dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (i = 0; i < nflush; i++) { 18484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans ptr = tbin->avail[i]; 185dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(ptr != NULL); 186dafde14e08ddfda747aabb2045b350848b601b2eJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 187dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (chunk->arena == arena) 188dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_dalloc_large(arena, chunk, ptr); 189dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else { 190dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* 191dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * This object was allocated via a different 192dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * arena than the one that is currently locked. 193dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Stash the object, so that it can be handled 194dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * in a future pass. 195dafde14e08ddfda747aabb2045b350848b601b2eJason Evans */ 19684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tbin->avail[ndeferred] = ptr; 197dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ndeferred++; 198dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 199dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 200dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 201dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 20284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans#ifdef JEMALLOC_STATS 20384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans if (merged_stats == false) { 20484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 20584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * The flush loop didn't happen to flush to this thread's 20684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * arena, so the stats didn't get merged. Manually do so now. 20784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 20884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena_t *arena = tcache->arena; 20984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans malloc_mutex_lock(&arena->lock); 21084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 21184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena->stats.lstats[binind - nbins].nrequests += 21284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tbin->tstats.nrequests; 21384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tbin->tstats.nrequests = 0; 21484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans malloc_mutex_unlock(&arena->lock); 21584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans } 21684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans#endif 217dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 21884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans memmove(tbin->avail, &tbin->avail[tbin->ncached - rem], 21984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans rem * sizeof(void *)); 220dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->ncached = rem; 2211dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans if ((int)tbin->ncached < tbin->low_water) 222dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->low_water = tbin->ncached; 223dafde14e08ddfda747aabb2045b350848b601b2eJason Evans} 224dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 225e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_t * 226e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_create(arena_t *arena) 227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache; 22984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans size_t size, stack_offset; 2303fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans unsigned i; 231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 232c2fc8c8b3afbd15ec3e8ed4ca38667ec0a01ade8Jason Evans size = offsetof(tcache_t, tbins) + (sizeof(tcache_bin_t) * nhbins); 23384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Naturally align the pointer stacks. */ 23484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans size = PTR_CEILING(size); 23584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans stack_offset = size; 23684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans size += stack_nelms * sizeof(void *); 2373fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans /* 2383fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * Round up to the nearest multiple of the cacheline size, in order to 2393fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * avoid the possibility of false cacheline sharing. 2403fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * 2418e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * That this works relies on the same logic as in ipalloc(), but we 2428e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * cannot directly call ipalloc() here due to tcache bootstrapping 2438e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * issues. 2443fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans */ 2453fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans size = (size + CACHELINE_MASK) & (-CACHELINE); 2463fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans 2473fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (size <= small_maxclass) 2483fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache = (tcache_t *)arena_malloc_small(arena, size, true); 24984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans else if (size <= tcache_maxclass) 25084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tcache = (tcache_t *)arena_malloc_large(arena, size, true); 2513fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans else 2523fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache = (tcache_t *)icalloc(size); 253e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tcache == NULL) 255e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 256e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 257e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 258e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Link into list of extant tcaches. */ 259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 260e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_elm_new(tcache, link); 261e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_tail_insert(&arena->tcache_ql, tcache, link); 262e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 263e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 264e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 265e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache->arena = arena; 266dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); 26784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans for (i = 0; i < nhbins; i++) { 2681dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans tcache->tbins[i].lg_fill_div = 1; 26984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tcache->tbins[i].avail = (void **)((uintptr_t)tcache + 27084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)stack_offset); 27184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); 2723fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans } 273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2742dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans TCACHE_SET(tcache); 275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 276e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (tcache); 277e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 279e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 280e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_destroy(tcache_t *tcache) 281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 28384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans size_t tcache_size; 284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Unlink from list of extant tcaches. */ 287e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&tcache->arena->lock); 288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_remove(&tcache->arena->tcache_ql, tcache, link); 289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&tcache->arena->lock); 29086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans tcache_stats_merge(tcache, tcache->arena); 291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 293e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < nbins; i++) { 2943fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 295dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_flush_small(tbin, i, 0 29686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 2973fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans , tcache 298d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 2993fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans ); 3003fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans 3013fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans#ifdef JEMALLOC_STATS 3023fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (tbin->tstats.nrequests != 0) { 3033fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_t *arena = tcache->arena; 3043fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_bin_t *bin = &arena->bins[i]; 30586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 3063fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans bin->stats.nrequests += tbin->tstats.nrequests; 30786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 3093fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans#endif 310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 312dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (; i < nhbins; i++) { 313dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 314dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_flush_large(tbin, i, 0 315dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 316dafde14e08ddfda747aabb2045b350848b601b2eJason Evans , tcache 317dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 318dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ); 319dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 320dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_STATS 321dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (tbin->tstats.nrequests != 0) { 322dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_t *arena = tcache->arena; 323dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 324dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 325dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.lstats[i - nbins].nrequests += 326dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests; 327dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 328dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 329dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 330dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 331dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 332d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 333d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans if (tcache->prof_accumbytes > 0) { 334d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans malloc_mutex_lock(&tcache->arena->lock); 335d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(tcache->arena, tcache->prof_accumbytes); 336d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans malloc_mutex_unlock(&tcache->arena->lock); 337d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans } 338d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 339d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 34084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tcache_size = arena_salloc(tcache); 34184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans if (tcache_size <= small_maxclass) { 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); 343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena = chunk->arena; 3447393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >> 3457393f44ff025ca67716fc53b68003fd65122fd97Jason Evans PAGE_SHIFT; 3467393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; 34786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 34819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans (uintptr_t)((pageind - (mapelm->bits >> PAGE_SHIFT)) << 34919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)); 35086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin = run->bin; 351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 35286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_bin(arena, chunk, tcache, mapelm); 35486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 35584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans } else if (tcache_size <= tcache_maxclass) { 35684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); 35784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena_t *arena = chunk->arena; 35884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 35984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans malloc_mutex_lock(&arena->lock); 36084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena_dalloc_large(arena, chunk, tcache); 36184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans malloc_mutex_unlock(&arena->lock); 362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans idalloc(tcache); 364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 366e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 367e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_thread_cleanup(void *arg) 368e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache = (tcache_t *)arg; 370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 3712dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans if (tcache == (void *)(uintptr_t)1) { 3722dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans /* 3732dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * The previous time this destructor was called, we set the key 3742dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * to 1 so that other destructors wouldn't cause re-creation of 3752dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * the tcache. This time, do nothing, so that the destructor 3762dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * will not be called again. 3772dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans */ 3782dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans } else if (tcache == (void *)(uintptr_t)2) { 3792dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans /* 3802dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * Another destructor called an allocator function after this 3812dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * destructor was called. Reset tcache to 1 in order to 3822dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * receive another callback. 3832dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans */ 3842dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans TCACHE_SET((uintptr_t)1); 3852dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans } else if (tcache != NULL) { 386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tcache != (void *)(uintptr_t)1); 387e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_destroy(tcache); 3882dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans TCACHE_SET((uintptr_t)1); 389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 390e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 392e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 393e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 394e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_stats_merge(tcache_t *tcache, arena_t *arena) 395e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Merge and reset tcache stats. */ 39986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans for (i = 0; i < nbins; i++) { 400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin = &arena->bins[i]; 4013fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 40286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 4033fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans bin->stats.nrequests += tbin->tstats.nrequests; 40486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 4053fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->tstats.nrequests = 0; 406e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 407dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 408dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (; i < nhbins; i++) { 409dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_large_stats_t *lstats = &arena->stats.lstats[i - nbins]; 410dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 411dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 412dafde14e08ddfda747aabb2045b350848b601b2eJason Evans lstats->nrequests += tbin->tstats.nrequests; 413dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests = 0; 414dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 415e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 416e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 417e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 41884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evansbool 419e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_boot(void) 420e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 421e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4223fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (opt_tcache) { 42384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned i; 42484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 425dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* 426e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evans * If necessary, clamp opt_lg_tcache_max, now that 427dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * small_maxclass and arena_maxclass are known. 428dafde14e08ddfda747aabb2045b350848b601b2eJason Evans */ 429e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evans if (opt_lg_tcache_max < 0 || (1U << 430e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evans opt_lg_tcache_max) < small_maxclass) 431dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_maxclass = small_maxclass; 432e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evans else if ((1U << opt_lg_tcache_max) > arena_maxclass) 433dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_maxclass = arena_maxclass; 434dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else 435e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evans tcache_maxclass = (1U << opt_lg_tcache_max); 436dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 437dafde14e08ddfda747aabb2045b350848b601b2eJason Evans nhbins = nbins + (tcache_maxclass >> PAGE_SHIFT); 438dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 43984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Initialize tcache_bin_info. */ 44084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins * 44184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans sizeof(tcache_bin_info_t)); 44284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans if (tcache_bin_info == NULL) 44384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans return (true); 44484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans stack_nelms = 0; 44584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans for (i = 0; i < nbins; i++) { 44684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans if ((arena_bin_info[i].nregs << 1) <= 44784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans TCACHE_NSLOTS_SMALL_MAX) { 44884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tcache_bin_info[i].ncached_max = 44984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (arena_bin_info[i].nregs << 1); 45084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans } else { 45184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tcache_bin_info[i].ncached_max = 45284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans TCACHE_NSLOTS_SMALL_MAX; 45384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans } 45484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans stack_nelms += tcache_bin_info[i].ncached_max; 45584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans } 45684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans for (; i < nhbins; i++) { 45784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE; 45884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans stack_nelms += tcache_bin_info[i].ncached_max; 45984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans } 46084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Compute incremental GC event threshold. */ 462e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_tcache_gc_sweep >= 0) { 463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_gc_incr = ((1U << opt_lg_tcache_gc_sweep) / 464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nbins) + (((1U << opt_lg_tcache_gc_sweep) % nbins == 465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 0) ? 0 : 1); 466e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 467e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_gc_incr = 0; 468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) != 470e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 0) { 471698805c5252465ddbdcf7048e5ccf16d02533877Jason Evans malloc_write( 472698805c5252465ddbdcf7048e5ccf16d02533877Jason Evans "<jemalloc>: Error in pthread_key_create()\n"); 473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans abort(); 474e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 475e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 47684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 47784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans return (false); 478e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 479e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 480e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_TCACHE */ 481