1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_TCACHE_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evansmalloc_tsd_data(, tcache, tcache_t *, NULL) 8d4be8b7b6ee2e21d079180455d4ccbf45cc1cee7Jason Evansmalloc_tsd_data(, tcache_enabled, tcache_enabled_t, tcache_enabled_default) 9cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 103fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evansbool opt_tcache = true; 11e73397062ac3ab28a9d377591b63ed19fd154ccaJason Evansssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; 12e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evanstcache_bin_info_t *tcache_bin_info; 1484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evansstatic unsigned stack_nelms; /* Total stack elms per tcache. */ 1584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 16cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evanssize_t nhbins; 17cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evanssize_t tcache_maxclass; 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 21f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evanssize_t tcache_salloc(const void *ptr) 22f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans{ 23f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 24f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans return (arena_salloc(ptr, false)); 25f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans} 26f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 27203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid 28203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanstcache_event_hard(tcache_t *tcache) 29203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 30203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind = tcache->next_gc_bin; 31203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tcache_bin_t *tbin = &tcache->tbins[binind]; 32203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tcache_bin_info_t *tbin_info = &tcache_bin_info[binind]; 33203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 34203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (tbin->low_water > 0) { 35203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans /* 36203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * Flush (ceiling) 3/4 of the objects below the low water mark. 37203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans */ 38203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (binind < NBINS) { 39203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tcache_bin_flush_small(tbin, binind, tbin->ncached - 40203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tbin->low_water + (tbin->low_water >> 2), tcache); 41203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } else { 42203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tcache_bin_flush_large(tbin, binind, tbin->ncached - 43203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tbin->low_water + (tbin->low_water >> 2), tcache); 44203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 45203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans /* 46203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * Reduce fill count by 2X. Limit lg_fill_div such that the 47203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * fill count is always at least 1. 48203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans */ 49203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) >= 1) 50203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tbin->lg_fill_div++; 51203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } else if (tbin->low_water < 0) { 52203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans /* 53203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * Increase fill count by 2X. Make sure lg_fill_div stays 54203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * greater than 0. 55203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans */ 56203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (tbin->lg_fill_div > 1) 57203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tbin->lg_fill_div--; 58203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 59203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tbin->low_water = tbin->ncached; 60203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 61203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tcache->next_gc_bin++; 62203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (tcache->next_gc_bin == nhbins) 63203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tcache->next_gc_bin = 0; 64203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tcache->ev_cnt = 0; 65203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 66203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 68dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind) 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 70e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 71e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 727372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena_tcache_fill_small(tcache->arena, tbin, binind, 737372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans config_prof ? tcache->prof_accumbytes : 0); 747372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) 757372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tcache->prof_accumbytes = 0; 76dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ret = tcache_alloc_easy(tbin); 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 827372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evanstcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, 837372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tcache_t *tcache) 84e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 8584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans void *ptr; 863fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans unsigned i, nflush, ndeferred; 87a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans bool merged_stats = false; 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 89b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 9086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans assert(rem <= tbin->ncached); 9186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 9284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { 9386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* Lock the arena bin associated with the first object. */ 9484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( 9584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tbin->avail[0]); 963fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_t *arena = chunk->arena; 9786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin = &arena->bins[binind]; 9886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 997372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof && arena == tcache->arena) { 10088c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans if (arena_prof_accum(arena, tcache->prof_accumbytes)) 10188c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans prof_idump(); 102d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans tcache->prof_accumbytes = 0; 103e69bee01de62b56d3e585042d341743239568043Jason Evans } 104e69bee01de62b56d3e585042d341743239568043Jason Evans 105e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 1067372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats && arena == tcache->arena) { 107a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans assert(merged_stats == false); 108a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans merged_stats = true; 10986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nflushes++; 11086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 11186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans tbin->tstats.nrequests = 0; 112d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans } 1133fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans ndeferred = 0; 1143fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans for (i = 0; i < nflush; i++) { 11584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans ptr = tbin->avail[i]; 1163fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans assert(ptr != NULL); 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->arena == arena) { 1197393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = ((uintptr_t)ptr - 120ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (uintptr_t)chunk) >> LG_PAGE; 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm = 122203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapp_get(chunk, pageind); 123122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (config_fill && opt_junk) { 124122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_alloc_junk_small(ptr, 125122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans &arena_bin_info[binind], true); 126122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 127203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_bin_locked(arena, chunk, ptr, 128203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapelm); 129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * This object was allocated via a different 13286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * arena bin than the one that is currently 13386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. Stash the object, so that it can be 13486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * handled in a future pass. 135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 13684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tbin->avail[ndeferred] = ptr; 137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ndeferred++; 138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 139e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 14086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1427372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats && merged_stats == false) { 143a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans /* 144a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans * The flush loop didn't happen to flush to this thread's 145a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans * arena, so the stats didn't get merged. Manually do so now. 146a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans */ 147a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans arena_bin_t *bin = &tcache->arena->bins[binind]; 148a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans malloc_mutex_lock(&bin->lock); 149a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans bin->stats.nflushes++; 150a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans bin->stats.nrequests += tbin->tstats.nrequests; 151a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans tbin->tstats.nrequests = 0; 152a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans malloc_mutex_unlock(&bin->lock); 153a8118233ec0369e00629fe853a5655c0dabf83d2Jason Evans } 154e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 15584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans memmove(tbin->avail, &tbin->avail[tbin->ncached - rem], 15684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans rem * sizeof(void *)); 1573fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->ncached = rem; 1581dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans if ((int)tbin->ncached < tbin->low_water) 15986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans tbin->low_water = tbin->ncached; 160e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 162dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid 1637372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evanstcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem, 1647372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tcache_t *tcache) 165dafde14e08ddfda747aabb2045b350848b601b2eJason Evans{ 16684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans void *ptr; 167dafde14e08ddfda747aabb2045b350848b601b2eJason Evans unsigned i, nflush, ndeferred; 16884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bool merged_stats = false; 169dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 170dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(binind < nhbins); 171dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(rem <= tbin->ncached); 172dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 17384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { 174dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* Lock the arena associated with the first object. */ 17584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( 17684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tbin->avail[0]); 177dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_t *arena = chunk->arena; 17888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans UNUSED bool idump; 179dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 18088c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans if (config_prof) 18188c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans idump = false; 182dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 1837372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if ((config_prof || config_stats) && arena == tcache->arena) { 1847372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) { 18588c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans idump = arena_prof_accum_locked(arena, 1867372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tcache->prof_accumbytes); 1877372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tcache->prof_accumbytes = 0; 1887372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1897372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 1907372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans merged_stats = true; 1917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large += 1927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tbin->tstats.nrequests; 193b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans arena->stats.lstats[binind - NBINS].nrequests += 1947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tbin->tstats.nrequests; 1957372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tbin->tstats.nrequests = 0; 1967372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 197dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 198dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ndeferred = 0; 199dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (i = 0; i < nflush; i++) { 20084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans ptr = tbin->avail[i]; 201dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(ptr != NULL); 202dafde14e08ddfda747aabb2045b350848b601b2eJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 203dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (chunk->arena == arena) 204203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_large_locked(arena, chunk, ptr); 205dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else { 206dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* 207dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * This object was allocated via a different 208dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * arena than the one that is currently locked. 209dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Stash the object, so that it can be handled 210dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * in a future pass. 211dafde14e08ddfda747aabb2045b350848b601b2eJason Evans */ 21284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tbin->avail[ndeferred] = ptr; 213dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ndeferred++; 214dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 215dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 216dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 21788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans if (config_prof && idump) 21888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans prof_idump(); 219dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 2207372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats && merged_stats == false) { 22184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 22284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * The flush loop didn't happen to flush to this thread's 22384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * arena, so the stats didn't get merged. Manually do so now. 22484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 22584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena_t *arena = tcache->arena; 22684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans malloc_mutex_lock(&arena->lock); 22784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 228b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans arena->stats.lstats[binind - NBINS].nrequests += 22984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tbin->tstats.nrequests; 23084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tbin->tstats.nrequests = 0; 23184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans malloc_mutex_unlock(&arena->lock); 23284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans } 233dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 23484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans memmove(tbin->avail, &tbin->avail[tbin->ncached - rem], 23584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans rem * sizeof(void *)); 236dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->ncached = rem; 2371dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans if ((int)tbin->ncached < tbin->low_water) 238dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->low_water = tbin->ncached; 239dafde14e08ddfda747aabb2045b350848b601b2eJason Evans} 240dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 241cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evansvoid 242cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evanstcache_arena_associate(tcache_t *tcache, arena_t *arena) 243cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans{ 244cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 245cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans if (config_stats) { 246cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans /* Link into list of extant tcaches. */ 247cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans malloc_mutex_lock(&arena->lock); 248cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans ql_elm_new(tcache, link); 249cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans ql_tail_insert(&arena->tcache_ql, tcache, link); 250cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans malloc_mutex_unlock(&arena->lock); 251cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans } 252cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans tcache->arena = arena; 253cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans} 254cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 255cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evansvoid 256cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evanstcache_arena_dissociate(tcache_t *tcache) 257cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans{ 258cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 259cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans if (config_stats) { 260cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans /* Unlink from list of extant tcaches. */ 261cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans malloc_mutex_lock(&tcache->arena->lock); 262cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans ql_remove(&tcache->arena->tcache_ql, tcache, link); 263cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans tcache_stats_merge(tcache, tcache->arena); 26430e7cb11186554eb3ee860856eb5b8d541d7740cJason Evans malloc_mutex_unlock(&tcache->arena->lock); 265cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans } 266cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans} 267cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 268e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_t * 269a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurertcache_get_hard(tcache_t *tcache, bool create) 270a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer{ 271a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer 272a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer if (tcache == NULL) { 273a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer if (create == false) { 274a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer /* 275a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer * Creating a tcache here would cause 276a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer * allocation as a side effect of free(). 277a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer * Ordinarily that would be okay since 278a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer * tcache_create() failure is a soft failure 279a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer * that doesn't propagate. However, if TLS 280a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer * data are freed via free() as in glibc, 281a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer * subtle corruption could result from setting 282a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer * a TLS variable after its backing memory is 283a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer * freed. 284a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer */ 285a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer return (NULL); 286a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer } 287a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer if (tcache_enabled_get() == false) { 288a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer tcache_enabled_set(false); /* Memoize. */ 289a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer return (NULL); 290a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer } 291a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer return (tcache_create(choose_arena(NULL))); 292a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer } 293a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer if (tcache == TCACHE_STATE_PURGATORY) { 294a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer /* 295a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer * Make a note that an allocator function was called 296a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer * after tcache_thread_cleanup() was called. 297a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer */ 298a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer tcache = TCACHE_STATE_REINCARNATED; 299a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer tcache_tsd_set(&tcache); 300a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer return (NULL); 301a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer } 302a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer if (tcache == TCACHE_STATE_REINCARNATED) 303a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer return (NULL); 304a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer not_reached(); 305a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer return (NULL); 306a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer} 307a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurer 308a7619b7fa56f98d1ca99a23b458696dd37c12b77Ben Maurertcache_t * 309e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_create(arena_t *arena) 310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache; 31284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans size_t size, stack_offset; 3133fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans unsigned i; 314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 315c2fc8c8b3afbd15ec3e8ed4ca38667ec0a01ade8Jason Evans size = offsetof(tcache_t, tbins) + (sizeof(tcache_bin_t) * nhbins); 31684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Naturally align the pointer stacks. */ 31784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans size = PTR_CEILING(size); 31884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans stack_offset = size; 31984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans size += stack_nelms * sizeof(void *); 3203fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans /* 3213fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * Round up to the nearest multiple of the cacheline size, in order to 3223fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * avoid the possibility of false cacheline sharing. 3233fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * 3248e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * That this works relies on the same logic as in ipalloc(), but we 3258e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * cannot directly call ipalloc() here due to tcache bootstrapping 3268e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * issues. 3273fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans */ 3283fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans size = (size + CACHELINE_MASK) & (-CACHELINE); 3293fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans 330b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (size <= SMALL_MAXCLASS) 3313fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache = (tcache_t *)arena_malloc_small(arena, size, true); 33284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans else if (size <= tcache_maxclass) 33384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tcache = (tcache_t *)arena_malloc_large(arena, size, true); 3343fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans else 335d82a5e6a34f20698ab9368bb2b4953b81d175552Jason Evans tcache = (tcache_t *)icalloct(size, false, arena); 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tcache == NULL) 338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 340cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans tcache_arena_associate(tcache, arena); 341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 342dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); 34384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans for (i = 0; i < nhbins; i++) { 3441dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans tcache->tbins[i].lg_fill_div = 1; 34584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans tcache->tbins[i].avail = (void **)((uintptr_t)tcache + 34684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)stack_offset); 34784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); 3483fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans } 349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 350cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans tcache_tsd_set(&tcache); 351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (tcache); 353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 355e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 356e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_destroy(tcache_t *tcache) 357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 35984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans size_t tcache_size; 360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 361cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans tcache_arena_dissociate(tcache); 362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 363b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans for (i = 0; i < NBINS; i++) { 3643fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 3657372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tcache_bin_flush_small(tbin, i, 0, tcache); 3663fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans 3677372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats && tbin->tstats.nrequests != 0) { 3683fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_t *arena = tcache->arena; 3693fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans arena_bin_t *bin = &arena->bins[i]; 37086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 3713fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans bin->stats.nrequests += tbin->tstats.nrequests; 37286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 376dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (; i < nhbins; i++) { 377dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 3787372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tcache_bin_flush_large(tbin, i, 0, tcache); 379dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 3807372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats && tbin->tstats.nrequests != 0) { 381dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_t *arena = tcache->arena; 382dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 383dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 384b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans arena->stats.lstats[i - NBINS].nrequests += 385dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests; 386dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 387dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 388dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 389dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 39088c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans if (config_prof && tcache->prof_accumbytes > 0 && 39188c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans arena_prof_accum(tcache->arena, tcache->prof_accumbytes)) 39288c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans prof_idump(); 393d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 394122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans tcache_size = arena_salloc(tcache, false); 395b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (tcache_size <= SMALL_MAXCLASS) { 396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); 397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena = chunk->arena; 3987393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >> 399ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE; 400203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 402203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_bin(arena, chunk, tcache, pageind, mapelm); 40384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans } else if (tcache_size <= tcache_maxclass) { 40484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); 40584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena_t *arena = chunk->arena; 40684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 40784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans arena_dalloc_large(arena, chunk, tcache); 408e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 409d82a5e6a34f20698ab9368bb2b4953b81d175552Jason Evans idalloct(tcache, false); 410e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 411e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 412cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evansvoid 413e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_thread_cleanup(void *arg) 414e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 415cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans tcache_t *tcache = *(tcache_t **)arg; 416e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 417d4be8b7b6ee2e21d079180455d4ccbf45cc1cee7Jason Evans if (tcache == TCACHE_STATE_DISABLED) { 418d4be8b7b6ee2e21d079180455d4ccbf45cc1cee7Jason Evans /* Do nothing. */ 419d4be8b7b6ee2e21d079180455d4ccbf45cc1cee7Jason Evans } else if (tcache == TCACHE_STATE_REINCARNATED) { 4202dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans /* 4212dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * Another destructor called an allocator function after this 4223701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans * destructor was called. Reset tcache to 4233701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans * TCACHE_STATE_PURGATORY in order to receive another callback. 4242dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans */ 425d4be8b7b6ee2e21d079180455d4ccbf45cc1cee7Jason Evans tcache = TCACHE_STATE_PURGATORY; 426cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans tcache_tsd_set(&tcache); 427d4be8b7b6ee2e21d079180455d4ccbf45cc1cee7Jason Evans } else if (tcache == TCACHE_STATE_PURGATORY) { 428d4be8b7b6ee2e21d079180455d4ccbf45cc1cee7Jason Evans /* 429d4be8b7b6ee2e21d079180455d4ccbf45cc1cee7Jason Evans * The previous time this destructor was called, we set the key 4303701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans * to TCACHE_STATE_PURGATORY so that other destructors wouldn't 4313701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans * cause re-creation of the tcache. This time, do nothing, so 4323701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans * that the destructor will not be called again. 433d4be8b7b6ee2e21d079180455d4ccbf45cc1cee7Jason Evans */ 4342dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans } else if (tcache != NULL) { 435d4be8b7b6ee2e21d079180455d4ccbf45cc1cee7Jason Evans assert(tcache != TCACHE_STATE_PURGATORY); 436e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_destroy(tcache); 437d4be8b7b6ee2e21d079180455d4ccbf45cc1cee7Jason Evans tcache = TCACHE_STATE_PURGATORY; 438cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans tcache_tsd_set(&tcache); 439e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 440e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 44230e7cb11186554eb3ee860856eb5b8d541d7740cJason Evans/* Caller must own arena->lock. */ 443e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 444e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_stats_merge(tcache_t *tcache, arena_t *arena) 445e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 447e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 44830e7cb11186554eb3ee860856eb5b8d541d7740cJason Evans cassert(config_stats); 44930e7cb11186554eb3ee860856eb5b8d541d7740cJason Evans 450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Merge and reset tcache stats. */ 451b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans for (i = 0; i < NBINS; i++) { 452e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin = &arena->bins[i]; 4533fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 45486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 4553fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans bin->stats.nrequests += tbin->tstats.nrequests; 45686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 4573fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->tstats.nrequests = 0; 458e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 459dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 460dafde14e08ddfda747aabb2045b350848b601b2eJason Evans for (; i < nhbins; i++) { 461b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans malloc_large_stats_t *lstats = &arena->stats.lstats[i - NBINS]; 462dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_t *tbin = &tcache->tbins[i]; 463dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena->stats.nrequests_large += tbin->tstats.nrequests; 464dafde14e08ddfda747aabb2045b350848b601b2eJason Evans lstats->nrequests += tbin->tstats.nrequests; 465dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests = 0; 466dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 467e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 46984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evansbool 470cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evanstcache_boot0(void) 471e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 4723701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans unsigned i; 473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4743701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans /* 4753701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans * If necessary, clamp opt_lg_tcache_max, now that arena_maxclass is 4763701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans * known. 4773701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans */ 4783701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans if (opt_lg_tcache_max < 0 || (1U << opt_lg_tcache_max) < SMALL_MAXCLASS) 4793701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans tcache_maxclass = SMALL_MAXCLASS; 4803701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans else if ((1U << opt_lg_tcache_max) > arena_maxclass) 4813701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans tcache_maxclass = arena_maxclass; 4823701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans else 4833701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans tcache_maxclass = (1U << opt_lg_tcache_max); 48484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 4853701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans nhbins = NBINS + (tcache_maxclass >> LG_PAGE); 4863701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans 4873701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans /* Initialize tcache_bin_info. */ 4883701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins * 4893701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans sizeof(tcache_bin_info_t)); 4903701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans if (tcache_bin_info == NULL) 4913701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans return (true); 4923701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans stack_nelms = 0; 4933701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans for (i = 0; i < NBINS; i++) { 4943701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans if ((arena_bin_info[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MAX) { 4953701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans tcache_bin_info[i].ncached_max = 4963701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans (arena_bin_info[i].nregs << 1); 4973701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans } else { 4983701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans tcache_bin_info[i].ncached_max = 4993701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans TCACHE_NSLOTS_SMALL_MAX; 50084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans } 5013701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans stack_nelms += tcache_bin_info[i].ncached_max; 5023701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans } 5033701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans for (; i < nhbins; i++) { 5043701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE; 5053701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans stack_nelms += tcache_bin_info[i].ncached_max; 506cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans } 50784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 508cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans return (false); 509cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans} 510cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 511cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evansbool 512cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evanstcache_boot1(void) 513cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans{ 514cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans 5153701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans if (tcache_tsd_boot() || tcache_enabled_tsd_boot()) 5163701367e4ca6b77109e1cce0a5b98a8ac69cf505Jason Evans return (true); 51784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 51884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans return (false); 519e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 520