arena.c revision dafde14e08ddfda747aabb2045b350848b601b2e
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 2e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_TYPES 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct tcache_bin_s tcache_bin_t; 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct tcache_s tcache_t; 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 8e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 9dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Absolute maximum number of cache slots for each small bin in the thread 10dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * cache. This is an additional constraint beyond that imposed as: twice the 11dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * number of regions per run for this size class. 123fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * 133fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * This constant must be an even number. 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 15dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#define TCACHE_NSLOTS_SMALL_MAX 200 16dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 17dafde14e08ddfda747aabb2045b350848b601b2eJason Evans/* Number of cache slots for large size classes. */ 18dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#define TCACHE_NSLOTS_LARGE 20 19dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 20dafde14e08ddfda747aabb2045b350848b601b2eJason Evans/* (1U << opt_lg_tcache_maxclass) is used to compute tcache_maxclass. */ 21dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#define LG_TCACHE_MAXCLASS_DEFAULT 15 22dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 23dafde14e08ddfda747aabb2045b350848b601b2eJason Evans/* 24dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * (1U << opt_lg_tcache_gc_sweep) is the approximate number of allocation 25dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * events between full GC sweeps (-1: disabled). Integer rounding may cause 26dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * the actual number to be slightly higher, since GC is performed 27dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * incrementally. 28dafde14e08ddfda747aabb2045b350848b601b2eJason Evans */ 29dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#define LG_TCACHE_GC_SWEEP_DEFAULT 13 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_TYPES */ 32e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_STRUCTS 34e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 35e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct tcache_bin_s { 36e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_STATS 37e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_bin_stats_t tstats; 38e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned low_water; /* Min # cached since last GC. */ 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned high_water; /* Max # cached since last GC. */ 41e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned ncached; /* # of cached objects. */ 423fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans unsigned ncached_max; /* Upper limit on ncached. */ 433fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans void *avail; /* Chain of available objects. */ 44e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 46e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct tcache_s { 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_STATS 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_elm(tcache_t) link; /* Used for aggregating stats. */ 49e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 50d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans# ifdef JEMALLOC_PROF 51d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans uint64_t prof_accumbytes;/* Cleared after arena_prof_accum() */ 52d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans# endif 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; /* This thread's arena. */ 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned ev_cnt; /* Event count since incremental GC. */ 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned next_gc_bin; /* Next bin to GC. */ 563fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache_bin_t tbins[1]; /* Dynamically sized. */ 57e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 58e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 59e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_STRUCTS */ 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 61e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_EXTERNS 62e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 633fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evansextern bool opt_tcache; 64dafde14e08ddfda747aabb2045b350848b601b2eJason Evansextern ssize_t opt_lg_tcache_maxclass; 65e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern ssize_t opt_lg_tcache_gc_sweep; 66e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Map of thread-specific caches. */ 68e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern __thread tcache_t *tcache_tls 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evans JEMALLOC_ATTR(tls_model("initial-exec")); 70e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 71dafde14e08ddfda747aabb2045b350848b601b2eJason Evans/* 72dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Number of tcache bins. There are nbins small-object bins, plus 0 or more 73dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * large-object bins. 74dafde14e08ddfda747aabb2045b350848b601b2eJason Evans */ 75dafde14e08ddfda747aabb2045b350848b601b2eJason Evansextern size_t nhbins; 76dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 77dafde14e08ddfda747aabb2045b350848b601b2eJason Evans/* Maximum cached size class. */ 78dafde14e08ddfda747aabb2045b350848b601b2eJason Evansextern size_t tcache_maxclass; 79dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Number of tcache allocation/deallocation events between incremental GCs. */ 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern unsigned tcache_gc_incr; 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 83dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem 84dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 85dafde14e08ddfda747aabb2045b350848b601b2eJason Evans , tcache_t *tcache 86dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 87dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ); 88dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem 8986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 90d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , tcache_t *tcache 91d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 92d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ); 93e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_t *tcache_create(arena_t *arena); 94dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid *tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, 95dafde14e08ddfda747aabb2045b350848b601b2eJason Evans size_t binind); 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid tcache_destroy(tcache_t *tcache); 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 98e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid tcache_stats_merge(tcache_t *tcache, arena_t *arena); 99e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 100e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid tcache_boot(void); 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_EXTERNS */ 103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_INLINES 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifndef JEMALLOC_ENABLE_INLINE 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid tcache_event(tcache_t *tcache); 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_t *tcache_get(void); 109dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid *tcache_alloc_easy(tcache_bin_t *tbin); 110dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero); 111dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid *tcache_alloc_large(tcache_t *tcache, size_t size, bool zero); 112dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid tcache_dalloc_small(tcache_t *tcache, void *ptr); 113dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size); 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) 117e476f8a161d445211fd6e54fe370275196e66bcbJason EvansJEMALLOC_INLINE tcache_t * 118e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_get(void) 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache; 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1223fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if ((isthreaded & opt_tcache) == false) 123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache = tcache_tls; 126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((uintptr_t)tcache <= (uintptr_t)1) { 127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tcache == NULL) { 128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache = tcache_create(choose_arena()); 129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tcache == NULL) 130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (tcache); 136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 138e476f8a161d445211fd6e54fe370275196e66bcbJason EvansJEMALLOC_INLINE void 139e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstcache_event(tcache_t *tcache) 140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 142e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tcache_gc_incr == 0) 143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return; 144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache->ev_cnt++; 146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tcache->ev_cnt <= tcache_gc_incr); 1473fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (tcache->ev_cnt == tcache_gc_incr) { 148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind = tcache->next_gc_bin; 1493fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache_bin_t *tbin = &tcache->tbins[binind]; 1503fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans 1513fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (tbin->low_water > 0) { 1523fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans /* 1533fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * Flush (ceiling) 3/4 of the objects below the low 1543fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans * water mark. 1553fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans */ 156dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (binind < nbins) { 157dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_flush_small(tbin, binind, 158dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->ncached - tbin->low_water + 159dafde14e08ddfda747aabb2045b350848b601b2eJason Evans (tbin->low_water >> 2) 160dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 161dafde14e08ddfda747aabb2045b350848b601b2eJason Evans , tcache 162dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 163dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ); 164dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } else { 165dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_flush_large(tbin, binind, 166dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->ncached - tbin->low_water + 167dafde14e08ddfda747aabb2045b350848b601b2eJason Evans (tbin->low_water >> 2) 16886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 169dafde14e08ddfda747aabb2045b350848b601b2eJason Evans , tcache 170d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 171dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ); 172dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1743fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->low_water = tbin->ncached; 1753fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->high_water = tbin->ncached; 176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 177e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache->next_gc_bin++; 178dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (tcache->next_gc_bin == nhbins) 179e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache->next_gc_bin = 0; 180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache->ev_cnt = 0; 181e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 183e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 184e476f8a161d445211fd6e54fe370275196e66bcbJason EvansJEMALLOC_INLINE void * 185dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_alloc_easy(tcache_bin_t *tbin) 186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1873fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans void *ret; 188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tbin->ncached == 0) 190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->ncached--; 192e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tbin->ncached < tbin->low_water) 193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->low_water = tbin->ncached; 1943fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans ret = tbin->avail; 1953fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->avail = *(void **)ret; 1963fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans return (ret); 197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 198e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 199e476f8a161d445211fd6e54fe370275196e66bcbJason EvansJEMALLOC_INLINE void * 200dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_alloc_small(tcache_t *tcache, size_t size, bool zero) 201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 2043fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tcache_bin_t *tbin; 205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 206dafde14e08ddfda747aabb2045b350848b601b2eJason Evans binind = small_size2bin[size]; 207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(binind < nbins); 2083fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin = &tcache->tbins[binind]; 209dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ret = tcache_alloc_easy(tbin); 210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 211dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ret = tcache_alloc_small_hard(tcache, tbin, binind); 212e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) 213e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 215f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans assert(arena_salloc(ret) == tcache->arena->bins[binind].reg_size); 216e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 217e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 219e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 220e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 224e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 226e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->tstats.nrequests++; 229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 230d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 231d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans tcache->prof_accumbytes += tcache->arena->bins[binind].reg_size; 232d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_event(tcache); 234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 237dafde14e08ddfda747aabb2045b350848b601b2eJason EvansJEMALLOC_INLINE void * 238dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_alloc_large(tcache_t *tcache, size_t size, bool zero) 239dafde14e08ddfda747aabb2045b350848b601b2eJason Evans{ 240dafde14e08ddfda747aabb2045b350848b601b2eJason Evans void *ret; 241dafde14e08ddfda747aabb2045b350848b601b2eJason Evans size_t binind; 242dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_t *tbin; 243dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 244dafde14e08ddfda747aabb2045b350848b601b2eJason Evans size = PAGE_CEILING(size); 245dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(size <= tcache_maxclass); 246dafde14e08ddfda747aabb2045b350848b601b2eJason Evans binind = nbins + (size >> PAGE_SHIFT) - 1; 247dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(binind < nhbins); 248dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin = &tcache->tbins[binind]; 249dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ret = tcache_alloc_easy(tbin); 250dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (ret == NULL) { 251dafde14e08ddfda747aabb2045b350848b601b2eJason Evans /* 252dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Only allocate one large object at a time, because it's quite 253dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * expensive to create one and not use it. 254dafde14e08ddfda747aabb2045b350848b601b2eJason Evans */ 255dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ret = arena_malloc_large(tcache->arena, size, zero); 256dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (ret == NULL) 257dafde14e08ddfda747aabb2045b350848b601b2eJason Evans return (NULL); 258dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } else { 259dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (zero == false) { 260dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_FILL 261dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (opt_junk) 262dafde14e08ddfda747aabb2045b350848b601b2eJason Evans memset(ret, 0xa5, size); 263dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else if (opt_zero) 264dafde14e08ddfda747aabb2045b350848b601b2eJason Evans memset(ret, 0, size); 265dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 266dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } else 267dafde14e08ddfda747aabb2045b350848b601b2eJason Evans memset(ret, 0, size); 268dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 269dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_STATS 270dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->tstats.nrequests++; 271dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 272dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_PROF 273dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache->prof_accumbytes += size; 274dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 275dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 276dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 277dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_event(tcache); 278dafde14e08ddfda747aabb2045b350848b601b2eJason Evans return (ret); 279dafde14e08ddfda747aabb2045b350848b601b2eJason Evans} 280dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 281e476f8a161d445211fd6e54fe370275196e66bcbJason EvansJEMALLOC_INLINE void 282dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_dalloc_small(tcache_t *tcache, void *ptr) 283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 287e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_bin_t *tbin; 289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind, binind; 290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm; 291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 293e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 294e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 295e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapelm = &chunk->map[pageind]; 296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((mapelm->bits & CHUNK_MAP_PG_MASK) >> CHUNK_MAP_PG_SHIFT)) << 298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)); 299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = run->bin; 301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ((uintptr_t)bin - (uintptr_t)&arena->bins) / 302e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sizeof(arena_bin_t); 303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(binind < nbins); 304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 3073fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans memset(ptr, 0x5a, bin->reg_size); 308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 3103fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin = &tcache->tbins[binind]; 3113fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (tbin->ncached == tbin->ncached_max) { 312dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_flush_small(tbin, binind, (tbin->ncached_max >> 1) 313dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 314dafde14e08ddfda747aabb2045b350848b601b2eJason Evans , tcache 315dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 316dafde14e08ddfda747aabb2045b350848b601b2eJason Evans ); 317dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 318dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(tbin->ncached < tbin->ncached_max); 319dafde14e08ddfda747aabb2045b350848b601b2eJason Evans *(void **)ptr = tbin->avail; 320dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->avail = ptr; 321dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->ncached++; 322dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (tbin->ncached > tbin->high_water) 323dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin->high_water = tbin->ncached; 324dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 325dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_event(tcache); 326dafde14e08ddfda747aabb2045b350848b601b2eJason Evans} 327dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 328dafde14e08ddfda747aabb2045b350848b601b2eJason EvansJEMALLOC_INLINE void 329dafde14e08ddfda747aabb2045b350848b601b2eJason Evanstcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) 330dafde14e08ddfda747aabb2045b350848b601b2eJason Evans{ 331dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_t *arena; 332dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_chunk_t *chunk; 333dafde14e08ddfda747aabb2045b350848b601b2eJason Evans size_t pageind, binind; 334dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_t *tbin; 335dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_chunk_map_t *mapelm; 336dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 337dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert((size & PAGE_MASK) == 0); 338dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 339dafde14e08ddfda747aabb2045b350848b601b2eJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 340dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena = chunk->arena; 341dafde14e08ddfda747aabb2045b350848b601b2eJason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 342dafde14e08ddfda747aabb2045b350848b601b2eJason Evans mapelm = &chunk->map[pageind]; 343dafde14e08ddfda747aabb2045b350848b601b2eJason Evans binind = nbins + (size >> PAGE_SHIFT) - 1; 344dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 345dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_FILL 346dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (opt_junk) 347dafde14e08ddfda747aabb2045b350848b601b2eJason Evans memset(ptr, 0x5a, bin->reg_size); 348dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 349dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 350dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tbin = &tcache->tbins[binind]; 351dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (tbin->ncached == tbin->ncached_max) { 352dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_bin_flush_large(tbin, binind, (tbin->ncached_max >> 1) 35386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) 354d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , tcache 355d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 356d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ); 3573fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans } 3583fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans assert(tbin->ncached < tbin->ncached_max); 3593fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans *(void **)ptr = tbin->avail; 3603fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->avail = ptr; 361e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->ncached++; 362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tbin->ncached > tbin->high_water) 363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->high_water = tbin->ncached; 364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_event(tcache); 366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 368e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_INLINES */ 370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_TCACHE */ 372