arena.c revision 86815df9dc7d2418a21c87b3dc9747ab42dea73d
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_ARENA_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; 8e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t opt_lg_medium_max = LG_MEDIUM_MAX_DEFAULT; 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evansssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 11e476f8a161d445211fd6e54fe370275196e66bcbJason Evansuint8_t const *small_size2bin; 12e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Various bin-related settings. */ 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nqbins; 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned ncbins; 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nsbins; 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nmbins; 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nbins; 19e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned mbin0; 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t qspace_max; 21e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t cspace_min; 22e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t cspace_max; 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t sspace_min; 24e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t sspace_max; 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t medium_max; 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t lg_mspace; 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t mspace_mask; 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 302caa4715ed4f787f263239ff97dd824636289286Jason Evans/* Used to prevent threads from concurrently calling madvise(2). */ 312caa4715ed4f787f263239ff97dd824636289286Jason Evansstatic malloc_mutex_t purge_lock; 322caa4715ed4f787f263239ff97dd824636289286Jason Evans 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 34e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * const_small_size2bin is a static constant lookup table that in the common 35e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * case can be used as-is for small_size2bin. For dynamically linked programs, 36e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * this avoids a page of memory overhead per process. 37e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 38e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_1(i) i, 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_2(i) S2B_1(i) S2B_1(i) 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_4(i) S2B_2(i) S2B_2(i) 41e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_8(i) S2B_4(i) S2B_4(i) 42e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_16(i) S2B_8(i) S2B_8(i) 43e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_32(i) S2B_16(i) S2B_16(i) 44e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_64(i) S2B_32(i) S2B_32(i) 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_128(i) S2B_64(i) S2B_64(i) 46e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_256(i) S2B_128(i) S2B_128(i) 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The number of elements in const_small_size2bin is dependent on page size 49e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * and on the definition for SUBPAGE. If SUBPAGE changes, the '- 255' must also 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * change, along with the addition/removal of static lookup table element 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * definitions. 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic const uint8_t const_small_size2bin[STATIC_PAGE_SIZE - 255] = { 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_1(0xffU) /* 0 */ 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (LG_QUANTUM == 4) 563fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans/* 16-byte quantum **********************/ 57e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TINY 581e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans# if (LG_TINY_MIN == 2) 593fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_4(0) /* 4 */ 603fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_4(1) /* 8 */ 613fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_8(2) /* 16 */ 621e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans# define S2B_QMIN 2 633fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# elif (LG_TINY_MIN == 3) 643fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_8(0) /* 8 */ 653fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_8(1) /* 16 */ 661e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans# define S2B_QMIN 1 673fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# else 683fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# error "Unsupported LG_TINY_MIN" 693fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# endif 70e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# else 71e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(0) /* 16 */ 72e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_QMIN 0 73e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 74e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 1) /* 32 */ 75e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 2) /* 48 */ 76e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 3) /* 64 */ 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 4) /* 80 */ 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 5) /* 96 */ 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 6) /* 112 */ 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 7) /* 128 */ 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_CMIN (S2B_QMIN + 8) 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#else 833fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans/* 8-byte quantum ***********************/ 84e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TINY 851e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans# if (LG_TINY_MIN == 2) 863fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_4(0) /* 4 */ 873fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans S2B_4(1) /* 8 */ 881e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans# define S2B_QMIN 1 893fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# else 903fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# error "Unsupported LG_TINY_MIN" 913fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans# endif 92e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# else 93e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(0) /* 8 */ 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_QMIN 0 95e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 1) /* 16 */ 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 2) /* 24 */ 98e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 3) /* 32 */ 99e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 4) /* 40 */ 100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 5) /* 48 */ 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 6) /* 56 */ 102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 7) /* 64 */ 103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 8) /* 72 */ 104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 9) /* 80 */ 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 10) /* 88 */ 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 11) /* 96 */ 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 12) /* 104 */ 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 13) /* 112 */ 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 14) /* 120 */ 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 15) /* 128 */ 111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_CMIN (S2B_QMIN + 16) 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/****************************************/ 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 0) /* 192 */ 115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 1) /* 256 */ 116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 2) /* 320 */ 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 3) /* 384 */ 118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 4) /* 448 */ 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 5) /* 512 */ 120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_SMIN (S2B_CMIN + 6) 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 0) /* 768 */ 122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 1) /* 1024 */ 123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 2) /* 1280 */ 124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 3) /* 1536 */ 125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 4) /* 1792 */ 126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 5) /* 2048 */ 127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 6) /* 2304 */ 128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 7) /* 2560 */ 129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 8) /* 2816 */ 130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 9) /* 3072 */ 131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 10) /* 3328 */ 132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 11) /* 3584 */ 133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 12) /* 3840 */ 134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (STATIC_PAGE_SHIFT == 13) 135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 13) /* 4096 */ 136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 14) /* 4352 */ 137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 15) /* 4608 */ 138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 16) /* 4864 */ 139e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 17) /* 5120 */ 140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 18) /* 5376 */ 141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 19) /* 5632 */ 142e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 20) /* 5888 */ 143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 21) /* 6144 */ 144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 22) /* 6400 */ 145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 23) /* 6656 */ 146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 24) /* 6912 */ 147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 25) /* 7168 */ 148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 26) /* 7424 */ 149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 27) /* 7680 */ 150e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 28) /* 7936 */ 151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 152e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_1 154e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_2 155e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_4 156e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_8 157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_16 158e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_32 159e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_64 160e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_128 161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_256 162e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_QMIN 163e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_CMIN 164e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_SMIN 165e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 166e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 167e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Function prototypes for non-inline static functions. */ 168e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 169e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, 170e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool large, bool zero); 171e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t *arena_chunk_alloc(arena_t *arena); 172e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); 173e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, 174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool zero); 175e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_purge(arena_t *arena); 176e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty); 177e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, 178e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize); 179e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, 180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); 181e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); 182e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); 183e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic size_t arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size); 184e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *arena_malloc_large(arena_t *arena, size_t size, bool zero); 185e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_is_large(const void *ptr); 186e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 187e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, arena_bin_t *bin); 188e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, 189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr, size_t size, size_t oldsize); 190e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, 191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr, size_t size, size_t oldsize); 192e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_ralloc_large(void *ptr, size_t size, size_t oldsize); 193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 194e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic size_t pow2_ceil(size_t x); 195e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 196e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool small_size2bin_init(void); 197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 198e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void small_size2bin_validate(void); 199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 200e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool small_size2bin_init_hard(void); 201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 204e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 205e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm = (uintptr_t)a; 208e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t b_mapelm = (uintptr_t)b; 209e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(a != NULL); 211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(b != NULL); 212e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 213e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 215e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 216f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 217f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evansrb_gen(static JEMALLOC_ATTR(unused), arena_run_tree_, arena_run_tree_t, 218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t, link, arena_run_comp) 219e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 220e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 221e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans int ret; 224e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t a_size = a->bits & ~PAGE_MASK; 225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t b_size = b->bits & ~PAGE_MASK; 226e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_size > b_size) - (a_size < b_size); 228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == 0) { 229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm, b_mapelm; 230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((a->bits & CHUNK_MAP_KEY) != CHUNK_MAP_KEY) 232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = (uintptr_t)a; 233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Treat keys as though they are lower than anything 236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * else. 237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = 0; 239e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 240e476f8a161d445211fd6e54fe370275196e66bcbJason Evans b_mapelm = (uintptr_t)b; 241e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 248f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 249f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evansrb_gen(static JEMALLOC_ATTR(unused), arena_avail_tree_, arena_avail_tree_t, 250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t, link, arena_avail_comp) 251e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 252e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void * 253e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin) 254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 255e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 256e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 257e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 2581e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(run->nfree > 0); 259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2601e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree--; 2611e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans ret = run->avail; 2621e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans if (ret != NULL) { 2631e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->avail = *(void **)ret; 2641e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Double free can cause assertion failure.*/ 2651e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(ret != NULL); 2661e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Write-after free can cause assertion failure. */ 2671e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert((uintptr_t)ret >= (uintptr_t)run + 2681e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans (uintptr_t)bin->reg0_offset); 2691e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert((uintptr_t)ret < (uintptr_t)run->next); 2701e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(((uintptr_t)ret - ((uintptr_t)run + 2711e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans (uintptr_t)bin->reg0_offset)) % (uintptr_t)bin->reg_size == 2721e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans 0); 273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2751e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans ret = run->next; 2761e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->next = (void *)((uintptr_t)ret + (uintptr_t)bin->reg_size); 2771e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(ret != NULL); 2781e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans return (ret); 2796109fe07a14b7a619365977d9523db9f8b333792Jason Evans} 2806109fe07a14b7a619365977d9523db9f8b333792Jason Evans 2816109fe07a14b7a619365977d9523db9f8b333792Jason Evansstatic inline void 2821e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evansarena_run_reg_dalloc(arena_run_t *run, void *ptr) 2836109fe07a14b7a619365977d9523db9f8b333792Jason Evans{ 2846109fe07a14b7a619365977d9523db9f8b333792Jason Evans 2851e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(run->nfree < run->bin->nregs); 2861e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Freeing an interior pointer can cause assertion failure. */ 2871e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 2881e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans (uintptr_t)run->bin->reg0_offset)) % (uintptr_t)run->bin->reg_size 2891e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans == 0); 290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2911e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans *(void **)ptr = run->avail; 2921e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->avail = ptr; 2931e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree++; 294e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 295e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 296e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 297e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, 298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool zero) 299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i; 302e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans old_ndirty = chunk->ndirty; 305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) 306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> PAGE_SHIFT); 307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans total_pages = (chunk->map[run_ind].bits & ~PAGE_MASK) >> 308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT; 309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans need_pages = (size >> PAGE_SHIFT); 310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages > 0); 311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages <= total_pages); 312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans rem_pages = total_pages - need_pages; 313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_remove(&arena->runs_avail, &chunk->map[run_ind]); 315e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive += need_pages; 316e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 317e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Keep track of trailing unused pages for later use. */ 318e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (rem_pages > 0) { 319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+need_pages].bits = (rem_pages << 320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT) | (chunk->map[run_ind+need_pages].bits & 321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+total_pages-1].bits = (rem_pages << 323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT) | (chunk->map[run_ind+total_pages-1].bits & 324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_insert(&arena->runs_avail, 326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_ind+need_pages]); 327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 328e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 329e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < need_pages; i++) { 330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Zero if necessary. */ 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero) { 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((chunk->map[run_ind + i].bits & CHUNK_MAP_ZEROED) 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans == 0) { 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)chunk + ((run_ind 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans + i) << PAGE_SHIFT)), 0, PAGE_SIZE); 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* CHUNK_MAP_ZEROED is cleared below. */ 337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Update dirty page accounting. */ 341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) { 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty--; 343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty--; 344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* CHUNK_MAP_DIRTY is cleared below. */ 345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize the chunk map. */ 348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (large) { 349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind + i].bits = CHUNK_MAP_LARGE 350e476f8a161d445211fd6e54fe370275196e66bcbJason Evans | CHUNK_MAP_ALLOCATED; 351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind + i].bits = (i << CHUNK_MAP_PG_SHIFT) 353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans | CHUNK_MAP_ALLOCATED; 354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 355e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 356e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (large) { 358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Set the run size only in the first element for large runs. 360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * This is primarily a debugging aid, since the lack of size 361e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * info for trailing pages only matters if the application 362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * tries to operate on an interior pointer. 363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits |= size; 365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 368e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t * 369e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_alloc(arena_t *arena) 370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare != NULL) { 375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena->spare; 376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 37841631d00618d7262125e501c91d31b4d70e605faJason Evans bool zero; 37941631d00618d7262125e501c91d31b4d70e605faJason Evans size_t zeroed; 38041631d00618d7262125e501c91d31b4d70e605faJason Evans 38141631d00618d7262125e501c91d31b4d70e605faJason Evans zero = false; 38241631d00618d7262125e501c91d31b4d70e605faJason Evans chunk = (arena_chunk_t *)chunk_alloc(chunksize, &zero); 383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk == NULL) 384e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.mapped += chunksize; 387e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->arena = arena; 3902caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_elm_new(chunk, link_dirty); 391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = false; 392e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 393e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 394e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Claim that no pages are in use, since the header is merely 395e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * overhead. 396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty = 0; 398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Initialize the map to contain one maximal free untouched run. 40141631d00618d7262125e501c91d31b4d70e605faJason Evans * Mark the pages as zeroed iff chunk_alloc() returned a zeroed 40241631d00618d7262125e501c91d31b4d70e605faJason Evans * chunk. 403e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 40441631d00618d7262125e501c91d31b4d70e605faJason Evans zeroed = zero ? CHUNK_MAP_ZEROED : 0; 405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < arena_chunk_header_npages; i++) 406e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[i].bits = 0; 40741631d00618d7262125e501c91d31b4d70e605faJason Evans chunk->map[i].bits = arena_maxclass | zeroed; 40841631d00618d7262125e501c91d31b4d70e605faJason Evans for (i++; i < chunk_npages-1; i++) 40941631d00618d7262125e501c91d31b4d70e605faJason Evans chunk->map[i].bits = zeroed; 41041631d00618d7262125e501c91d31b4d70e605faJason Evans chunk->map[chunk_npages-1].bits = arena_maxclass | zeroed; 411e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 412e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 413e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert the run into the runs_avail tree. */ 414e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_insert(&arena->runs_avail, 415e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[arena_chunk_header_npages]); 416e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 417e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (chunk); 418e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 419e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 420e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 421e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) 422e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 423e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 424e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare != NULL) { 425e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare->dirtied) { 4262caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_remove(&chunk->arena->chunks_dirty, arena->spare, 4272caa4715ed4f787f263239ff97dd824636289286Jason Evans link_dirty); 428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty -= arena->spare->ndirty; 429e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 430e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk_dealloc((void *)arena->spare, chunksize); 431e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 432e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.mapped -= chunksize; 433e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 434e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 435e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 436e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 4372caa4715ed4f787f263239ff97dd824636289286Jason Evans * Remove run from runs_avail, regardless of whether this chunk will be 4382caa4715ed4f787f263239ff97dd824636289286Jason Evans * cached, so that the arena does not use it. 439e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 440e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_remove(&arena->runs_avail, 441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[arena_chunk_header_npages]); 442e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 443e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = chunk; 444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 445e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 446e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 447e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) 448e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 451e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm, key; 452e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 453e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 454e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 455e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 456e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Search the arena's chunks for the lowest best fit. */ 457e476f8a161d445211fd6e54fe370275196e66bcbJason Evans key.bits = size | CHUNK_MAP_KEY; 458e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key); 459e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 460e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)mapelm - (uintptr_t)run_chunk->map) 462e476f8a161d445211fd6e54fe370275196e66bcbJason Evans / sizeof(arena_chunk_map_t); 463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind 465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans << PAGE_SHIFT)); 466e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, run, size, large, zero); 467e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 470e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 471e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * No usable runs. Create a new chunk from which to allocate the run. 472e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena_chunk_alloc(arena); 474e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk == NULL) 475e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 476e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)chunk + (arena_chunk_header_npages << 477e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)); 478e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Update page map. */ 479e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, run, size, large, zero); 480e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 481e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 482e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 483e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 484e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_purge(arena_t *arena) 485e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 486e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 4872caa4715ed4f787f263239ff97dd824636289286Jason Evans size_t i, j, npages; 488e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 489e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty = 0; 490e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4912caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_foreach(chunk, &arena->chunks_dirty, link_dirty) { 4922caa4715ed4f787f263239ff97dd824636289286Jason Evans assert(chunk->dirtied); 4932caa4715ed4f787f263239ff97dd824636289286Jason Evans ndirty += chunk->ndirty; 4942caa4715ed4f787f263239ff97dd824636289286Jason Evans } 495e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ndirty == arena->ndirty); 496e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 4972caa4715ed4f787f263239ff97dd824636289286Jason Evans assert(arena->ndirty > chunk_npages); 498e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((arena->nactive >> opt_lg_dirty_mult) < arena->ndirty); 499e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 500e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 501e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.npurge++; 502e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 503e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 504e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 5052caa4715ed4f787f263239ff97dd824636289286Jason Evans * Only allow one thread at a time to purge dirty pages. madvise(2) 5062caa4715ed4f787f263239ff97dd824636289286Jason Evans * causes the kernel to modify virtual memory data structures that are 5072caa4715ed4f787f263239ff97dd824636289286Jason Evans * typically protected by a lock, and purging isn't important enough to 5082caa4715ed4f787f263239ff97dd824636289286Jason Evans * suffer lock contention in the kernel. The result of failing to 5092caa4715ed4f787f263239ff97dd824636289286Jason Evans * acquire purge_lock here is that this arena will operate with ndirty 5102caa4715ed4f787f263239ff97dd824636289286Jason Evans * above the threshold until some dirty pages are re-used, or the 5112caa4715ed4f787f263239ff97dd824636289286Jason Evans * creation of more dirty pages causes this function to be called 5122caa4715ed4f787f263239ff97dd824636289286Jason Evans * again. 513e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 5142caa4715ed4f787f263239ff97dd824636289286Jason Evans if (malloc_mutex_trylock(&purge_lock)) 5152caa4715ed4f787f263239ff97dd824636289286Jason Evans return; 516e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 5172caa4715ed4f787f263239ff97dd824636289286Jason Evans /* 5182caa4715ed4f787f263239ff97dd824636289286Jason Evans * Iterate through chunks until enough dirty memory has been 5192caa4715ed4f787f263239ff97dd824636289286Jason Evans * purged (all dirty pages in one chunk, or enough pages to drop to 5202caa4715ed4f787f263239ff97dd824636289286Jason Evans * threshold, whichever is greater). Terminate as soon as possible in 5212caa4715ed4f787f263239ff97dd824636289286Jason Evans * order to minimize the number of system calls, even if a chunk has 5222caa4715ed4f787f263239ff97dd824636289286Jason Evans * only been partially purged. 5232caa4715ed4f787f263239ff97dd824636289286Jason Evans */ 5242caa4715ed4f787f263239ff97dd824636289286Jason Evans for (i = 0; (arena->nactive >> opt_lg_dirty_mult) < arena->ndirty; 5252caa4715ed4f787f263239ff97dd824636289286Jason Evans i++) { 5262caa4715ed4f787f263239ff97dd824636289286Jason Evans chunk = ql_first(&arena->chunks_dirty); 527e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(chunk != NULL); 528e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 5292caa4715ed4f787f263239ff97dd824636289286Jason Evans /* Purge pages from high to low within each chunk. */ 5302caa4715ed4f787f263239ff97dd824636289286Jason Evans for (j = chunk_npages - 1; chunk->ndirty > 0; j--) { 5312caa4715ed4f787f263239ff97dd824636289286Jason Evans assert(j >= arena_chunk_header_npages); 5322caa4715ed4f787f263239ff97dd824636289286Jason Evans if (chunk->map[j].bits & CHUNK_MAP_DIRTY) { 5332caa4715ed4f787f263239ff97dd824636289286Jason Evans chunk->map[j].bits ^= CHUNK_MAP_DIRTY; 534e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Find adjacent dirty run(s). */ 5352caa4715ed4f787f263239ff97dd824636289286Jason Evans for (npages = 1; j > arena_chunk_header_npages 5362caa4715ed4f787f263239ff97dd824636289286Jason Evans && (chunk->map[j - 1].bits & 537e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_DIRTY); npages++) { 5382caa4715ed4f787f263239ff97dd824636289286Jason Evans j--; 5392caa4715ed4f787f263239ff97dd824636289286Jason Evans chunk->map[j].bits ^= CHUNK_MAP_DIRTY; 540e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 541e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty -= npages; 542e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty -= npages; 543e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 5442caa4715ed4f787f263239ff97dd824636289286Jason Evans madvise((void *)((uintptr_t)chunk + (j << 545e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)), (npages << PAGE_SHIFT), 546e476f8a161d445211fd6e54fe370275196e66bcbJason Evans MADV_DONTNEED); 547e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 548e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmadvise++; 549e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.purged += npages; 550e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 5512caa4715ed4f787f263239ff97dd824636289286Jason Evans if ((arena->nactive >> opt_lg_dirty_mult) >= 5522caa4715ed4f787f263239ff97dd824636289286Jason Evans arena->ndirty && i > 0) 553e476f8a161d445211fd6e54fe370275196e66bcbJason Evans break; 554e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 555e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 556e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 557e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->ndirty == 0) { 5582caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_remove(&arena->chunks_dirty, chunk, link_dirty); 559e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = false; 560e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 561e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 5622caa4715ed4f787f263239ff97dd824636289286Jason Evans malloc_mutex_unlock(&purge_lock); 563e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 564e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 565e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 566e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) 567e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 568e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 569e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size, run_ind, run_pages; 570e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 571e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 572e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) 573e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> PAGE_SHIFT); 574e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run_ind >= arena_chunk_header_npages); 575e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run_ind < chunk_npages); 576e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((chunk->map[run_ind].bits & CHUNK_MAP_LARGE) != 0) 577e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = chunk->map[run_ind].bits & ~PAGE_MASK; 578e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 579e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = run->bin->run_size; 580e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_pages = (size >> PAGE_SHIFT); 581e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive -= run_pages; 582e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 583e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Mark pages as unallocated in the chunk map. */ 584e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (dirty) { 585e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 586e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 587e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < run_pages; i++) { 588e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 589e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * When (dirty == true), *all* pages within the run 590e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * need to have their dirty bits set, because only 591e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * small runs can create a mixture of clean/dirty 592e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * pages, but such runs are passed to this function 593e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * with (dirty == false). 594e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 595e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) 596e476f8a161d445211fd6e54fe370275196e66bcbJason Evans == 0); 597e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty++; 598e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty++; 599e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind + i].bits = CHUNK_MAP_DIRTY; 600e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 601e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 602e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 603e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 604e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < run_pages; i++) { 605e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind + i].bits &= ~(CHUNK_MAP_LARGE | 606e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED); 607e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 608e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 609e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits & 610e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 611e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+run_pages-1].bits = size | 612e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (chunk->map[run_ind+run_pages-1].bits & CHUNK_MAP_FLAGS_MASK); 613e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 614e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce forward. */ 615e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run_ind + run_pages < chunk_npages && 616e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (chunk->map[run_ind+run_pages].bits & CHUNK_MAP_ALLOCATED) == 0) { 617e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t nrun_size = chunk->map[run_ind+run_pages].bits & 618e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ~PAGE_MASK; 619e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 620e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 621e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove successor from runs_avail; the coalesced run is 622e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 623e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 624e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_remove(&arena->runs_avail, 625e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_ind+run_pages]); 626e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 627e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += nrun_size; 628e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_pages = size >> PAGE_SHIFT; 629e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 630e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[run_ind+run_pages-1].bits & ~PAGE_MASK) 631e476f8a161d445211fd6e54fe370275196e66bcbJason Evans == nrun_size); 632e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits & 633e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 634e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+run_pages-1].bits = size | 635e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (chunk->map[run_ind+run_pages-1].bits & 636e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 637e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 638e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 639e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce backward. */ 640e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run_ind > arena_chunk_header_npages && (chunk->map[run_ind-1].bits & 641e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED) == 0) { 642e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t prun_size = chunk->map[run_ind-1].bits & ~PAGE_MASK; 643e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 644e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind -= prun_size >> PAGE_SHIFT; 645e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 646e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 647e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove predecessor from runs_avail; the coalesced run is 648e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 649e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 650e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_remove(&arena->runs_avail, 651e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_ind]); 652e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 653e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += prun_size; 654e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_pages = size >> PAGE_SHIFT; 655e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 656e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[run_ind].bits & ~PAGE_MASK) == prun_size); 657e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits & 658e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 659e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+run_pages-1].bits = size | 660e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (chunk->map[run_ind+run_pages-1].bits & 661e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 662e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 663e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 664e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert into runs_avail, now that coalescing is complete. */ 665e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_insert(&arena->runs_avail, &chunk->map[run_ind]); 666e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 6674fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 6684fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * Deallocate chunk if it is now completely unused. The bit 6694fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * manipulation checks whether the first run is unallocated and extends 6704fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * to the end of the chunk. 6714fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 672e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((chunk->map[arena_chunk_header_npages].bits & (~PAGE_MASK | 673e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED)) == arena_maxclass) 674e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_dealloc(arena, chunk); 675e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 6764fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 6774fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * It is okay to do dirty page processing even if the chunk was 6784fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * deallocated above, since in that case it is the spare. Waiting 6794fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * until after possible chunk deallocation to do dirty processing 6804fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * allows for an old spare to be fully deallocated, thus decreasing the 6814fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * chances of spuriously crossing the dirty page purging threshold. 6824fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 683e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (dirty) { 684e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->dirtied == false) { 6852caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty); 686e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = true; 687e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 688e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 689e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Enforce opt_lg_dirty_mult. */ 6902caa4715ed4f787f263239ff97dd824636289286Jason Evans if (opt_lg_dirty_mult >= 0 && arena->ndirty > chunk_npages && 6912caa4715ed4f787f263239ff97dd824636289286Jason Evans (arena->nactive >> opt_lg_dirty_mult) < arena->ndirty) 692e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_purge(arena); 693e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 694e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 695e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 696e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 697e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 698e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize) 699e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 700e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; 701e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t head_npages = (oldsize - newsize) >> PAGE_SHIFT; 702e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 703e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 704e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 705e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 706e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 707e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * leading run as separately allocated. 708e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 709e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[pageind].bits & CHUNK_MAP_DIRTY) == 0); 710e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind].bits = (oldsize - newsize) | CHUNK_MAP_LARGE | 711e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 712e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[pageind+head_npages].bits & CHUNK_MAP_DIRTY) == 0); 713e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind+head_npages].bits = newsize | CHUNK_MAP_LARGE | 714e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 715e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 716e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, run, false); 717e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 718e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 719e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 720e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 721e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize, bool dirty) 722e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 723e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; 724e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t npages = newsize >> PAGE_SHIFT; 725e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 726e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 727e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 728e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 729e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 730e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * trailing run as separately allocated. 731e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 732e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[pageind].bits & CHUNK_MAP_DIRTY) == 0); 733e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind].bits = newsize | CHUNK_MAP_LARGE | 734e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 735e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[pageind+npages].bits & CHUNK_MAP_DIRTY) == 0); 736e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind+npages].bits = (oldsize - newsize) | CHUNK_MAP_LARGE 737e476f8a161d445211fd6e54fe370275196e66bcbJason Evans | CHUNK_MAP_ALLOCATED; 738e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 739e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 740e476f8a161d445211fd6e54fe370275196e66bcbJason Evans dirty); 741e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 742e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 743e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 744e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 745e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 746e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm; 747e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 748e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 749e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Look for a usable run. */ 750e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapelm = arena_run_tree_first(&bin->runs); 751e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 752e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 753e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 754e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 755e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* run is guaranteed to have available space. */ 756e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_remove(&bin->runs, mapelm); 757e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 758e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 759e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 760e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sizeof(arena_chunk_map_t)); 761e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 762e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((mapelm->bits & CHUNK_MAP_PG_MASK) >> CHUNK_MAP_PG_SHIFT)) 763e476f8a161d445211fd6e54fe370275196e66bcbJason Evans << PAGE_SHIFT)); 764e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 765e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.reruns++; 766e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 767e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 768e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 769e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* No existing runs have any space available. */ 770e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 771e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Allocate a new run. */ 77286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 773e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = arena_run_alloc(arena, bin->run_size, false, false); 77486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 775e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run == NULL) 776e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 777e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 778e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize run internals. */ 779e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->bin = bin; 7801e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->avail = NULL; 7811e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->next = (void *)(((uintptr_t)run) + (uintptr_t)bin->reg0_offset); 782e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->nfree = bin->nregs; 783e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 784e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->magic = ARENA_RUN_MAGIC; 785e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 786e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 787e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 788e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nruns++; 789e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.curruns++; 790e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->stats.curruns > bin->stats.highruns) 791e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.highruns = bin->stats.curruns; 792e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 793e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 794e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 795e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7961e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 797e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void * 798e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 799e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 800e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 801e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = arena_bin_nonfull_run_get(arena, bin); 802e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->runcur == NULL) 803e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 804e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->runcur->magic == ARENA_RUN_MAGIC); 805e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->runcur->nfree > 0); 806e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 8071e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans return (arena_run_reg_alloc(bin->runcur, bin)); 808e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 809e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 81086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#ifdef JEMALLOC_PROF 81186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evansvoid 81286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evansarena_prof_accum(arena_t *arena, uint64_t accumbytes) 81386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans{ 81486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 81586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (prof_interval != 0) { 81686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena->prof_accumbytes += accumbytes; 81786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (arena->prof_accumbytes >= prof_interval) { 81886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans prof_idump(); 81986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena->prof_accumbytes -= prof_interval; 82086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 82186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 82286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans} 82386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#endif 82486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 825e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 826e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 827d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evansarena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind 828d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans# ifdef JEMALLOC_PROF 829d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , uint64_t prof_accumbytes 830d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans# endif 831d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ) 832e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 833e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, nfill; 834e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 835e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 836e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr; 837e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 838e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tbin->ncached == 0); 839e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 840e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 84186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 842d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 84386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 844d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, prof_accumbytes); 84586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 846d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 8473fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans for (i = 0, nfill = (tbin->ncached_max >> 1); i < nfill; i++) { 848e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 8491e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans ptr = arena_run_reg_alloc(run, bin); 850e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 851e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ptr = arena_bin_malloc_hard(arena, bin); 8523fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (ptr == NULL) 853e476f8a161d445211fd6e54fe370275196e66bcbJason Evans break; 8543fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans *(void **)ptr = tbin->avail; 8553fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans tbin->avail = ptr; 856e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 857e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 85886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.allocated += (i - tbin->ncached) * bin->reg_size; 85986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nmalloc += i; 860e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 86186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nfills++; 862e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->tstats.nrequests = 0; 863e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 86486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 865e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->ncached = i; 866e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tbin->ncached > tbin->high_water) 867e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->high_water = tbin->ncached; 868e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 869e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 870e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 871e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 872e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Calculate bin->run_size such that it meets the following constraints: 873e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 874e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) bin->run_size >= min_run_size 875e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) bin->run_size <= arena_maxclass 876e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 877e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) run header size < PAGE_SIZE 878e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 8791e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * bin->nregs and bin->reg0_offset are also calculated here, since these 8801e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * settings are all interdependent. 881e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 882e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic size_t 883e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size) 884e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 885e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t try_run_size, good_run_size; 8866109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint32_t try_nregs, good_nregs; 8876109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint32_t try_hdr_size, good_hdr_size; 8886109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 8896109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint32_t try_cnt0_offset, good_cnt0_offset; 8906109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 8916109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint32_t try_reg0_offset, good_reg0_offset; 892e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 893e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(min_run_size >= PAGE_SIZE); 894e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(min_run_size <= arena_maxclass); 895e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 896e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 897e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Calculate known-valid settings before entering the run_size 898e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * expansion loop, so that the first part of the loop always copies 899e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * valid settings. 900e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 901e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The do..while loop iteratively reduces the number of regions until 902e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * the run header and the regions no longer overlap. A closed formula 903e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * would be quite messy, since there is an interdependency between the 904e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * header's mask length and the number of regions. 905e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 906e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_run_size = min_run_size; 907e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size) 908e476f8a161d445211fd6e54fe370275196e66bcbJason Evans + 1; /* Counter-act try_nregs-- in loop. */ 909e476f8a161d445211fd6e54fe370275196e66bcbJason Evans do { 910e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_nregs--; 9111e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans try_hdr_size = sizeof(arena_run_t); 9126109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 9136109fe07a14b7a619365977d9523db9f8b333792Jason Evans if (opt_prof) { 9146109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Pad to a quantum boundary. */ 9156109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 9166109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_cnt0_offset = try_hdr_size; 9176109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Add space for one (prof_thr_cnt_t *) per region. */ 9186109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_hdr_size += try_nregs * sizeof(prof_thr_cnt_t *); 9196109fe07a14b7a619365977d9523db9f8b333792Jason Evans } else 9206109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_cnt0_offset = 0; 9216109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 922e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_reg0_offset = try_run_size - (try_nregs * bin->reg_size); 9236109fe07a14b7a619365977d9523db9f8b333792Jason Evans } while (try_hdr_size > try_reg0_offset); 924e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 925e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* run_size expansion loop. */ 926e476f8a161d445211fd6e54fe370275196e66bcbJason Evans do { 927e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 928e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Copy valid settings before trying more aggressive settings. 929e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 930e476f8a161d445211fd6e54fe370275196e66bcbJason Evans good_run_size = try_run_size; 931e476f8a161d445211fd6e54fe370275196e66bcbJason Evans good_nregs = try_nregs; 9326109fe07a14b7a619365977d9523db9f8b333792Jason Evans good_hdr_size = try_hdr_size; 9336109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 9346109fe07a14b7a619365977d9523db9f8b333792Jason Evans good_cnt0_offset = try_cnt0_offset; 9356109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 936e476f8a161d445211fd6e54fe370275196e66bcbJason Evans good_reg0_offset = try_reg0_offset; 937e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 938e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try more aggressive settings. */ 939e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_run_size += PAGE_SIZE; 940e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / 941e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */ 942e476f8a161d445211fd6e54fe370275196e66bcbJason Evans do { 943e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_nregs--; 9441e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans try_hdr_size = sizeof(arena_run_t); 9456109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 9466109fe07a14b7a619365977d9523db9f8b333792Jason Evans if (opt_prof) { 9476109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Pad to a quantum boundary. */ 9486109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 9496109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_cnt0_offset = try_hdr_size; 9506109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* 9516109fe07a14b7a619365977d9523db9f8b333792Jason Evans * Add space for one (prof_thr_cnt_t *) per 9526109fe07a14b7a619365977d9523db9f8b333792Jason Evans * region. 9536109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 9546109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_hdr_size += try_nregs * 9556109fe07a14b7a619365977d9523db9f8b333792Jason Evans sizeof(prof_thr_cnt_t *); 9566109fe07a14b7a619365977d9523db9f8b333792Jason Evans } 9576109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 958e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_reg0_offset = try_run_size - (try_nregs * 959e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size); 9606109fe07a14b7a619365977d9523db9f8b333792Jason Evans } while (try_hdr_size > try_reg0_offset); 9611e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans } while (try_run_size <= arena_maxclass 9621e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans && try_run_size <= arena_maxclass 963e476f8a161d445211fd6e54fe370275196e66bcbJason Evans && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX 964e476f8a161d445211fd6e54fe370275196e66bcbJason Evans && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size 9656109fe07a14b7a619365977d9523db9f8b333792Jason Evans && try_hdr_size < PAGE_SIZE); 966e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 9676109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(good_hdr_size <= good_reg0_offset); 968e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 969e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Copy final settings. */ 970e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->run_size = good_run_size; 971e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->nregs = good_nregs; 9726109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 9736109fe07a14b7a619365977d9523db9f8b333792Jason Evans bin->cnt0_offset = good_cnt0_offset; 9746109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 975e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg0_offset = good_reg0_offset; 976e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 977e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (good_run_size); 978e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 979e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 980e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 981e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_small(arena_t *arena, size_t size, bool zero) 982e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 983e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 984e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 985e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 986e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 987e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 988e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = small_size2bin[size]; 989e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(binind < mbin0); 990e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 991e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = bin->reg_size; 992e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 99386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 994e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 9951e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans ret = arena_run_reg_alloc(run, bin); 996e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 997e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_hard(arena, bin); 998e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 999e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 100086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1001e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1002e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1003e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1004e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 100586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.allocated += size; 100686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nmalloc++; 100786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nrequests++; 1008e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 100986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1010d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 101186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (isthreaded == false) { 101286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1013d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, size); 101486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 101586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 1016d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 1017e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1018e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 1019e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1020e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1021e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1022e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1023e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1024e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1025e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 1026e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1027e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1028e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1029e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1030e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1031e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1032e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_medium(arena_t *arena, size_t size, bool zero) 1033e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1034e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1035e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1036e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1037e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 1038e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1039e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = MEDIUM_CEILING(size); 1040e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = mbin0 + ((size - medium_min) >> lg_mspace); 1041e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(binind < nbins); 1042e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 1043e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->reg_size == size); 1044e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 104586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 1046e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 10471e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans ret = arena_run_reg_alloc(run, bin); 1048e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1049e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_hard(arena, bin); 1050e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1051e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 105286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1053e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1054e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1055e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1056e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 105786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.allocated += size; 105886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nmalloc++; 105986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.nrequests++; 1060e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 106186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1062d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 106386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (isthreaded == false) { 106486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1065d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, size); 106686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 106786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 1068d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 1069e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1070e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 1071e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1072e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1073e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1074e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1075e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1076e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1077e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 1078e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1079e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1080e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1081e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1082e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1083e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void * 1084e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_large(arena_t *arena, size_t size, bool zero) 1085e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1086e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1087e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1088e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1089e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = PAGE_CEILING(size); 1090e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1091e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)arena_run_alloc(arena, size, true, zero); 1092e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1093e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1094e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1095e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1096e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1097e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_large++; 1098e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large += size; 1099e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1107d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 1108d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, size); 1109d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 1110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 1113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1124e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1125e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc(size_t size, bool zero) 1126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size != 0); 1129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(QUANTUM_CEILING(size) <= arena_maxclass); 1130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= bin_maxclass) { 1132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 1133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache; 1134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((tcache = tcache_get()) != NULL) 1136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (tcache_alloc(tcache, size, zero)); 1137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= small_maxclass) 1139e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (arena_malloc_small(choose_arena(), size, zero)); 1140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 1141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (arena_malloc_medium(choose_arena(), size, 1142e476f8a161d445211fd6e54fe370275196e66bcbJason Evans zero)); 1143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 1145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (arena_malloc_large(choose_arena(), size, zero)); 1146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Only handles large allocations that require more than page alignment. */ 1149e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1150e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size) 1151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1152e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t offset; 1154e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1155e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1156e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 1157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((alignment & PAGE_MASK) == 0); 1158e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1159e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1160e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)arena_run_alloc(arena, alloc_size, true, false); 1161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1162e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1163e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1164e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1165e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1166e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); 1167e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1168e476f8a161d445211fd6e54fe370275196e66bcbJason Evans offset = (uintptr_t)ret & (alignment - 1); 1169e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((offset & PAGE_MASK) == 0); 1170e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(offset < alloc_size); 1171e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (offset == 0) 1172e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, ret, alloc_size, size, false); 1173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 1174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t leadsize, trailsize; 1175e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans leadsize = alignment - offset; 1177e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (leadsize > 0) { 1178e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_head(arena, chunk, ret, alloc_size, 1179e476f8a161d445211fd6e54fe370275196e66bcbJason Evans alloc_size - leadsize); 1180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)((uintptr_t)ret + leadsize); 1181e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1183e476f8a161d445211fd6e54fe370275196e66bcbJason Evans trailsize = alloc_size - leadsize - size; 1184e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (trailsize != 0) { 1185e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Trim trailing space. */ 1186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(trailsize < alloc_size); 1187e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, ret, size + trailsize, 1188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size, false); 1189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1192e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_large++; 1194e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large += size; 1195e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1198e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1200e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1204e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1208e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1209e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1212e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1213e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1214e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1215e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_is_large(const void *ptr) 1216e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1217e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind, mapbits; 1219e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1220e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1224e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 1225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits = chunk->map[pageind].bits; 1226e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 1227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((mapbits & CHUNK_MAP_LARGE) != 0); 1228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Return the size of the allocation pointed to by ptr. */ 1231e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t 1232e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_salloc(const void *ptr) 1233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ret; 1235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind, mapbits; 1237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1239e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1240e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1241e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 1243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits = chunk->map[pageind].bits; 1244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 1245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 1246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 1247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)((pageind - ((mapbits & CHUNK_MAP_PG_MASK) >> 1248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT)); 1249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 1250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = run->bin->reg_size; 1251e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1252e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = mapbits & ~PAGE_MASK; 1253e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ret != 0); 1254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1255e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1256e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1257e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1258e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 12596109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 12601e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evansstatic inline unsigned 12611e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evansarena_run_regind(arena_run_t *run, arena_bin_t *bin, const void *ptr, 12621e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans size_t size) 12631e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans{ 12641e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans unsigned shift, diff, regind; 12651e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans 12661e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(run->magic == ARENA_RUN_MAGIC); 12671e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans 12681e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* 12691e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * Avoid doing division with a variable divisor if possible. Using 12701e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * actual division here can reduce allocator throughput by over 20%! 12711e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans */ 12721e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset); 12731e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans 12741e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Rescale (factor powers of 2 out of the numerator and denominator). */ 12751e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans shift = ffs(size) - 1; 12761e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans diff >>= shift; 12771e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans size >>= shift; 12781e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans 12791e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans if (size == 1) { 12801e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* The divisor was a power of 2. */ 12811e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans regind = diff; 12821e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans } else { 12831e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* 12841e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * To divide by a number D that is not a power of two we 12851e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * multiply by (2^21 / D) and then right shift by 21 positions. 12861e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * 12871e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * X / D 12881e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * 12891e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * becomes 12901e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * 12911e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * (X * size_invs[D - 3]) >> SIZE_INV_SHIFT 12921e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * 12931e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * We can omit the first three elements, because we never 12941e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * divide by 0, and 1 and 2 are both powers of two, which are 12951e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * handled above. 12961e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans */ 12971e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans#define SIZE_INV_SHIFT 21 12981e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) 12991e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans static const unsigned size_invs[] = { 13001e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans SIZE_INV(3), 13011e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 13021e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 13031e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 13041e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 13051e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 13061e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 13071e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 13081e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans }; 13091e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans 13101e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans if (size <= ((sizeof(size_invs) / sizeof(unsigned)) + 2)) 13111e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans regind = (diff * size_invs[size - 3]) >> SIZE_INV_SHIFT; 13121e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans else 13131e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans regind = diff / size; 13141e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans#undef SIZE_INV 13151e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans#undef SIZE_INV_SHIFT 13161e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans } 13171e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(diff == regind * size); 13181e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(regind < bin->nregs); 13191e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans 13201e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans return (regind); 13211e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans} 13221e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans 13236109fe07a14b7a619365977d9523db9f8b333792Jason Evansprof_thr_cnt_t * 13246109fe07a14b7a619365977d9523db9f8b333792Jason Evansarena_prof_cnt_get(const void *ptr) 13256109fe07a14b7a619365977d9523db9f8b333792Jason Evans{ 13266109fe07a14b7a619365977d9523db9f8b333792Jason Evans prof_thr_cnt_t *ret; 13276109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena_chunk_t *chunk; 13286109fe07a14b7a619365977d9523db9f8b333792Jason Evans size_t pageind, mapbits; 13296109fe07a14b7a619365977d9523db9f8b333792Jason Evans 13306109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(ptr != NULL); 13316109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 13326109fe07a14b7a619365977d9523db9f8b333792Jason Evans 13336109fe07a14b7a619365977d9523db9f8b333792Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 13346109fe07a14b7a619365977d9523db9f8b333792Jason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 13356109fe07a14b7a619365977d9523db9f8b333792Jason Evans mapbits = chunk->map[pageind].bits; 13366109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 13376109fe07a14b7a619365977d9523db9f8b333792Jason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 13386109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 13396109fe07a14b7a619365977d9523db9f8b333792Jason Evans (uintptr_t)((pageind - ((mapbits & CHUNK_MAP_PG_MASK) >> 13406109fe07a14b7a619365977d9523db9f8b333792Jason Evans CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT)); 13416109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena_bin_t *bin = run->bin; 13426109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned regind; 13436109fe07a14b7a619365977d9523db9f8b333792Jason Evans 13446109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(run->magic == ARENA_RUN_MAGIC); 13456109fe07a14b7a619365977d9523db9f8b333792Jason Evans regind = arena_run_regind(run, bin, ptr, bin->reg_size); 13466109fe07a14b7a619365977d9523db9f8b333792Jason Evans ret = *(prof_thr_cnt_t **)((uintptr_t)run + bin->cnt0_offset + 13476109fe07a14b7a619365977d9523db9f8b333792Jason Evans (regind * sizeof(prof_thr_cnt_t *))); 13486109fe07a14b7a619365977d9523db9f8b333792Jason Evans } else { 13496109fe07a14b7a619365977d9523db9f8b333792Jason Evans ret = chunk->map[pageind].prof_cnt; 13506109fe07a14b7a619365977d9523db9f8b333792Jason Evans } 13516109fe07a14b7a619365977d9523db9f8b333792Jason Evans 13526109fe07a14b7a619365977d9523db9f8b333792Jason Evans return (ret); 13536109fe07a14b7a619365977d9523db9f8b333792Jason Evans} 13546109fe07a14b7a619365977d9523db9f8b333792Jason Evans 13556109fe07a14b7a619365977d9523db9f8b333792Jason Evansvoid 13566109fe07a14b7a619365977d9523db9f8b333792Jason Evansarena_prof_cnt_set(const void *ptr, prof_thr_cnt_t *cnt) 13576109fe07a14b7a619365977d9523db9f8b333792Jason Evans{ 13586109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena_chunk_t *chunk; 13596109fe07a14b7a619365977d9523db9f8b333792Jason Evans size_t pageind, mapbits; 13606109fe07a14b7a619365977d9523db9f8b333792Jason Evans 13616109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(ptr != NULL); 13626109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 13636109fe07a14b7a619365977d9523db9f8b333792Jason Evans 13646109fe07a14b7a619365977d9523db9f8b333792Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 13656109fe07a14b7a619365977d9523db9f8b333792Jason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 13666109fe07a14b7a619365977d9523db9f8b333792Jason Evans mapbits = chunk->map[pageind].bits; 13676109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 13686109fe07a14b7a619365977d9523db9f8b333792Jason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 13696109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 13706109fe07a14b7a619365977d9523db9f8b333792Jason Evans (uintptr_t)((pageind - ((mapbits & CHUNK_MAP_PG_MASK) >> 13716109fe07a14b7a619365977d9523db9f8b333792Jason Evans CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT)); 13726109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena_bin_t *bin = run->bin; 13736109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned regind; 13746109fe07a14b7a619365977d9523db9f8b333792Jason Evans 13756109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(run->magic == ARENA_RUN_MAGIC); 13766109fe07a14b7a619365977d9523db9f8b333792Jason Evans regind = arena_run_regind(run, bin, ptr, bin->reg_size); 13776109fe07a14b7a619365977d9523db9f8b333792Jason Evans 13786109fe07a14b7a619365977d9523db9f8b333792Jason Evans *((prof_thr_cnt_t **)((uintptr_t)run + bin->cnt0_offset + 13796109fe07a14b7a619365977d9523db9f8b333792Jason Evans (regind * sizeof(prof_thr_cnt_t *)))) = cnt; 13806109fe07a14b7a619365977d9523db9f8b333792Jason Evans } else { 13816109fe07a14b7a619365977d9523db9f8b333792Jason Evans chunk->map[pageind].prof_cnt = cnt; 13826109fe07a14b7a619365977d9523db9f8b333792Jason Evans } 13836109fe07a14b7a619365977d9523db9f8b333792Jason Evans} 13846109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 13856109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1386e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1387e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin) 1389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 13901e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans size_t run_ind, past; 1391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1392e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Deallocate run. */ 1393e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run == bin->runcur) 1394e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1395e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (bin->nregs != 1) { 1396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t run_pageind = (((uintptr_t)run - 1397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)chunk)) >> PAGE_SHIFT; 1398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *run_mapelm = 1399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_pageind]; 1400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * This block's conditional is necessary because if the 1402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * run only contains one region, then it never gets 1403e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted into the non-full runs tree. 1404e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_remove(&bin->runs, run_mapelm); 1406e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 14071e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Mark all pages that were ever used for allocations as dirty. */ 1408e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT); 14091e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans past = (size_t)(((uintptr_t)run->next - (uintptr_t)1U - 14101e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans (uintptr_t)chunk) >> PAGE_SHIFT) + 1; 141186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 141286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 14131e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans chunk->ndirty += past - run_ind; 14141e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans arena->ndirty += past - run_ind; 14151e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans for (; run_ind < past; run_ind++) { 14161e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert((chunk->map[run_ind].bits & CHUNK_MAP_DIRTY) == 0); 14171e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans chunk->map[run_ind].bits |= CHUNK_MAP_DIRTY; 14181e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans } 1419e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1420e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 1421e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->magic = 0; 1422e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1423e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, run, false); 1424e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1425e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->dirtied == false) { 14262caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty); 1427e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = true; 1428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1429e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Enforce opt_lg_dirty_mult. */ 14302caa4715ed4f787f263239ff97dd824636289286Jason Evans if (opt_lg_dirty_mult >= 0 && arena->ndirty > chunk_npages && 14312caa4715ed4f787f263239ff97dd824636289286Jason Evans (arena->nactive >> opt_lg_dirty_mult) < arena->ndirty) 1432e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_purge(arena); 143386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 143486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 143586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#ifdef JEMALLOC_STATS 143686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.curruns--; 143786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#endif 1438e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1439e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1440e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1441e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1442e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm) 1443e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 1445e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 14471e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) 1448e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size; 14491e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans#endif 1450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1451e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 1452e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1453e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((mapelm->bits & CHUNK_MAP_PG_MASK) >> CHUNK_MAP_PG_SHIFT)) << 1454e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)); 1455e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 1456e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = run->bin; 14571e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) 1458e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = bin->reg_size; 14591e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans#endif 1460e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1462e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ptr, 0x5a, size); 1464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 14661e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans arena_run_reg_dalloc(run, ptr); 1467e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run->nfree == bin->nregs) 1469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 1470e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (run->nfree == 1 && run != bin->runcur) { 1471e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1472e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Make sure that bin->runcur always refers to the lowest 1473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * non-full run, if one exists. 1474e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1475e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->runcur == NULL) 1476e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = run; 1477e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if ((uintptr_t)run < (uintptr_t)bin->runcur) { 1478e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Switch runcur. */ 1479e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->runcur->nfree > 0) { 1480e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *runcur_chunk = 1481e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_ADDR2BASE(bin->runcur); 1482e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t runcur_pageind = 1483e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (((uintptr_t)bin->runcur - 1484e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)runcur_chunk)) >> PAGE_SHIFT; 1485e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *runcur_mapelm = 1486e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &runcur_chunk->map[runcur_pageind]; 1487e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1488e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert runcur. */ 1489e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_insert(&bin->runs, 1490e476f8a161d445211fd6e54fe370275196e66bcbJason Evans runcur_mapelm); 1491e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1492e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = run; 1493e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1494e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t run_pageind = (((uintptr_t)run - 1495e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)chunk)) >> PAGE_SHIFT; 1496e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *run_mapelm = 1497e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_pageind]; 1498e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1499e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(arena_run_tree_search(&bin->runs, run_mapelm) == 1500e476f8a161d445211fd6e54fe370275196e66bcbJason Evans NULL); 1501e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_insert(&bin->runs, run_mapelm); 1502e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1503e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1504e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 150686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.allocated -= size; 150786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin->stats.ndalloc++; 1508e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1509e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1510e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1511e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1512e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1513b34e8684ec025aa780997c11f847c19fb269755bJason Evansarena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, 1514b34e8684ec025aa780997c11f847c19fb269755bJason Evans arena_stats_t *astats, malloc_bin_stats_t *bstats, 1515b34e8684ec025aa780997c11f847c19fb269755bJason Evans malloc_large_stats_t *lstats) 1516b34e8684ec025aa780997c11f847c19fb269755bJason Evans{ 15173c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans unsigned i; 1518b34e8684ec025aa780997c11f847c19fb269755bJason Evans 151986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1520b34e8684ec025aa780997c11f847c19fb269755bJason Evans *nactive += arena->nactive; 1521b34e8684ec025aa780997c11f847c19fb269755bJason Evans *ndirty += arena->ndirty; 1522b34e8684ec025aa780997c11f847c19fb269755bJason Evans 15234201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans astats->mapped += arena->stats.mapped; 1524b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->npurge += arena->stats.npurge; 1525b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmadvise += arena->stats.nmadvise; 1526b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->purged += arena->stats.purged; 1527b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->allocated_large += arena->stats.allocated_large; 1528b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmalloc_large += arena->stats.nmalloc_large; 1529b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->ndalloc_large += arena->stats.ndalloc_large; 1530b34e8684ec025aa780997c11f847c19fb269755bJason Evans 15313c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans for (i = 0; i < nlclasses; i++) { 1532b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].nrequests += arena->stats.lstats[i].nrequests; 1533b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].highruns += arena->stats.lstats[i].highruns; 1534b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].curruns += arena->stats.lstats[i].curruns; 1535b34e8684ec025aa780997c11f847c19fb269755bJason Evans } 153686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 153786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 153886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans for (i = 0; i < nbins; i++) { 153986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin = &arena->bins[i]; 154086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 154186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 154286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].allocated += bin->stats.allocated; 154386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nmalloc += bin->stats.nmalloc; 154486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].ndalloc += bin->stats.ndalloc; 154586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nrequests += bin->stats.nrequests; 154686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#ifdef JEMALLOC_TCACHE 154786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nfills += bin->stats.nfills; 154886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nflushes += bin->stats.nflushes; 154986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans#endif 155086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nruns += bin->stats.nruns; 155186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].reruns += bin->stats.reruns; 155286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].highruns += bin->stats.highruns; 155386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].curruns += bin->stats.curruns; 155486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 155586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 1556b34e8684ec025aa780997c11f847c19fb269755bJason Evans} 1557e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1558e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1559e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1560e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1561e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 156213668262d17fb5950e2441bc9d56a15db9c93877Jason Evans 1563e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1564e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1565e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1566e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1567990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans# ifndef JEMALLOC_STATS 1568e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1569990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans# endif 1570e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1571e476f8a161d445211fd6e54fe370275196e66bcbJason Evans { 1572e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) 1573e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 1574e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT; 1575e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size = chunk->map[pageind].bits & ~PAGE_MASK; 1576e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1577e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1578e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1579990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans# ifdef JEMALLOC_STATS 1580e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1581990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans# endif 1582e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ptr, 0x5a, size); 1583e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1584e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1585990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.ndalloc_large++; 1586e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large -= size; 1587e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns--; 1588e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1589e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1590e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1591e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)ptr, true); 1592e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1593e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1594e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1595e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1596e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1597e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size, size_t oldsize) 1598e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1599e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1600e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size < oldsize); 1601e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1602e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1603e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Shrink the run, and make trailing pages available for other 1604e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocations. 1605e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1606e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1607e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 1608e476f8a161d445211fd6e54fe370275196e66bcbJason Evans true); 1609e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1610990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.ndalloc_large++; 1611990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.allocated_large -= oldsize; 1612990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; 1613990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans 1614990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.nmalloc_large++; 1615990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.allocated_large += size; 1616990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1617990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1618990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1619990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1620990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1621990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1622e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1623e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1624e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1625e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1626e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1627e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1628e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1629e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size, size_t oldsize) 1630e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1631e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 1632e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t npages = oldsize >> PAGE_SHIFT; 1633e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1634e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize == (chunk->map[pageind].bits & ~PAGE_MASK)); 1635e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1636e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to extend the run. */ 1637e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size > oldsize); 1638e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1639e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (pageind + npages < chunk_npages && (chunk->map[pageind+npages].bits 1640e476f8a161d445211fd6e54fe370275196e66bcbJason Evans & CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[pageind+npages].bits & 1641e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ~PAGE_MASK) >= size - oldsize) { 1642e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1643e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The next run is available and sufficiently large. Split the 1644e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * following run, then merge the first part with the existing 1645e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocation. 1646e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1647e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + 1648e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((pageind+npages) << PAGE_SHIFT)), size - oldsize, true, 1649e476f8a161d445211fd6e54fe370275196e66bcbJason Evans false); 1650e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1651e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind].bits = size | CHUNK_MAP_LARGE | 1652e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 1653e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind+npages].bits = CHUNK_MAP_LARGE | 1654e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 1655e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1656e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1657990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.ndalloc_large++; 1658990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.allocated_large -= oldsize; 1659990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; 1660990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans 1661990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.nmalloc_large++; 1662990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.allocated_large += size; 1663990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1664990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1665990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1666990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1667990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1668990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1669990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans } 1670e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1671e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1672e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1673e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1674e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1675e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1676e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 1677e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1678e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1679e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 1680e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Try to resize a large allocation, in order to avoid copying. This will 1681e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * always fail if growing an object, and the following run is already in use. 1682e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1683e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1684e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large(void *ptr, size_t size, size_t oldsize) 1685e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1686e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t psize; 1687e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1688e476f8a161d445211fd6e54fe370275196e66bcbJason Evans psize = PAGE_CEILING(size); 1689e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize == oldsize) { 1690e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Same size class. */ 1691e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1692e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk && size < oldsize) { 1693e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - 1694e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size); 1695e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1696e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1697e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1698e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1699e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1700e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 1701e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1702e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1703e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 1704e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(arena->magic == ARENA_MAGIC); 1705e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1706e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize < oldsize) { 1707e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1708e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Fill before shrinking in order avoid a race. */ 1709e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) { 1710e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, 1711e476f8a161d445211fd6e54fe370275196e66bcbJason Evans oldsize - size); 1712e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1713e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1714e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_ralloc_large_shrink(arena, chunk, ptr, psize, 1715e476f8a161d445211fd6e54fe370275196e66bcbJason Evans oldsize); 1716e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1717e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1718e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 1719e476f8a161d445211fd6e54fe370275196e66bcbJason Evans psize, oldsize); 1720e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1721e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == false && opt_zero) { 1722e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + oldsize), 0, 1723e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size - oldsize); 1724e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1725e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1726e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1727e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1728e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1729e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1730e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1731e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1732e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc(void *ptr, size_t size, size_t oldsize) 1733e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1734e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1735e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t copysize; 1736e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1737e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1738e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Try to avoid moving the allocation. 1739e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 1740e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * posix_memalign() can cause allocation of "large" objects that are 1741e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * smaller than bin_maxclass (in order to meet alignment requirements). 1742e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Therefore, do not assume that (oldsize <= bin_maxclass) indicates 1743e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * ptr refers to a bin-allocated object. 1744e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1745e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= arena_maxclass) { 1746e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena_is_large(ptr) == false ) { 1747e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= small_maxclass) { 1748e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= small_maxclass && 1749e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin[size] == 1750e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin[oldsize]) 1751e476f8a161d445211fd6e54fe370275196e66bcbJason Evans goto IN_PLACE; 1752e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else if (size <= bin_maxclass) { 1753e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (small_maxclass < oldsize && oldsize <= 1754e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin_maxclass && MEDIUM_CEILING(size) == 1755e476f8a161d445211fd6e54fe370275196e66bcbJason Evans MEDIUM_CEILING(oldsize)) 1756e476f8a161d445211fd6e54fe370275196e66bcbJason Evans goto IN_PLACE; 1757e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1758e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1759e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 1760e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size > bin_maxclass) { 1761e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena_ralloc_large(ptr, size, oldsize) == 1762e476f8a161d445211fd6e54fe370275196e66bcbJason Evans false) 1763e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 1764e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1765e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1766e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1767e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1768e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to avoid moving the allocation. */ 1769e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= small_maxclass) { 1770e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= small_maxclass && small_size2bin[size] == 1771e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin[oldsize]) 1772e476f8a161d445211fd6e54fe370275196e66bcbJason Evans goto IN_PLACE; 1773e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else if (size <= bin_maxclass) { 1774e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (small_maxclass < oldsize && oldsize <= bin_maxclass && 1775e476f8a161d445211fd6e54fe370275196e66bcbJason Evans MEDIUM_CEILING(size) == MEDIUM_CEILING(oldsize)) 1776e476f8a161d445211fd6e54fe370275196e66bcbJason Evans goto IN_PLACE; 1777e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1778e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin_maxclass < oldsize && oldsize <= arena_maxclass) { 1779e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size > bin_maxclass); 1780e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena_ralloc_large(ptr, size, oldsize) == false) 1781e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 1782e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1783e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1784e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1785e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1786e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * If we get here, then size and oldsize are different enough that we 1787e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * need to move the object. In that case, fall back to allocating new 1788e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * space and copying. 1789e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1790e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_malloc(size, false); 1791e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) 1792e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1793e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1794e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Junk/zero-filling were already done by arena_malloc(). */ 1795e476f8a161d445211fd6e54fe370275196e66bcbJason Evans copysize = (size < oldsize) ? size : oldsize; 1796e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(ret, ptr, copysize); 1797e476f8a161d445211fd6e54fe370275196e66bcbJason Evans idalloc(ptr); 1798e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1799e476f8a161d445211fd6e54fe370275196e66bcbJason EvansIN_PLACE: 1800e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1801e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk && size < oldsize) 1802e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size); 1803e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero && size > oldsize) 1804e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize); 1805e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1806e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 1807e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1808e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1809e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 1810e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_new(arena_t *arena, unsigned ind) 1811e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1812e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 1813e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1814e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t prev_run_size; 1815e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18166109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena->ind = ind; 18176109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1818e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (malloc_mutex_init(&arena->lock)) 1819e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 1820e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1821e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1822e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&arena->stats, 0, sizeof(arena_stats_t)); 1823e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats = (malloc_large_stats_t *)base_alloc( 1824e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sizeof(malloc_large_stats_t) * ((chunksize - PAGE_SIZE) >> 1825e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)); 1826e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats == NULL) 1827e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 1828e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(arena->stats.lstats, 0, sizeof(malloc_large_stats_t) * 1829e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((chunksize - PAGE_SIZE) >> PAGE_SHIFT)); 1830e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 1831e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_new(&arena->tcache_ql); 1832e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 1833e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1834e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1835d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 1836d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena->prof_accumbytes = 0; 1837d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 1838d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 1839e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize chunks. */ 18402caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_new(&arena->chunks_dirty); 1841e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 1842e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1843e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive = 0; 1844e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty = 0; 1845e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1846e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_new(&arena->runs_avail); 1847e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1848e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize bins. */ 1849e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = PAGE_SIZE; 1850e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1851e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 0; 1852e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 1853e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* (2^n)-spaced tiny bins. */ 1854e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins; i++) { 1855e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 185686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 185786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 1858e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1859e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 1860e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1861e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = (1U << (LG_TINY_MIN + i)); 1862e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1863e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 1864e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1865e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1866e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 1867e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1868e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1869e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1870e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1871e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Quantum-spaced bins. */ 1872e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins + nqbins; i++) { 1873e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 187486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 187586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 1876e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1877e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 1878e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1879e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = (i - ntbins + 1) << LG_QUANTUM; 1880e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1881e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 1882e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1883e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1884e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 1885e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1886e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1887e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1888e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Cacheline-spaced bins. */ 1889e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins + nqbins + ncbins; i++) { 1890e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 189186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 189286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 1893e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1894e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 1895e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1896e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = cspace_min + ((i - (ntbins + nqbins)) << 1897e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_CACHELINE); 1898e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1899e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 1900e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1901e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1902e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 1903e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1904e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1905e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1906e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Subpage-spaced bins. */ 1907e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins + nqbins + ncbins + nsbins; i++) { 1908e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 190986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 191086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 1911e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1912e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 1913e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1914e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = sspace_min + ((i - (ntbins + nqbins + ncbins)) 1915e476f8a161d445211fd6e54fe370275196e66bcbJason Evans << LG_SUBPAGE); 1916e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1917e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 1918e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1919e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1920e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 1921e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1922e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1923e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1924e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Medium bins. */ 1925e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < nbins; i++) { 1926e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 192786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 192886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 1929e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1930e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 1931e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1932e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = medium_min + ((i - (ntbins + nqbins + ncbins + 1933e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nsbins)) << lg_mspace); 1934e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1935e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 1936e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1937e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1938e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 1939e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1940e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1941e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1942e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 1943e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->magic = ARENA_MAGIC; 1944e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1945e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1946e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1947e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1948e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1949e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 1950e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Compute the smallest power of 2 that is >= x. */ 1951e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic size_t 1952e476f8a161d445211fd6e54fe370275196e66bcbJason Evanspow2_ceil(size_t x) 1953e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1954e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1955e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x--; 1956e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 1; 1957e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 2; 1958e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 4; 1959e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 8; 1960e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 16; 1961e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (SIZEOF_PTR == 8) 1962e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 32; 1963e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1964e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x++; 1965e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (x); 1966e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1967e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1968e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1969e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 1970e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1971e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssmall_size2bin_validate(void) 1972e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1973e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i, size, binind; 1974e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1975e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[0] == 0xffU); 1976e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 1; 1977e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TINY 1978e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Tiny. */ 1979e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < (1U << LG_TINY_MIN); i++) { 1980e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(1U << LG_TINY_MIN); 1981e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 1982e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 1983e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1984e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < qspace_min; i++) { 1985e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(i); 1986e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 1987e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 1988e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1989e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 1990e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Quantum-spaced. */ 1991e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= qspace_max; i++) { 1992e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = QUANTUM_CEILING(i); 1993e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + (size >> LG_QUANTUM) - 1; 1994e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 1995e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1996e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Cacheline-spaced. */ 1997e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= cspace_max; i++) { 1998e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = CACHELINE_CEILING(i); 1999e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ((size - cspace_min) >> 2000e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_CACHELINE); 2001e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 2002e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2003e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Sub-page. */ 2004e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= sspace_max; i++) { 2005e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = SUBPAGE_CEILING(i); 2006e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ncbins + ((size - sspace_min) 2007e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> LG_SUBPAGE); 2008e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 2009e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2010e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2011e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2012e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2013e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 2014e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssmall_size2bin_init(void) 2015e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2016e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2017e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT 2018e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT 2019e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || sizeof(const_small_size2bin) != small_maxclass + 1) 2020e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (small_size2bin_init_hard()); 2021e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2022e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin = const_small_size2bin; 2023e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2024e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(sizeof(const_small_size2bin) == small_maxclass + 1); 2025e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin_validate(); 2026e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2027e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2028e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2029e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2030e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 2031e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssmall_size2bin_init_hard(void) 2032e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2033e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i, size, binind; 2034e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uint8_t *custom_small_size2bin; 2035e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2036e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT 2037e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT 2038e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || sizeof(const_small_size2bin) != small_maxclass + 1); 2039e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2040e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin = (uint8_t *)base_alloc(small_maxclass + 1); 2041e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (custom_small_size2bin == NULL) 2042e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2043e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2044e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[0] = 0xffU; 2045e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 1; 2046e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 2047e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Tiny. */ 2048e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < (1U << LG_TINY_MIN); i++) { 2049e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(1U << LG_TINY_MIN); 2050e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 2051e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2052e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2053e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < qspace_min; i++) { 2054e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(i); 2055e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 2056e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2057e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2058e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2059e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Quantum-spaced. */ 2060e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= qspace_max; i++) { 2061e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = QUANTUM_CEILING(i); 2062e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + (size >> LG_QUANTUM) - 1; 2063e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2064e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2065e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Cacheline-spaced. */ 2066e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= cspace_max; i++) { 2067e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = CACHELINE_CEILING(i); 2068e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ((size - cspace_min) >> 2069e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_CACHELINE); 2070e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2071e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2072e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Sub-page. */ 2073e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= sspace_max; i++) { 2074e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = SUBPAGE_CEILING(i); 2075e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ncbins + ((size - sspace_min) >> 2076e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_SUBPAGE); 2077e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2078e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2079e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2080e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin = custom_small_size2bin; 2081e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2082e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin_validate(); 2083e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2084e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2085e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2086e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2087e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 2088a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evansarena_boot(void) 2089e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2090a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans size_t header_size; 2091e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2092e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Set variables according to the value of opt_lg_[qc]space_max. */ 2093e476f8a161d445211fd6e54fe370275196e66bcbJason Evans qspace_max = (1U << opt_lg_qspace_max); 2094e476f8a161d445211fd6e54fe370275196e66bcbJason Evans cspace_min = CACHELINE_CEILING(qspace_max); 2095e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (cspace_min == qspace_max) 2096e476f8a161d445211fd6e54fe370275196e66bcbJason Evans cspace_min += CACHELINE; 2097e476f8a161d445211fd6e54fe370275196e66bcbJason Evans cspace_max = (1U << opt_lg_cspace_max); 2098e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sspace_min = SUBPAGE_CEILING(cspace_max); 2099e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (sspace_min == cspace_max) 2100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sspace_min += SUBPAGE; 2101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(sspace_min < PAGE_SIZE); 2102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sspace_max = PAGE_SIZE - SUBPAGE; 2103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans medium_max = (1U << opt_lg_medium_max); 2104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 2106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(LG_QUANTUM >= LG_TINY_MIN); 2107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ntbins <= LG_QUANTUM); 2109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nqbins = qspace_max >> LG_QUANTUM; 2110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ncbins = ((cspace_max - cspace_min) >> LG_CACHELINE) + 1; 2111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nsbins = ((sspace_max - sspace_min) >> LG_SUBPAGE) + 1; 2112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Compute medium size class spacing and the number of medium size 2115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * classes. Limit spacing to no more than pagesize, but if possible 2116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * use the smallest spacing that does not exceed NMBINS_MAX medium size 2117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * classes. 2118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans lg_mspace = LG_SUBPAGE; 2120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nmbins = ((medium_max - medium_min) >> lg_mspace) + 1; 2121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans while (lg_mspace < PAGE_SHIFT && nmbins > NMBINS_MAX) { 2122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans lg_mspace = lg_mspace + 1; 2123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nmbins = ((medium_max - medium_min) >> lg_mspace) + 1; 2124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mspace_mask = (1U << lg_mspace) - 1U; 2126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mbin0 = ntbins + nqbins + ncbins + nsbins; 2128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nbins = mbin0 + nmbins; 2129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The small_size2bin lookup table uses uint8_t to encode each bin 2131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * index, so we cannot support more than 256 small size classes. This 2132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * limit is difficult to exceed (not even possible with 16B quantum and 2133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 4KiB pages), and such configurations are impractical, but 2134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * nonetheless we need to protect against this case in order to avoid 2135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * undefined behavior. 2136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mbin0 > 256) { 2138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans char line_buf[UMAX2S_BUFSIZE]; 2139698805c5252465ddbdcf7048e5ccf16d02533877Jason Evans malloc_write("<jemalloc>: Too many small size classes ("); 2140698805c5252465ddbdcf7048e5ccf16d02533877Jason Evans malloc_write(umax2s(mbin0, 10, line_buf)); 2141698805c5252465ddbdcf7048e5ccf16d02533877Jason Evans malloc_write(" > max 256)\n"); 2142e476f8a161d445211fd6e54fe370275196e66bcbJason Evans abort(); 2143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (small_size2bin_init()) 2146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Compute the header size such that it is large enough to contain the 2150e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * page map. 2151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2152e476f8a161d445211fd6e54fe370275196e66bcbJason Evans header_size = sizeof(arena_chunk_t) + 2153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (sizeof(arena_chunk_map_t) * (chunk_npages - 1)); 2154e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_header_npages = (header_size >> PAGE_SHIFT) + 2155e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((header_size & PAGE_MASK) != 0); 2156e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_maxclass = chunksize - (arena_chunk_header_npages << PAGE_SHIFT); 2157a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans 21582caa4715ed4f787f263239ff97dd824636289286Jason Evans if (malloc_mutex_init(&purge_lock)) 21592caa4715ed4f787f263239ff97dd824636289286Jason Evans return (true); 21602caa4715ed4f787f263239ff97dd824636289286Jason Evans 2161a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans return (false); 2162e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2163