arena.c revision 3c2343518c2b1fbbd66065c75a3c19f908de1d78
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_ARENA_C_ 2b0fd5016db74678476868abb380170b319a50480Jason Evans#include "internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; 8e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t opt_lg_medium_max = LG_MEDIUM_MAX_DEFAULT; 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evansssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 11e476f8a161d445211fd6e54fe370275196e66bcbJason Evansuint8_t const *small_size2bin; 12e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Various bin-related settings. */ 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nqbins; 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned ncbins; 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nsbins; 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nmbins; 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nbins; 19e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned mbin0; 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t qspace_max; 21e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t cspace_min; 22e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t cspace_max; 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t sspace_min; 24e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t sspace_max; 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t medium_max; 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t lg_mspace; 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t mspace_mask; 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * const_small_size2bin is a static constant lookup table that in the common 32e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * case can be used as-is for small_size2bin. For dynamically linked programs, 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * this avoids a page of memory overhead per process. 34e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 35e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_1(i) i, 36e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_2(i) S2B_1(i) S2B_1(i) 37e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_4(i) S2B_2(i) S2B_2(i) 38e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_8(i) S2B_4(i) S2B_4(i) 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_16(i) S2B_8(i) S2B_8(i) 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_32(i) S2B_16(i) S2B_16(i) 41e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_64(i) S2B_32(i) S2B_32(i) 42e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_128(i) S2B_64(i) S2B_64(i) 43e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_256(i) S2B_128(i) S2B_128(i) 44e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The number of elements in const_small_size2bin is dependent on page size 46e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * and on the definition for SUBPAGE. If SUBPAGE changes, the '- 255' must also 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * change, along with the addition/removal of static lookup table element 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * definitions. 49e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic const uint8_t const_small_size2bin[STATIC_PAGE_SIZE - 255] = { 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_1(0xffU) /* 0 */ 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (LG_QUANTUM == 4) 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 64-bit system ************************/ 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TINY 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_2(0) /* 2 */ 56e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_2(1) /* 4 */ 57e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_4(2) /* 8 */ 58e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(3) /* 16 */ 59e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_QMIN 3 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# else 61e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(0) /* 16 */ 62e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_QMIN 0 63e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 64e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 1) /* 32 */ 65e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 2) /* 48 */ 66e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 3) /* 64 */ 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 4) /* 80 */ 68e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 5) /* 96 */ 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 6) /* 112 */ 70e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 7) /* 128 */ 71e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_CMIN (S2B_QMIN + 8) 72e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#else 73e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 32-bit system ************************/ 74e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TINY 75e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_2(0) /* 2 */ 76e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_2(1) /* 4 */ 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_4(2) /* 8 */ 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_QMIN 2 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# else 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(0) /* 8 */ 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_QMIN 0 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 83e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 1) /* 16 */ 84e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 2) /* 24 */ 85e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 3) /* 32 */ 86e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 4) /* 40 */ 87e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 5) /* 48 */ 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 6) /* 56 */ 89e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 7) /* 64 */ 90e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 8) /* 72 */ 91e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 9) /* 80 */ 92e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 10) /* 88 */ 93e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 11) /* 96 */ 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 12) /* 104 */ 95e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 13) /* 112 */ 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 14) /* 120 */ 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 15) /* 128 */ 98e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_CMIN (S2B_QMIN + 16) 99e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/****************************************/ 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 0) /* 192 */ 102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 1) /* 256 */ 103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 2) /* 320 */ 104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 3) /* 384 */ 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 4) /* 448 */ 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 5) /* 512 */ 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_SMIN (S2B_CMIN + 6) 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 0) /* 768 */ 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 1) /* 1024 */ 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 2) /* 1280 */ 111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 3) /* 1536 */ 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 4) /* 1792 */ 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 5) /* 2048 */ 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 6) /* 2304 */ 115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 7) /* 2560 */ 116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 8) /* 2816 */ 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 9) /* 3072 */ 118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 10) /* 3328 */ 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 11) /* 3584 */ 120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 12) /* 3840 */ 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (STATIC_PAGE_SHIFT == 13) 122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 13) /* 4096 */ 123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 14) /* 4352 */ 124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 15) /* 4608 */ 125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 16) /* 4864 */ 126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 17) /* 5120 */ 127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 18) /* 5376 */ 128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 19) /* 5632 */ 129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 20) /* 5888 */ 130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 21) /* 6144 */ 131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 22) /* 6400 */ 132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 23) /* 6656 */ 133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 24) /* 6912 */ 134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 25) /* 7168 */ 135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 26) /* 7424 */ 136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 27) /* 7680 */ 137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 28) /* 7936 */ 138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 139e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_1 141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_2 142e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_4 143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_8 144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_16 145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_32 146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_64 147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_128 148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_256 149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_QMIN 150e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_CMIN 151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_SMIN 152e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 154e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Function prototypes for non-inline static functions. */ 155e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 156e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, 157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool large, bool zero); 158e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t *arena_chunk_alloc(arena_t *arena); 159e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); 160e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, 161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool zero); 162e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_purge(arena_t *arena); 163e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty); 164e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, 165e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize); 166e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, 167e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); 168e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); 169e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); 170e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic size_t arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size); 171e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *arena_malloc_large(arena_t *arena, size_t size, bool zero); 172e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_is_large(const void *ptr); 173e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, arena_bin_t *bin); 175e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, 176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr, size_t size, size_t oldsize); 177e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, 178e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr, size_t size, size_t oldsize); 179e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_ralloc_large(void *ptr, size_t size, size_t oldsize); 180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 181e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic size_t pow2_ceil(size_t x); 182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 183e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool small_size2bin_init(void); 184e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 185e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void small_size2bin_validate(void); 186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 187e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool small_size2bin_init_hard(void); 188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 191e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 192e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_comp(arena_chunk_t *a, arena_chunk_t *b) 193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 194e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_chunk = (uintptr_t)a; 195e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t b_chunk = (uintptr_t)b; 196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(a != NULL); 198e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(b != NULL); 199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 200e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((a_chunk > b_chunk) - (a_chunk < b_chunk)); 201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Wrap red-black tree macros in functions. */ 204e476f8a161d445211fd6e54fe370275196e66bcbJason Evansrb_wrap(static JEMALLOC_ATTR(unused), arena_chunk_tree_dirty_, 205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_t, arena_chunk_t, link_dirty, arena_chunk_comp) 206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 207e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 208e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 209e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm = (uintptr_t)a; 211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t b_mapelm = (uintptr_t)b; 212e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 213e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(a != NULL); 214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(b != NULL); 215e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 216e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 217e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 219e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Wrap red-black tree macros in functions. */ 220e476f8a161d445211fd6e54fe370275196e66bcbJason Evansrb_wrap(static JEMALLOC_ATTR(unused), arena_run_tree_, arena_run_tree_t, 221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t, link, arena_run_comp) 222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 223e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 224e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 226e476f8a161d445211fd6e54fe370275196e66bcbJason Evans int ret; 227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t a_size = a->bits & ~PAGE_MASK; 228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t b_size = b->bits & ~PAGE_MASK; 229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_size > b_size) - (a_size < b_size); 231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == 0) { 232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm, b_mapelm; 233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((a->bits & CHUNK_MAP_KEY) != CHUNK_MAP_KEY) 235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = (uintptr_t)a; 236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Treat keys as though they are lower than anything 239e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * else. 240e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 241e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = 0; 242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans b_mapelm = (uintptr_t)b; 244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 251e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Wrap red-black tree macros in functions. */ 252e476f8a161d445211fd6e54fe370275196e66bcbJason Evansrb_wrap(static JEMALLOC_ATTR(unused), arena_avail_tree_, arena_avail_tree_t, 253e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t, link, arena_avail_comp) 254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 255e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void 256e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_rc_incr(arena_run_t *run, arena_bin_t *bin, const void *ptr) 257e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 258e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 260e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pagebeg, pageend, i; 261e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 262e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 263e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 264e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pagebeg = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 265e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageend = ((uintptr_t)ptr + (uintptr_t)(bin->reg_size - 1) - 266e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)chunk) >> PAGE_SHIFT; 267e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = pagebeg; i <= pageend; i++) { 269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t mapbits = chunk->map[i].bits; 270e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapbits & CHUNK_MAP_DIRTY) { 272e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_RC_MASK) == 0); 273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty--; 274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty--; 275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits ^= CHUNK_MAP_DIRTY; 276e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 277e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_RC_MASK) != CHUNK_MAP_RC_MASK); 278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits += CHUNK_MAP_RC_ONE; 279e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[i].bits = mapbits; 280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 283e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void 284e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_rc_decr(arena_run_t *run, arena_bin_t *bin, const void *ptr) 285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 287e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pagebeg, pageend, mapbits, i; 289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool dirtier = false; 290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 293e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pagebeg = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 294e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageend = ((uintptr_t)ptr + (uintptr_t)(bin->reg_size - 1) - 295e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)chunk) >> PAGE_SHIFT; 296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* First page. */ 298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits = chunk->map[pagebeg].bits; 299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits -= CHUNK_MAP_RC_ONE; 300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((mapbits & CHUNK_MAP_RC_MASK) == 0) { 301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans dirtier = true; 302e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_DIRTY) == 0); 303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits |= CHUNK_MAP_DIRTY; 304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty++; 305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty++; 306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pagebeg].bits = mapbits; 308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (pageend - pagebeg >= 1) { 310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Interior pages are completely consumed by the object being 312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * deallocated, which means that the pages can be 313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * unconditionally marked dirty. 314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 315e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = pagebeg + 1; i < pageend; i++) { 316e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits = chunk->map[i].bits; 317e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits -= CHUNK_MAP_RC_ONE; 318e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_RC_MASK) == 0); 319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans dirtier = true; 320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_DIRTY) == 0); 321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits |= CHUNK_MAP_DIRTY; 322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty++; 323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty++; 324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[i].bits = mapbits; 325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Last page. */ 328e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits = chunk->map[pageend].bits; 329e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits -= CHUNK_MAP_RC_ONE; 330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((mapbits & CHUNK_MAP_RC_MASK) == 0) { 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans dirtier = true; 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_DIRTY) == 0); 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits |= CHUNK_MAP_DIRTY; 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty++; 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty++; 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageend].bits = mapbits; 338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (dirtier) { 341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->dirtied == false) { 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_dirty_insert(&arena->chunks_dirty, 343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk); 344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = true; 345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Enforce opt_lg_dirty_mult. */ 348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_dirty_mult >= 0 && (arena->nactive >> 349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans opt_lg_dirty_mult) < arena->ndirty) 350e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_purge(arena); 351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 354e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void * 355e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin) 356e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, mask, bit, regind; 359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 361e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->regs_minelm < bin->regs_mask_nelms); 362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Move the first check outside the loop, so that run->regs_minelm can 365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * be updated unconditionally, without the possibility of updating it 366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * multiple times. 367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 368e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = run->regs_minelm; 369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mask = run->regs_mask[i]; 370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mask != 0) { 371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Usable allocation found. */ 372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bit = ffs((int)mask) - 1; 373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans regind = ((i << (LG_SIZEOF_INT + 3)) + bit); 375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(regind < bin->nregs); 376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)(((uintptr_t)run) + bin->reg0_offset 377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans + (bin->reg_size * regind)); 378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 379e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Clear bit. */ 380e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mask ^= (1U << bit); 381e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_mask[i] = mask; 382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_rc_incr(run, bin, ret); 384e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 387e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i++; i < bin->regs_mask_nelms; i++) { 389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mask = run->regs_mask[i]; 390e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mask != 0) { 391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Usable allocation found. */ 392e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bit = ffs((int)mask) - 1; 393e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 394e476f8a161d445211fd6e54fe370275196e66bcbJason Evans regind = ((i << (LG_SIZEOF_INT + 3)) + bit); 395e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(regind < bin->nregs); 396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)(((uintptr_t)run) + bin->reg0_offset 397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans + (bin->reg_size * regind)); 398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Clear bit. */ 400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mask ^= (1U << bit); 401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_mask[i] = mask; 402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 403e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 404e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Make a note that nothing before this element 405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * contains a free region. 406e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 407e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_minelm = i; /* Low payoff: + (mask == 0); */ 408e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 409e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_rc_incr(run, bin, ret); 410e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 411e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 412e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 413e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 414e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Not reached. */ 415e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(0); 416e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 417e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 418e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 419e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void 420e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size) 421e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 422e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned shift, diff, regind, elm, bit; 423e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 424e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 425e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 426e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 427e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Avoid doing division with a variable divisor if possible. Using 428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * actual division here can reduce allocator throughput by over 20%! 429e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 430e476f8a161d445211fd6e54fe370275196e66bcbJason Evans diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset); 431e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 432e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Rescale (factor powers of 2 out of the numerator and denominator). */ 433e476f8a161d445211fd6e54fe370275196e66bcbJason Evans shift = ffs(size) - 1; 434e476f8a161d445211fd6e54fe370275196e66bcbJason Evans diff >>= shift; 435e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size >>= shift; 436e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 437e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size == 1) { 438e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* The divisor was a power of 2. */ 439e476f8a161d445211fd6e54fe370275196e66bcbJason Evans regind = diff; 440e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 442e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * To divide by a number D that is not a power of two we 443e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * multiply by (2^21 / D) and then right shift by 21 positions. 444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 445e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * X / D 446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 447e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * becomes 448e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * (X * size_invs[D - 3]) >> SIZE_INV_SHIFT 450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 451e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * We can omit the first three elements, because we never 452e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * divide by 0, and 1 and 2 are both powers of two, which are 453e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * handled above. 454e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 455e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define SIZE_INV_SHIFT 21 456e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) 457e476f8a161d445211fd6e54fe370275196e66bcbJason Evans static const unsigned size_invs[] = { 458e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(3), 459e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 460e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 462e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 466e476f8a161d445211fd6e54fe370275196e66bcbJason Evans }; 467e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= ((sizeof(size_invs) / sizeof(unsigned)) + 2)) 469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans regind = (diff * size_invs[size - 3]) >> SIZE_INV_SHIFT; 470e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 471e476f8a161d445211fd6e54fe370275196e66bcbJason Evans regind = diff / size; 472e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef SIZE_INV 473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef SIZE_INV_SHIFT 474e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 475e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(diff == regind * size); 476e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(regind < bin->nregs); 477e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 478e476f8a161d445211fd6e54fe370275196e66bcbJason Evans elm = regind >> (LG_SIZEOF_INT + 3); 479e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (elm < run->regs_minelm) 480e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_minelm = elm; 481e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bit = regind - (elm << (LG_SIZEOF_INT + 3)); 482e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((run->regs_mask[elm] & (1U << bit)) == 0); 483e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_mask[elm] |= (1U << bit); 484e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 485e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_rc_decr(run, bin, ptr); 486e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 487e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 488e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 489e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, 490e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool zero) 491e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 492e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 493e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i; 494e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 495e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 496e476f8a161d445211fd6e54fe370275196e66bcbJason Evans old_ndirty = chunk->ndirty; 497e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) 498e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> PAGE_SHIFT); 499e476f8a161d445211fd6e54fe370275196e66bcbJason Evans total_pages = (chunk->map[run_ind].bits & ~PAGE_MASK) >> 500e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT; 501e476f8a161d445211fd6e54fe370275196e66bcbJason Evans need_pages = (size >> PAGE_SHIFT); 502e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages > 0); 503e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages <= total_pages); 504e476f8a161d445211fd6e54fe370275196e66bcbJason Evans rem_pages = total_pages - need_pages; 505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 506e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_remove(&arena->runs_avail, &chunk->map[run_ind]); 507e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive += need_pages; 508e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 509e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Keep track of trailing unused pages for later use. */ 510e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (rem_pages > 0) { 511e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+need_pages].bits = (rem_pages << 512e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT) | (chunk->map[run_ind+need_pages].bits & 513e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 514e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+total_pages-1].bits = (rem_pages << 515e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT) | (chunk->map[run_ind+total_pages-1].bits & 516e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 517e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_insert(&arena->runs_avail, 518e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_ind+need_pages]); 519e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 520e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 521e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < need_pages; i++) { 522e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Zero if necessary. */ 523e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero) { 524e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((chunk->map[run_ind + i].bits & CHUNK_MAP_ZEROED) 525e476f8a161d445211fd6e54fe370275196e66bcbJason Evans == 0) { 526e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)chunk + ((run_ind 527e476f8a161d445211fd6e54fe370275196e66bcbJason Evans + i) << PAGE_SHIFT)), 0, PAGE_SIZE); 528e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* CHUNK_MAP_ZEROED is cleared below. */ 529e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 530e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 531e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 532e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Update dirty page accounting. */ 533e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) { 534e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty--; 535e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty--; 536e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* CHUNK_MAP_DIRTY is cleared below. */ 537e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 538e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 539e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize the chunk map. */ 540e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (large) { 541e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind + i].bits = CHUNK_MAP_LARGE 542e476f8a161d445211fd6e54fe370275196e66bcbJason Evans | CHUNK_MAP_ALLOCATED; 543e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 544e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind + i].bits = (i << CHUNK_MAP_PG_SHIFT) 545e476f8a161d445211fd6e54fe370275196e66bcbJason Evans | CHUNK_MAP_ALLOCATED; 546e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 547e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 548e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 549e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (large) { 550e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 551e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Set the run size only in the first element for large runs. 552e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * This is primarily a debugging aid, since the lack of size 553e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * info for trailing pages only matters if the application 554e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * tries to operate on an interior pointer. 555e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 556e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits |= size; 557e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 558e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 559e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Initialize the first page's refcount to 1, so that the run 560e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * header is protected from dirty page purging. 561e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 562e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits += CHUNK_MAP_RC_ONE; 563e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 564e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 565e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 566e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t * 567e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_alloc(arena_t *arena) 568e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 569e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 570e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 571e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 572e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare != NULL) { 573e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena->spare; 574e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 575e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 57641631d00618d7262125e501c91d31b4d70e605faJason Evans bool zero; 57741631d00618d7262125e501c91d31b4d70e605faJason Evans size_t zeroed; 57841631d00618d7262125e501c91d31b4d70e605faJason Evans 57941631d00618d7262125e501c91d31b4d70e605faJason Evans zero = false; 58041631d00618d7262125e501c91d31b4d70e605faJason Evans chunk = (arena_chunk_t *)chunk_alloc(chunksize, &zero); 581e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk == NULL) 582e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 583e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 584e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.mapped += chunksize; 585e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 586e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 587e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->arena = arena; 588e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = false; 589e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 590e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 591e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Claim that no pages are in use, since the header is merely 592e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * overhead. 593e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 594e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty = 0; 595e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 596e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 597e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Initialize the map to contain one maximal free untouched run. 59841631d00618d7262125e501c91d31b4d70e605faJason Evans * Mark the pages as zeroed iff chunk_alloc() returned a zeroed 59941631d00618d7262125e501c91d31b4d70e605faJason Evans * chunk. 600e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 60141631d00618d7262125e501c91d31b4d70e605faJason Evans zeroed = zero ? CHUNK_MAP_ZEROED : 0; 602e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < arena_chunk_header_npages; i++) 603e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[i].bits = 0; 60441631d00618d7262125e501c91d31b4d70e605faJason Evans chunk->map[i].bits = arena_maxclass | zeroed; 60541631d00618d7262125e501c91d31b4d70e605faJason Evans for (i++; i < chunk_npages-1; i++) 60641631d00618d7262125e501c91d31b4d70e605faJason Evans chunk->map[i].bits = zeroed; 60741631d00618d7262125e501c91d31b4d70e605faJason Evans chunk->map[chunk_npages-1].bits = arena_maxclass | zeroed; 608e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 609e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 610e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert the run into the runs_avail tree. */ 611e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_insert(&arena->runs_avail, 612e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[arena_chunk_header_npages]); 613e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 614e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (chunk); 615e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 616e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 617e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 618e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) 619e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 620e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 621e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare != NULL) { 622e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare->dirtied) { 623e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_dirty_remove( 624e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->arena->chunks_dirty, arena->spare); 625e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty -= arena->spare->ndirty; 626e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 627e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk_dealloc((void *)arena->spare, chunksize); 628e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 629e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.mapped -= chunksize; 630e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 631e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 632e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 633e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 634e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove run from runs_avail, regardless of whether this chunk 635e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * will be cached, so that the arena does not use it. Dirty page 636e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * flushing only uses the chunks_dirty tree, so leaving this chunk in 637e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * the chunks_* trees is sufficient for that purpose. 638e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 639e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_remove(&arena->runs_avail, 640e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[arena_chunk_header_npages]); 641e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 642e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = chunk; 643e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 644e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 645e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 646e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) 647e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 648e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 649e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 650e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm, key; 651e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 652e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 653e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 654e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 655e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Search the arena's chunks for the lowest best fit. */ 656e476f8a161d445211fd6e54fe370275196e66bcbJason Evans key.bits = size | CHUNK_MAP_KEY; 657e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key); 658e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 659e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 660e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)mapelm - (uintptr_t)run_chunk->map) 661e476f8a161d445211fd6e54fe370275196e66bcbJason Evans / sizeof(arena_chunk_map_t); 662e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 663e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind 664e476f8a161d445211fd6e54fe370275196e66bcbJason Evans << PAGE_SHIFT)); 665e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, run, size, large, zero); 666e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 667e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 668e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 669e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 670e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * No usable runs. Create a new chunk from which to allocate the run. 671e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 672e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena_chunk_alloc(arena); 673e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk == NULL) 674e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 675e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)chunk + (arena_chunk_header_npages << 676e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)); 677e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Update page map. */ 678e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, run, size, large, zero); 679e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 680e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 681e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 682e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 683e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_purge(arena_t *arena) 684e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 685e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 686e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i, npages; 687e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 688e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty = 0; 689e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 690e476f8a161d445211fd6e54fe370275196e66bcbJason Evans rb_foreach_begin(arena_chunk_t, link_dirty, &arena->chunks_dirty, 691e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk) { 692e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(chunk->dirtied); 693e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ndirty += chunk->ndirty; 694e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } rb_foreach_end(arena_chunk_t, link_dirty, &arena->chunks_dirty, chunk) 695e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ndirty == arena->ndirty); 696e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 697e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((arena->nactive >> opt_lg_dirty_mult) < arena->ndirty); 698e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 699e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 700e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.npurge++; 701e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 702e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 703e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 704e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Iterate downward through chunks until enough dirty memory has been 705e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * purged. Terminate as soon as possible in order to minimize the 706e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * number of system calls, even if a chunk has only been partially 707e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * purged. 708e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 709e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 710e476f8a161d445211fd6e54fe370275196e66bcbJason Evans while ((arena->nactive >> (opt_lg_dirty_mult + 1)) < arena->ndirty) { 711e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena_chunk_tree_dirty_last(&arena->chunks_dirty); 712e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(chunk != NULL); 713e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 714e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = chunk_npages - 1; chunk->ndirty > 0; i--) { 715e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(i >= arena_chunk_header_npages); 716e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->map[i].bits & CHUNK_MAP_DIRTY) { 717e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[i].bits ^= CHUNK_MAP_DIRTY; 718e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Find adjacent dirty run(s). */ 719e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (npages = 1; i > arena_chunk_header_npages 720e476f8a161d445211fd6e54fe370275196e66bcbJason Evans && (chunk->map[i - 1].bits & 721e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_DIRTY); npages++) { 722e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i--; 723e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[i].bits ^= CHUNK_MAP_DIRTY; 724e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 725e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty -= npages; 726e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty -= npages; 727e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 728e476f8a161d445211fd6e54fe370275196e66bcbJason Evans madvise((void *)((uintptr_t)chunk + (i << 729e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)), (npages << PAGE_SHIFT), 730e476f8a161d445211fd6e54fe370275196e66bcbJason Evans MADV_DONTNEED); 731e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 732e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmadvise++; 733e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.purged += npages; 734e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 735e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((arena->nactive >> (opt_lg_dirty_mult + 1)) 736e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >= arena->ndirty) 737e476f8a161d445211fd6e54fe370275196e66bcbJason Evans break; 738e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 739e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 740e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 741e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->ndirty == 0) { 742e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_dirty_remove(&arena->chunks_dirty, 743e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk); 744e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = false; 745e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 746e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 747e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 748e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 749e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 750e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) 751e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 752e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 753e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size, run_ind, run_pages; 754e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 755e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 756e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) 757e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> PAGE_SHIFT); 758e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run_ind >= arena_chunk_header_npages); 759e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run_ind < chunk_npages); 760e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((chunk->map[run_ind].bits & CHUNK_MAP_LARGE) != 0) 761e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = chunk->map[run_ind].bits & ~PAGE_MASK; 762e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 763e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = run->bin->run_size; 764e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_pages = (size >> PAGE_SHIFT); 765e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive -= run_pages; 766e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 767e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Mark pages as unallocated in the chunk map. */ 768e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (dirty) { 769e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 770e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 771e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < run_pages; i++) { 772e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 773e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * When (dirty == true), *all* pages within the run 774e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * need to have their dirty bits set, because only 775e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * small runs can create a mixture of clean/dirty 776e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * pages, but such runs are passed to this function 777e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * with (dirty == false). 778e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 779e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) 780e476f8a161d445211fd6e54fe370275196e66bcbJason Evans == 0); 781e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty++; 782e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty++; 783e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind + i].bits = CHUNK_MAP_DIRTY; 784e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 785e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 786e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 787e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 788e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < run_pages; i++) { 789e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind + i].bits &= ~(CHUNK_MAP_LARGE | 790e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED); 791e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 792e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 793e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits & 794e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 795e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+run_pages-1].bits = size | 796e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (chunk->map[run_ind+run_pages-1].bits & CHUNK_MAP_FLAGS_MASK); 797e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 798e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce forward. */ 799e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run_ind + run_pages < chunk_npages && 800e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (chunk->map[run_ind+run_pages].bits & CHUNK_MAP_ALLOCATED) == 0) { 801e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t nrun_size = chunk->map[run_ind+run_pages].bits & 802e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ~PAGE_MASK; 803e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 804e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 805e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove successor from runs_avail; the coalesced run is 806e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 807e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 808e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_remove(&arena->runs_avail, 809e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_ind+run_pages]); 810e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 811e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += nrun_size; 812e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_pages = size >> PAGE_SHIFT; 813e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 814e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[run_ind+run_pages-1].bits & ~PAGE_MASK) 815e476f8a161d445211fd6e54fe370275196e66bcbJason Evans == nrun_size); 816e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits & 817e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 818e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+run_pages-1].bits = size | 819e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (chunk->map[run_ind+run_pages-1].bits & 820e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 821e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 822e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 823e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce backward. */ 824e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run_ind > arena_chunk_header_npages && (chunk->map[run_ind-1].bits & 825e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED) == 0) { 826e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t prun_size = chunk->map[run_ind-1].bits & ~PAGE_MASK; 827e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 828e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind -= prun_size >> PAGE_SHIFT; 829e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 830e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 831e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove predecessor from runs_avail; the coalesced run is 832e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 833e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 834e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_remove(&arena->runs_avail, 835e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_ind]); 836e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 837e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += prun_size; 838e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_pages = size >> PAGE_SHIFT; 839e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 840e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[run_ind].bits & ~PAGE_MASK) == prun_size); 841e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits & 842e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 843e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+run_pages-1].bits = size | 844e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (chunk->map[run_ind+run_pages-1].bits & 845e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 846e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 847e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 848e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert into runs_avail, now that coalescing is complete. */ 849e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_insert(&arena->runs_avail, &chunk->map[run_ind]); 850e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 851e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Deallocate chunk if it is now completely unused. */ 852e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((chunk->map[arena_chunk_header_npages].bits & (~PAGE_MASK | 853e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED)) == arena_maxclass) 854e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_dealloc(arena, chunk); 855e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 856e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (dirty) { 857e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->dirtied == false) { 858e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_dirty_insert(&arena->chunks_dirty, 859e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk); 860e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = true; 861e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 862e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 863e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Enforce opt_lg_dirty_mult. */ 864e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_dirty_mult >= 0 && (arena->nactive >> 865e476f8a161d445211fd6e54fe370275196e66bcbJason Evans opt_lg_dirty_mult) < arena->ndirty) 866e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_purge(arena); 867e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 868e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 869e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 870e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 871e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 872e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize) 873e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 874e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; 875e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t head_npages = (oldsize - newsize) >> PAGE_SHIFT; 876e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 877e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 878e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 879e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 880e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 881e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * leading run as separately allocated. 882e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 883e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[pageind].bits & CHUNK_MAP_DIRTY) == 0); 884e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind].bits = (oldsize - newsize) | CHUNK_MAP_LARGE | 885e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 886e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[pageind+head_npages].bits & CHUNK_MAP_DIRTY) == 0); 887e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind+head_npages].bits = newsize | CHUNK_MAP_LARGE | 888e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 889e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 890e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, run, false); 891e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 892e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 893e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 894e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 895e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize, bool dirty) 896e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 897e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; 898e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t npages = newsize >> PAGE_SHIFT; 899e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 900e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 901e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 902e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 903e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 904e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * trailing run as separately allocated. 905e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 906e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[pageind].bits & CHUNK_MAP_DIRTY) == 0); 907e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind].bits = newsize | CHUNK_MAP_LARGE | 908e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 909e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[pageind+npages].bits & CHUNK_MAP_DIRTY) == 0); 910e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind+npages].bits = (oldsize - newsize) | CHUNK_MAP_LARGE 911e476f8a161d445211fd6e54fe370275196e66bcbJason Evans | CHUNK_MAP_ALLOCATED; 912e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 913e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 914e476f8a161d445211fd6e54fe370275196e66bcbJason Evans dirty); 915e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 916e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 917e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 918e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 919e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 920e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm; 921e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 922e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, remainder; 923e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 924e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Look for a usable run. */ 925e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapelm = arena_run_tree_first(&bin->runs); 926e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 927e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 928e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 929e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 930e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* run is guaranteed to have available space. */ 931e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_remove(&bin->runs, mapelm); 932e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 933e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 934e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 935e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sizeof(arena_chunk_map_t)); 936e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 937e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((mapelm->bits & CHUNK_MAP_PG_MASK) >> CHUNK_MAP_PG_SHIFT)) 938e476f8a161d445211fd6e54fe370275196e66bcbJason Evans << PAGE_SHIFT)); 939e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 940e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.reruns++; 941e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 942e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 943e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 944e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* No existing runs have any space available. */ 945e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 946e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Allocate a new run. */ 947e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = arena_run_alloc(arena, bin->run_size, false, false); 948e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run == NULL) 949e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 950e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 951e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize run internals. */ 952e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->bin = bin; 953e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 954e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < bin->regs_mask_nelms - 1; i++) 955e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_mask[i] = UINT_MAX; 956e476f8a161d445211fd6e54fe370275196e66bcbJason Evans remainder = bin->nregs & ((1U << (LG_SIZEOF_INT + 3)) - 1); 957e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (remainder == 0) 958e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_mask[i] = UINT_MAX; 959e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 960e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* The last element has spare bits that need to be unset. */ 961e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_mask[i] = (UINT_MAX >> ((1U << (LG_SIZEOF_INT + 3)) 962e476f8a161d445211fd6e54fe370275196e66bcbJason Evans - remainder)); 963e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 964e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 965e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_minelm = 0; 966e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 967e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->nfree = bin->nregs; 968e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 969e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->magic = ARENA_RUN_MAGIC; 970e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 971e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 972e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 973e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nruns++; 974e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.curruns++; 975e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->stats.curruns > bin->stats.highruns) 976e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.highruns = bin->stats.curruns; 977e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 978e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 979e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 980e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 981e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* bin->runcur must have space available before this function is called. */ 982e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void * 983e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_malloc_easy(arena_t *arena, arena_bin_t *bin, arena_run_t *run) 984e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 985e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 986e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 987e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 988e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->nfree > 0); 989e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 990e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_run_reg_alloc(run, bin); 991e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ret != NULL); 992e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->nfree--; 993e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 994e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 995e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 996e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 997e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Re-fill bin->runcur, then call arena_bin_malloc_easy(). */ 998e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void * 999e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1000e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1001e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1002e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = arena_bin_nonfull_run_get(arena, bin); 1003e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->runcur == NULL) 1004e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1005e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->runcur->magic == ARENA_RUN_MAGIC); 1006e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->runcur->nfree > 0); 1007e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1008e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (arena_bin_malloc_easy(arena, bin, bin->runcur)); 1009e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1010e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1011e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 1012e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1013e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind) 1014e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1015e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, nfill; 1016e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1017e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1018e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr; 1019e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1020e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tbin->ncached == 0); 1021e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1022e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 1023e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1024e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0, nfill = (tcache_nslots >> 1); i < nfill; i++) { 1025e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 1026e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ptr = arena_bin_malloc_easy(arena, bin, run); 1027e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1028e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ptr = arena_bin_malloc_hard(arena, bin); 1029e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ptr == NULL) { 1030e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (i > 0) { 1031e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1032e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Move valid pointers to the base of 1033e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * tbin->slots. 1034e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1035e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memmove(&tbin->slots[0], 1036e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &tbin->slots[nfill - i], 1037e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i * sizeof(void *)); 1038e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1039e476f8a161d445211fd6e54fe370275196e66bcbJason Evans break; 1040e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1041e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1042e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Fill slots such that the objects lowest in memory come last. 1043e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * This causes tcache to use low objects first. 1044e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1045e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->slots[nfill - 1 - i] = ptr; 1046e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1047e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1048e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nfills++; 1049e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 1050e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->reg_size <= small_maxclass) { 1051e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_small += (i - tbin->ncached); 1052e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_small += (i - tbin->ncached) * 1053e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size; 1054e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_small += tbin->tstats.nrequests; 1055e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1056e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_medium += (i - tbin->ncached); 1057e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_medium += (i - tbin->ncached) * 1058e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size; 1059e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_medium += tbin->tstats.nrequests; 1060e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1061e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->tstats.nrequests = 0; 1062e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1063e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1064e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->ncached = i; 1065e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tbin->ncached > tbin->high_water) 1066e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->high_water = tbin->ncached; 1067e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1068e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1069e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1070e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 1071e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Calculate bin->run_size such that it meets the following constraints: 1072e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 1073e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) bin->run_size >= min_run_size 1074e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) bin->run_size <= arena_maxclass 1075e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) bin->run_size <= RUN_MAX_SMALL 1076e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 1077e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) run header size < PAGE_SIZE 1078e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 1079e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * bin->nregs, bin->regs_mask_nelms, and bin->reg0_offset are 1080e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * also calculated here, since these settings are all interdependent. 1081e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1082e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic size_t 1083e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size) 1084e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1085e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t try_run_size, good_run_size; 1086e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned good_nregs, good_mask_nelms, good_reg0_offset; 1087e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned try_nregs, try_mask_nelms, try_reg0_offset; 1088e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1089e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(min_run_size >= PAGE_SIZE); 1090e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(min_run_size <= arena_maxclass); 1091e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(min_run_size <= RUN_MAX_SMALL); 1092e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1093e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1094e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Calculate known-valid settings before entering the run_size 1095e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * expansion loop, so that the first part of the loop always copies 1096e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * valid settings. 1097e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 1098e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The do..while loop iteratively reduces the number of regions until 1099e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * the run header and the regions no longer overlap. A closed formula 1100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * would be quite messy, since there is an interdependency between the 1101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * header's mask length and the number of regions. 1102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_run_size = min_run_size; 1104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size) 1105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans + 1; /* Counter-act try_nregs-- in loop. */ 1106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans do { 1107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_nregs--; 1108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_mask_nelms = (try_nregs >> (LG_SIZEOF_INT + 3)) + 1109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((try_nregs & ((1U << (LG_SIZEOF_INT + 3)) - 1)) ? 1 : 0); 1110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_reg0_offset = try_run_size - (try_nregs * bin->reg_size); 1111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } while (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1)) 1112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans > try_reg0_offset); 1113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* run_size expansion loop. */ 1115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans do { 1116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Copy valid settings before trying more aggressive settings. 1118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans good_run_size = try_run_size; 1120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans good_nregs = try_nregs; 1121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans good_mask_nelms = try_mask_nelms; 1122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans good_reg0_offset = try_reg0_offset; 1123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try more aggressive settings. */ 1125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_run_size += PAGE_SIZE; 1126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / 1127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */ 1128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans do { 1129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_nregs--; 1130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_mask_nelms = (try_nregs >> (LG_SIZEOF_INT + 3)) + 1131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((try_nregs & ((1U << (LG_SIZEOF_INT + 3)) - 1)) ? 1132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1 : 0); 1133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_reg0_offset = try_run_size - (try_nregs * 1134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size); 1135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } while (sizeof(arena_run_t) + (sizeof(unsigned) * 1136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (try_mask_nelms - 1)) > try_reg0_offset); 1137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } while (try_run_size <= arena_maxclass && try_run_size <= RUN_MAX_SMALL 1138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX 1139e476f8a161d445211fd6e54fe370275196e66bcbJason Evans && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size 1140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans && (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1))) 1141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans < PAGE_SIZE); 1142e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(sizeof(arena_run_t) + (sizeof(unsigned) * (good_mask_nelms - 1)) 1144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans <= good_reg0_offset); 1145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((good_mask_nelms << (LG_SIZEOF_INT + 3)) >= good_nregs); 1146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Copy final settings. */ 1148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->run_size = good_run_size; 1149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->nregs = good_nregs; 1150e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->regs_mask_nelms = good_mask_nelms; 1151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg0_offset = good_reg0_offset; 1152e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (good_run_size); 1154e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1155e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1156e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1157e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_small(arena_t *arena, size_t size, bool zero) 1158e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1159e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1160e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1162e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 1163e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1164e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = small_size2bin[size]; 1165e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(binind < mbin0); 1166e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 1167e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = bin->reg_size; 1168e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1169e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1170e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 1171e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_easy(arena, bin, run); 1172e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_hard(arena, bin); 1174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1175e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1177e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1178e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1179e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1181e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 1182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (isthreaded == false) { 1183e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 1184e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nrequests++; 1185e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_small++; 1186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 1187e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 1189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_small += size; 1190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1192e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 1194e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1195e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1198e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1200e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 1201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1204e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1206e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1207e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_medium(arena_t *arena, size_t size, bool zero) 1208e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1209e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1212e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 1213e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = MEDIUM_CEILING(size); 1215e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = mbin0 + ((size - medium_min) >> lg_mspace); 1216e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(binind < nbins); 1217e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 1218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->reg_size == size); 1219e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1220e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 1222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_easy(arena, bin, run); 1223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1224e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_hard(arena, bin); 1225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1226e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 1233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (isthreaded == false) { 1234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 1235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nrequests++; 1236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_medium++; 1237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 1238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1239e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 1240e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_medium += size; 1241e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 1245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1251e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 1252e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1253e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1255e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1256e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1257e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void * 1258e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_large(arena_t *arena, size_t size, bool zero) 1259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1260e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1261e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1262e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1263e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = PAGE_CEILING(size); 1264e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1265e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)arena_run_alloc(arena, size, true, zero); 1266e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1267e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1270e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_large++; 1272e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large += size; 1273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1276e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1277e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1279e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 1284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1287e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1293e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1294e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1295e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1296e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc(size_t size, bool zero) 1297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size != 0); 1300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(QUANTUM_CEILING(size) <= arena_maxclass); 1301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1302e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= bin_maxclass) { 1303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 1304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache; 1305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((tcache = tcache_get()) != NULL) 1307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (tcache_alloc(tcache, size, zero)); 1308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= small_maxclass) 1310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (arena_malloc_small(choose_arena(), size, zero)); 1311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 1312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (arena_malloc_medium(choose_arena(), size, 1313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans zero)); 1314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1315e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 1316e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (arena_malloc_large(choose_arena(), size, zero)); 1317e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1318e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Only handles large allocations that require more than page alignment. */ 1320e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1321e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size) 1322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t offset; 1325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 1328e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((alignment & PAGE_MASK) == 0); 1329e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)arena_run_alloc(arena, alloc_size, true, false); 1332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); 1338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans offset = (uintptr_t)ret & (alignment - 1); 1340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((offset & PAGE_MASK) == 0); 1341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(offset < alloc_size); 1342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (offset == 0) 1343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, ret, alloc_size, size, false); 1344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 1345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t leadsize, trailsize; 1346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans leadsize = alignment - offset; 1348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (leadsize > 0) { 1349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_head(arena, chunk, ret, alloc_size, 1350e476f8a161d445211fd6e54fe370275196e66bcbJason Evans alloc_size - leadsize); 1351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)((uintptr_t)ret + leadsize); 1352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans trailsize = alloc_size - leadsize - size; 1355e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (trailsize != 0) { 1356e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Trim trailing space. */ 1357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(trailsize < alloc_size); 1358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, ret, size + trailsize, 1359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size, false); 1360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1361e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_large++; 1365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large += size; 1366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1368e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1379e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1380e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1381e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1384e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1385e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1386e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_is_large(const void *ptr) 1387e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind, mapbits; 1390e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1392e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1393e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1394e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1395e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 1396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits = chunk->map[pageind].bits; 1397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 1398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((mapbits & CHUNK_MAP_LARGE) != 0); 1399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Return the size of the allocation pointed to by ptr. */ 1402e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t 1403e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_salloc(const void *ptr) 1404e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ret; 1406e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1407e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind, mapbits; 1408e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1409e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1410e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1411e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1412e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1413e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 1414e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits = chunk->map[pageind].bits; 1415e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 1416e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 1417e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 1418e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)((pageind - ((mapbits & CHUNK_MAP_PG_MASK) >> 1419e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT)); 1420e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 1421e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = run->bin->reg_size; 1422e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1423e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = mapbits & ~PAGE_MASK; 1424e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ret != 0); 1425e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1426e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1427e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1429e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1430e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1431e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1432e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin) 1433e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1434e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t run_ind; 1435e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1436e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Deallocate run. */ 1437e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run == bin->runcur) 1438e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1439e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (bin->nregs != 1) { 1440e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t run_pageind = (((uintptr_t)run - 1441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)chunk)) >> PAGE_SHIFT; 1442e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *run_mapelm = 1443e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_pageind]; 1444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1445e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * This block's conditional is necessary because if the 1446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * run only contains one region, then it never gets 1447e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted into the non-full runs tree. 1448e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_remove(&bin->runs, run_mapelm); 1450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1451e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1452e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Mark the first page as dirty. The dirty bit for every other page in 1453e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * the run is already properly set, which means we can call 1454e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * arena_run_dalloc(..., false), thus potentially avoiding the needless 1455e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * creation of many dirty pages. 1456e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1457e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT); 1458e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[run_ind].bits & CHUNK_MAP_DIRTY) == 0); 1459e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits |= CHUNK_MAP_DIRTY; 1460e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty++; 1461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty++; 1462e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 1464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->magic = 0; 1465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1466e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, run, false); 1467e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.curruns--; 1469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1470e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1471e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->dirtied == false) { 1472e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_dirty_insert(&arena->chunks_dirty, chunk); 1473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = true; 1474e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1475e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Enforce opt_lg_dirty_mult. */ 1476e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_dirty_mult >= 0 && (arena->nactive >> opt_lg_dirty_mult) < 1477e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty) 1478e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_purge(arena); 1479e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1480e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1481e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1482e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1483e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm) 1484e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1485e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 1486e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1487e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1488e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size; 1489e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1490e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 1491e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1492e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((mapelm->bits & CHUNK_MAP_PG_MASK) >> CHUNK_MAP_PG_SHIFT)) << 1493e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)); 1494e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 1495e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = run->bin; 1496e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = bin->reg_size; 1497e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1498e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1499e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1500e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ptr, 0x5a, size); 1501e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1502e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1503e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_reg_dalloc(run, bin, ptr, size); 1504e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->nfree++; 1505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1506e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run->nfree == bin->nregs) 1507e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 1508e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (run->nfree == 1 && run != bin->runcur) { 1509e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1510e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Make sure that bin->runcur always refers to the lowest 1511e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * non-full run, if one exists. 1512e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1513e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->runcur == NULL) 1514e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = run; 1515e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if ((uintptr_t)run < (uintptr_t)bin->runcur) { 1516e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Switch runcur. */ 1517e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->runcur->nfree > 0) { 1518e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *runcur_chunk = 1519e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_ADDR2BASE(bin->runcur); 1520e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t runcur_pageind = 1521e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (((uintptr_t)bin->runcur - 1522e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)runcur_chunk)) >> PAGE_SHIFT; 1523e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *runcur_mapelm = 1524e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &runcur_chunk->map[runcur_pageind]; 1525e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1526e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert runcur. */ 1527e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_insert(&bin->runs, 1528e476f8a161d445211fd6e54fe370275196e66bcbJason Evans runcur_mapelm); 1529e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1530e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = run; 1531e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1532e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t run_pageind = (((uintptr_t)run - 1533e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)chunk)) >> PAGE_SHIFT; 1534e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *run_mapelm = 1535e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_pageind]; 1536e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1537e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(arena_run_tree_search(&bin->runs, run_mapelm) == 1538e476f8a161d445211fd6e54fe370275196e66bcbJason Evans NULL); 1539e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_insert(&bin->runs, run_mapelm); 1540e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1541e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1542e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1543e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1544e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= small_maxclass) { 1545e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_small -= size; 1546e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.ndalloc_small++; 1547e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1548e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_medium -= size; 1549e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.ndalloc_medium++; 1550e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1551e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1552e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1553e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1554e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1555e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1556b34e8684ec025aa780997c11f847c19fb269755bJason Evansarena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, 1557b34e8684ec025aa780997c11f847c19fb269755bJason Evans arena_stats_t *astats, malloc_bin_stats_t *bstats, 1558b34e8684ec025aa780997c11f847c19fb269755bJason Evans malloc_large_stats_t *lstats) 1559b34e8684ec025aa780997c11f847c19fb269755bJason Evans{ 15603c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans unsigned i; 1561b34e8684ec025aa780997c11f847c19fb269755bJason Evans 1562b34e8684ec025aa780997c11f847c19fb269755bJason Evans *nactive += arena->nactive; 1563b34e8684ec025aa780997c11f847c19fb269755bJason Evans *ndirty += arena->ndirty; 1564b34e8684ec025aa780997c11f847c19fb269755bJason Evans 15654201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans astats->mapped += arena->stats.mapped; 1566b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->npurge += arena->stats.npurge; 1567b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmadvise += arena->stats.nmadvise; 1568b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->purged += arena->stats.purged; 1569b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->allocated_small += arena->stats.allocated_small; 1570b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmalloc_small += arena->stats.nmalloc_small; 1571b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->ndalloc_small += arena->stats.ndalloc_small; 1572b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->allocated_medium += arena->stats.allocated_medium; 1573b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmalloc_medium += arena->stats.nmalloc_medium; 1574b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->ndalloc_medium += arena->stats.ndalloc_medium; 1575b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->allocated_large += arena->stats.allocated_large; 1576b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmalloc_large += arena->stats.nmalloc_large; 1577b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->ndalloc_large += arena->stats.ndalloc_large; 1578b34e8684ec025aa780997c11f847c19fb269755bJason Evans 1579b34e8684ec025aa780997c11f847c19fb269755bJason Evans for (i = 0; i < nbins; i++) { 1580b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].nrequests += arena->bins[i].stats.nrequests; 1581b34e8684ec025aa780997c11f847c19fb269755bJason Evans#ifdef JEMALLOC_TCACHE 1582b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].nfills += arena->bins[i].stats.nfills; 1583b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].nflushes += arena->bins[i].stats.nflushes; 1584b34e8684ec025aa780997c11f847c19fb269755bJason Evans#endif 1585b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].nruns += arena->bins[i].stats.nruns; 1586b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].reruns += arena->bins[i].stats.reruns; 1587b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].highruns += arena->bins[i].stats.highruns; 1588b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].curruns += arena->bins[i].stats.curruns; 1589b34e8684ec025aa780997c11f847c19fb269755bJason Evans } 1590b34e8684ec025aa780997c11f847c19fb269755bJason Evans 15913c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans for (i = 0; i < nlclasses; i++) { 1592b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].nrequests += arena->stats.lstats[i].nrequests; 1593b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].highruns += arena->stats.lstats[i].highruns; 1594b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].curruns += arena->stats.lstats[i].curruns; 1595b34e8684ec025aa780997c11f847c19fb269755bJason Evans } 1596b34e8684ec025aa780997c11f847c19fb269755bJason Evans} 1597e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1598e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1599e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1600e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1601e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1602e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1603e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1604e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1605e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1606e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifndef JEMALLOC_STATS 1607e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1608e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1609e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1610e476f8a161d445211fd6e54fe370275196e66bcbJason Evans { 1611e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) 1612e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 1613e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT; 1614e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size = chunk->map[pageind].bits & ~PAGE_MASK; 1615e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1616e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1617e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1618e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1619e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1620e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1621e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ptr, 0x5a, size); 1622e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1623e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1624e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large -= size; 1625e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns--; 1626e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1627e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1628e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1629e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.ndalloc_large++; 1630e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1631e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1632e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)ptr, true); 1633e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1634e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1635e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1636e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1637e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1638e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size, size_t oldsize) 1639e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1640e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1641e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size < oldsize); 1642e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1643e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1644e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Shrink the run, and make trailing pages available for other 1645e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocations. 1646e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1647e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1648e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 1649e476f8a161d445211fd6e54fe370275196e66bcbJason Evans true); 1650e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1651e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large -= oldsize - size; 1652e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[size >> PAGE_SHIFT].nrequests++; 1653e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[size >> PAGE_SHIFT].curruns++; 1654e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats[size >> PAGE_SHIFT].curruns > 1655e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[size >> PAGE_SHIFT].highruns) { 1656e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[size >> PAGE_SHIFT].highruns = 1657e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[size >> PAGE_SHIFT].curruns; 1658e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1659e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[oldsize >> PAGE_SHIFT].curruns--; 1660e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1661e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1662e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1663e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1664e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1665e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1666e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size, size_t oldsize) 1667e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1668e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 1669e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t npages = oldsize >> PAGE_SHIFT; 1670e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1671e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize == (chunk->map[pageind].bits & ~PAGE_MASK)); 1672e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1673e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to extend the run. */ 1674e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size > oldsize); 1675e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1676e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (pageind + npages < chunk_npages && (chunk->map[pageind+npages].bits 1677e476f8a161d445211fd6e54fe370275196e66bcbJason Evans & CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[pageind+npages].bits & 1678e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ~PAGE_MASK) >= size - oldsize) { 1679e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1680e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The next run is available and sufficiently large. Split the 1681e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * following run, then merge the first part with the existing 1682e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocation. 1683e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1684e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + 1685e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((pageind+npages) << PAGE_SHIFT)), size - oldsize, true, 1686e476f8a161d445211fd6e54fe370275196e66bcbJason Evans false); 1687e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1688e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind].bits = size | CHUNK_MAP_LARGE | 1689e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 1690e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind+npages].bits = CHUNK_MAP_LARGE | 1691e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 1692e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1693e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1694e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large += size - oldsize; 1695e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[size >> PAGE_SHIFT].nrequests++; 1696e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[size >> PAGE_SHIFT].curruns++; 1697e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats[size >> PAGE_SHIFT].curruns > 1698e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[size >> PAGE_SHIFT].highruns) { 1699e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[size >> PAGE_SHIFT].highruns = 1700e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[size >> PAGE_SHIFT].curruns; 1701e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1702e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[oldsize >> PAGE_SHIFT].curruns--; 1703e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1704e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1705e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1706e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1707e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1708e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1709e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 1710e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1711e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1712e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 1713e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Try to resize a large allocation, in order to avoid copying. This will 1714e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * always fail if growing an object, and the following run is already in use. 1715e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1716e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1717e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large(void *ptr, size_t size, size_t oldsize) 1718e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1719e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t psize; 1720e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1721e476f8a161d445211fd6e54fe370275196e66bcbJason Evans psize = PAGE_CEILING(size); 1722e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize == oldsize) { 1723e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Same size class. */ 1724e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1725e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk && size < oldsize) { 1726e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - 1727e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size); 1728e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1729e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1730e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1731e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1732e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1733e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 1734e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1735e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1736e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 1737e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(arena->magic == ARENA_MAGIC); 1738e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1739e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize < oldsize) { 1740e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1741e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Fill before shrinking in order avoid a race. */ 1742e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) { 1743e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, 1744e476f8a161d445211fd6e54fe370275196e66bcbJason Evans oldsize - size); 1745e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1746e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1747e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_ralloc_large_shrink(arena, chunk, ptr, psize, 1748e476f8a161d445211fd6e54fe370275196e66bcbJason Evans oldsize); 1749e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1750e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1751e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 1752e476f8a161d445211fd6e54fe370275196e66bcbJason Evans psize, oldsize); 1753e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1754e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == false && opt_zero) { 1755e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + oldsize), 0, 1756e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size - oldsize); 1757e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1758e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1759e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1760e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1761e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1762e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1763e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1764e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1765e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc(void *ptr, size_t size, size_t oldsize) 1766e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1767e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1768e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t copysize; 1769e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1770e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1771e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Try to avoid moving the allocation. 1772e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 1773e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * posix_memalign() can cause allocation of "large" objects that are 1774e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * smaller than bin_maxclass (in order to meet alignment requirements). 1775e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Therefore, do not assume that (oldsize <= bin_maxclass) indicates 1776e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * ptr refers to a bin-allocated object. 1777e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1778e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= arena_maxclass) { 1779e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena_is_large(ptr) == false ) { 1780e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= small_maxclass) { 1781e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= small_maxclass && 1782e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin[size] == 1783e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin[oldsize]) 1784e476f8a161d445211fd6e54fe370275196e66bcbJason Evans goto IN_PLACE; 1785e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else if (size <= bin_maxclass) { 1786e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (small_maxclass < oldsize && oldsize <= 1787e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin_maxclass && MEDIUM_CEILING(size) == 1788e476f8a161d445211fd6e54fe370275196e66bcbJason Evans MEDIUM_CEILING(oldsize)) 1789e476f8a161d445211fd6e54fe370275196e66bcbJason Evans goto IN_PLACE; 1790e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1791e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1792e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 1793e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size > bin_maxclass) { 1794e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena_ralloc_large(ptr, size, oldsize) == 1795e476f8a161d445211fd6e54fe370275196e66bcbJason Evans false) 1796e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 1797e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1798e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1799e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1800e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1801e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to avoid moving the allocation. */ 1802e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= small_maxclass) { 1803e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= small_maxclass && small_size2bin[size] == 1804e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin[oldsize]) 1805e476f8a161d445211fd6e54fe370275196e66bcbJason Evans goto IN_PLACE; 1806e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else if (size <= bin_maxclass) { 1807e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (small_maxclass < oldsize && oldsize <= bin_maxclass && 1808e476f8a161d445211fd6e54fe370275196e66bcbJason Evans MEDIUM_CEILING(size) == MEDIUM_CEILING(oldsize)) 1809e476f8a161d445211fd6e54fe370275196e66bcbJason Evans goto IN_PLACE; 1810e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1811e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin_maxclass < oldsize && oldsize <= arena_maxclass) { 1812e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size > bin_maxclass); 1813e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena_ralloc_large(ptr, size, oldsize) == false) 1814e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 1815e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1816e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1817e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1818e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1819e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * If we get here, then size and oldsize are different enough that we 1820e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * need to move the object. In that case, fall back to allocating new 1821e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * space and copying. 1822e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1823e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_malloc(size, false); 1824e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) 1825e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1826e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1827e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Junk/zero-filling were already done by arena_malloc(). */ 1828e476f8a161d445211fd6e54fe370275196e66bcbJason Evans copysize = (size < oldsize) ? size : oldsize; 1829e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(ret, ptr, copysize); 1830e476f8a161d445211fd6e54fe370275196e66bcbJason Evans idalloc(ptr); 1831e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1832e476f8a161d445211fd6e54fe370275196e66bcbJason EvansIN_PLACE: 1833e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1834e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk && size < oldsize) 1835e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size); 1836e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero && size > oldsize) 1837e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize); 1838e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1839e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 1840e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1841e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1842e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 1843e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_new(arena_t *arena, unsigned ind) 1844e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1845e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 1846e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1847e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t prev_run_size; 1848e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1849e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (malloc_mutex_init(&arena->lock)) 1850e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 1851e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1852e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1853e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&arena->stats, 0, sizeof(arena_stats_t)); 1854e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats = (malloc_large_stats_t *)base_alloc( 1855e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sizeof(malloc_large_stats_t) * ((chunksize - PAGE_SIZE) >> 1856e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)); 1857e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats == NULL) 1858e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 1859e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(arena->stats.lstats, 0, sizeof(malloc_large_stats_t) * 1860e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((chunksize - PAGE_SIZE) >> PAGE_SHIFT)); 1861e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 1862e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_new(&arena->tcache_ql); 1863e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 1864e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1865e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1866e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TRACE 1867e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_trace) { 1868e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* "jemtr.<pid>.<arena>" */ 1869e476f8a161d445211fd6e54fe370275196e66bcbJason Evans char buf[UMAX2S_BUFSIZE]; 1870e476f8a161d445211fd6e54fe370275196e66bcbJason Evans char filename[6 + UMAX2S_BUFSIZE + 1 + UMAX2S_BUFSIZE + 1]; 1871e476f8a161d445211fd6e54fe370275196e66bcbJason Evans char *s; 1872e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, slen; 1873e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1874e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->trace_buf_end = 0; 1875e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1876e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 0; 1877e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1878e476f8a161d445211fd6e54fe370275196e66bcbJason Evans s = "jemtr."; 1879e476f8a161d445211fd6e54fe370275196e66bcbJason Evans slen = strlen(s); 1880e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(&filename[i], s, slen); 1881e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i += slen; 1882e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1883e476f8a161d445211fd6e54fe370275196e66bcbJason Evans s = umax2s(getpid(), 10, buf); 1884e476f8a161d445211fd6e54fe370275196e66bcbJason Evans slen = strlen(s); 1885e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(&filename[i], s, slen); 1886e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i += slen; 1887e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1888e476f8a161d445211fd6e54fe370275196e66bcbJason Evans s = "."; 1889e476f8a161d445211fd6e54fe370275196e66bcbJason Evans slen = strlen(s); 1890e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(&filename[i], s, slen); 1891e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i += slen; 1892e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1893e476f8a161d445211fd6e54fe370275196e66bcbJason Evans s = umax2s(ind, 10, buf); 1894e476f8a161d445211fd6e54fe370275196e66bcbJason Evans slen = strlen(s); 1895e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(&filename[i], s, slen); 1896e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i += slen; 1897e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1898e476f8a161d445211fd6e54fe370275196e66bcbJason Evans filename[i] = '\0'; 1899e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1900e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->trace_fd = creat(filename, 0644); 1901e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->trace_fd == -1) { 1902e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_write4("<jemalloc>", 1903e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ": creat(\"", filename, "\", O_RDWR) failed\n"); 1904e476f8a161d445211fd6e54fe370275196e66bcbJason Evans abort(); 1905e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1906e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1907e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1908e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1909e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize chunks. */ 1910e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_dirty_new(&arena->chunks_dirty); 1911e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 1912e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1913e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive = 0; 1914e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty = 0; 1915e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1916e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_new(&arena->runs_avail); 1917e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1918e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize bins. */ 1919e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = PAGE_SIZE; 1920e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1921e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 0; 1922e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 1923e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* (2^n)-spaced tiny bins. */ 1924e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins; i++) { 1925e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 1926e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1927e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 1928e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1929e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = (1U << (LG_TINY_MIN + i)); 1930e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1931e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 1932e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1933e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1934e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 1935e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1936e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1937e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1938e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1939e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Quantum-spaced bins. */ 1940e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins + nqbins; i++) { 1941e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 1942e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1943e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 1944e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1945e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = (i - ntbins + 1) << LG_QUANTUM; 1946e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1947e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 1948e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1949e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1950e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 1951e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1952e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1953e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1954e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Cacheline-spaced bins. */ 1955e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins + nqbins + ncbins; i++) { 1956e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 1957e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1958e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 1959e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1960e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = cspace_min + ((i - (ntbins + nqbins)) << 1961e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_CACHELINE); 1962e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1963e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 1964e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1965e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1966e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 1967e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1968e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1969e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1970e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Subpage-spaced bins. */ 1971e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins + nqbins + ncbins + nsbins; i++) { 1972e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 1973e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1974e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 1975e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1976e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = sspace_min + ((i - (ntbins + nqbins + ncbins)) 1977e476f8a161d445211fd6e54fe370275196e66bcbJason Evans << LG_SUBPAGE); 1978e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1979e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 1980e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1981e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1982e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 1983e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1984e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1985e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1986e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Medium bins. */ 1987e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < nbins; i++) { 1988e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 1989e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1990e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 1991e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1992e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = medium_min + ((i - (ntbins + nqbins + ncbins + 1993e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nsbins)) << lg_mspace); 1994e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1995e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 1996e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1997e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1998e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 1999e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2000e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2001e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2002e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2003e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->magic = ARENA_MAGIC; 2004e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2005e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2006e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2007e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2008e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2009e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 2010e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Compute the smallest power of 2 that is >= x. */ 2011e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic size_t 2012e476f8a161d445211fd6e54fe370275196e66bcbJason Evanspow2_ceil(size_t x) 2013e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2014e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2015e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x--; 2016e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 1; 2017e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 2; 2018e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 4; 2019e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 8; 2020e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 16; 2021e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (SIZEOF_PTR == 8) 2022e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 32; 2023e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2024e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x++; 2025e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (x); 2026e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2027e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2028e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2029e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2030e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 2031e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssmall_size2bin_validate(void) 2032e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2033e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i, size, binind; 2034e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2035e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[0] == 0xffU); 2036e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 1; 2037e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TINY 2038e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Tiny. */ 2039e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < (1U << LG_TINY_MIN); i++) { 2040e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(1U << LG_TINY_MIN); 2041e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 2042e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 2043e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2044e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < qspace_min; i++) { 2045e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(i); 2046e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 2047e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 2048e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2049e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 2050e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Quantum-spaced. */ 2051e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= qspace_max; i++) { 2052e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = QUANTUM_CEILING(i); 2053e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + (size >> LG_QUANTUM) - 1; 2054e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 2055e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2056e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Cacheline-spaced. */ 2057e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= cspace_max; i++) { 2058e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = CACHELINE_CEILING(i); 2059e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ((size - cspace_min) >> 2060e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_CACHELINE); 2061e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 2062e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2063e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Sub-page. */ 2064e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= sspace_max; i++) { 2065e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = SUBPAGE_CEILING(i); 2066e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ncbins + ((size - sspace_min) 2067e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> LG_SUBPAGE); 2068e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 2069e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2070e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2071e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2072e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2073e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 2074e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssmall_size2bin_init(void) 2075e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2076e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2077e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT 2078e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT 2079e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || sizeof(const_small_size2bin) != small_maxclass + 1) 2080e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (small_size2bin_init_hard()); 2081e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2082e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin = const_small_size2bin; 2083e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2084e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(sizeof(const_small_size2bin) == small_maxclass + 1); 2085e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin_validate(); 2086e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2087e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2088e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2089e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2090e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 2091e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssmall_size2bin_init_hard(void) 2092e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2093e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i, size, binind; 2094e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uint8_t *custom_small_size2bin; 2095e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2096e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT 2097e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT 2098e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || sizeof(const_small_size2bin) != small_maxclass + 1); 2099e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin = (uint8_t *)base_alloc(small_maxclass + 1); 2101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (custom_small_size2bin == NULL) 2102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[0] = 0xffU; 2105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 1; 2106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 2107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Tiny. */ 2108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < (1U << LG_TINY_MIN); i++) { 2109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(1U << LG_TINY_MIN); 2110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 2111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < qspace_min; i++) { 2114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(i); 2115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 2116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Quantum-spaced. */ 2120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= qspace_max; i++) { 2121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = QUANTUM_CEILING(i); 2122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + (size >> LG_QUANTUM) - 1; 2123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Cacheline-spaced. */ 2126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= cspace_max; i++) { 2127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = CACHELINE_CEILING(i); 2128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ((size - cspace_min) >> 2129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_CACHELINE); 2130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Sub-page. */ 2133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= sspace_max; i++) { 2134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = SUBPAGE_CEILING(i); 2135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ncbins + ((size - sspace_min) >> 2136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_SUBPAGE); 2137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2139e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin = custom_small_size2bin; 2141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2142e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin_validate(); 2143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2147e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 2148e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_boot0(void) 2149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2150e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Set variables according to the value of opt_lg_[qc]space_max. */ 2152e476f8a161d445211fd6e54fe370275196e66bcbJason Evans qspace_max = (1U << opt_lg_qspace_max); 2153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans cspace_min = CACHELINE_CEILING(qspace_max); 2154e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (cspace_min == qspace_max) 2155e476f8a161d445211fd6e54fe370275196e66bcbJason Evans cspace_min += CACHELINE; 2156e476f8a161d445211fd6e54fe370275196e66bcbJason Evans cspace_max = (1U << opt_lg_cspace_max); 2157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sspace_min = SUBPAGE_CEILING(cspace_max); 2158e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (sspace_min == cspace_max) 2159e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sspace_min += SUBPAGE; 2160e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(sspace_min < PAGE_SIZE); 2161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sspace_max = PAGE_SIZE - SUBPAGE; 2162e476f8a161d445211fd6e54fe370275196e66bcbJason Evans medium_max = (1U << opt_lg_medium_max); 2163e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2164e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 2165e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(LG_QUANTUM >= LG_TINY_MIN); 2166e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2167e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ntbins <= LG_QUANTUM); 2168e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nqbins = qspace_max >> LG_QUANTUM; 2169e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ncbins = ((cspace_max - cspace_min) >> LG_CACHELINE) + 1; 2170e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nsbins = ((sspace_max - sspace_min) >> LG_SUBPAGE) + 1; 2171e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2172e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Compute medium size class spacing and the number of medium size 2174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * classes. Limit spacing to no more than pagesize, but if possible 2175e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * use the smallest spacing that does not exceed NMBINS_MAX medium size 2176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * classes. 2177e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2178e476f8a161d445211fd6e54fe370275196e66bcbJason Evans lg_mspace = LG_SUBPAGE; 2179e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nmbins = ((medium_max - medium_min) >> lg_mspace) + 1; 2180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans while (lg_mspace < PAGE_SHIFT && nmbins > NMBINS_MAX) { 2181e476f8a161d445211fd6e54fe370275196e66bcbJason Evans lg_mspace = lg_mspace + 1; 2182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nmbins = ((medium_max - medium_min) >> lg_mspace) + 1; 2183e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2184e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mspace_mask = (1U << lg_mspace) - 1U; 2185e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mbin0 = ntbins + nqbins + ncbins + nsbins; 2187e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nbins = mbin0 + nmbins; 2188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The small_size2bin lookup table uses uint8_t to encode each bin 2190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * index, so we cannot support more than 256 small size classes. This 2191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * limit is difficult to exceed (not even possible with 16B quantum and 2192e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 4KiB pages), and such configurations are impractical, but 2193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * nonetheless we need to protect against this case in order to avoid 2194e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * undefined behavior. 2195e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mbin0 > 256) { 2197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans char line_buf[UMAX2S_BUFSIZE]; 2198e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_write4("<jemalloc>: Too many small size classes (", 2199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans umax2s(mbin0, 10, line_buf), " > max 256)\n", ""); 2200e476f8a161d445211fd6e54fe370275196e66bcbJason Evans abort(); 2201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (small_size2bin_init()) 2204e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2208e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2209e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 2210e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_boot1(void) 2211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2212e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t header_size; 2213e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2215e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Compute the header size such that it is large enough to contain the 2216e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * page map. 2217e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans header_size = sizeof(arena_chunk_t) + 2219e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (sizeof(arena_chunk_map_t) * (chunk_npages - 1)); 2220e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_header_npages = (header_size >> PAGE_SHIFT) + 2221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((header_size & PAGE_MASK) != 0); 2222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_maxclass = chunksize - (arena_chunk_header_npages << PAGE_SHIFT); 2223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2224