arena.c revision f3ff75289be32382fa455b4436871e4958fe6bf9
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_ARENA_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; 8e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t opt_lg_medium_max = LG_MEDIUM_MAX_DEFAULT; 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evansssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 11e476f8a161d445211fd6e54fe370275196e66bcbJason Evansuint8_t const *small_size2bin; 12e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Various bin-related settings. */ 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nqbins; 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned ncbins; 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nsbins; 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nmbins; 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned nbins; 19e476f8a161d445211fd6e54fe370275196e66bcbJason Evansunsigned mbin0; 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t qspace_max; 21e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t cspace_min; 22e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t cspace_max; 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t sspace_min; 24e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t sspace_max; 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t medium_max; 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t lg_mspace; 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t mspace_mask; 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * const_small_size2bin is a static constant lookup table that in the common 32e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * case can be used as-is for small_size2bin. For dynamically linked programs, 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * this avoids a page of memory overhead per process. 34e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 35e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_1(i) i, 36e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_2(i) S2B_1(i) S2B_1(i) 37e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_4(i) S2B_2(i) S2B_2(i) 38e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_8(i) S2B_4(i) S2B_4(i) 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_16(i) S2B_8(i) S2B_8(i) 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_32(i) S2B_16(i) S2B_16(i) 41e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_64(i) S2B_32(i) S2B_32(i) 42e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_128(i) S2B_64(i) S2B_64(i) 43e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_256(i) S2B_128(i) S2B_128(i) 44e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The number of elements in const_small_size2bin is dependent on page size 46e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * and on the definition for SUBPAGE. If SUBPAGE changes, the '- 255' must also 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * change, along with the addition/removal of static lookup table element 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * definitions. 49e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic const uint8_t const_small_size2bin[STATIC_PAGE_SIZE - 255] = { 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_1(0xffU) /* 0 */ 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (LG_QUANTUM == 4) 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 64-bit system ************************/ 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TINY 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_2(0) /* 2 */ 56e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_2(1) /* 4 */ 57e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_4(2) /* 8 */ 58e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(3) /* 16 */ 59e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_QMIN 3 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# else 61e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(0) /* 16 */ 62e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_QMIN 0 63e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 64e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 1) /* 32 */ 65e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 2) /* 48 */ 66e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 3) /* 64 */ 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 4) /* 80 */ 68e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 5) /* 96 */ 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 6) /* 112 */ 70e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_16(S2B_QMIN + 7) /* 128 */ 71e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_CMIN (S2B_QMIN + 8) 72e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#else 73e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 32-bit system ************************/ 74e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TINY 75e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_2(0) /* 2 */ 76e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_2(1) /* 4 */ 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_4(2) /* 8 */ 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_QMIN 2 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# else 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(0) /* 8 */ 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_QMIN 0 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 83e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 1) /* 16 */ 84e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 2) /* 24 */ 85e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 3) /* 32 */ 86e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 4) /* 40 */ 87e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 5) /* 48 */ 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 6) /* 56 */ 89e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 7) /* 64 */ 90e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 8) /* 72 */ 91e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 9) /* 80 */ 92e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 10) /* 88 */ 93e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 11) /* 96 */ 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 12) /* 104 */ 95e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 13) /* 112 */ 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 14) /* 120 */ 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_8(S2B_QMIN + 15) /* 128 */ 98e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_CMIN (S2B_QMIN + 16) 99e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/****************************************/ 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 0) /* 192 */ 102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 1) /* 256 */ 103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 2) /* 320 */ 104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 3) /* 384 */ 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 4) /* 448 */ 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_64(S2B_CMIN + 5) /* 512 */ 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define S2B_SMIN (S2B_CMIN + 6) 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 0) /* 768 */ 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 1) /* 1024 */ 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 2) /* 1280 */ 111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 3) /* 1536 */ 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 4) /* 1792 */ 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 5) /* 2048 */ 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 6) /* 2304 */ 115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 7) /* 2560 */ 116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 8) /* 2816 */ 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 9) /* 3072 */ 118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 10) /* 3328 */ 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 11) /* 3584 */ 120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 12) /* 3840 */ 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (STATIC_PAGE_SHIFT == 13) 122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 13) /* 4096 */ 123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 14) /* 4352 */ 124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 15) /* 4608 */ 125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 16) /* 4864 */ 126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 17) /* 5120 */ 127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 18) /* 5376 */ 128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 19) /* 5632 */ 129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 20) /* 5888 */ 130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 21) /* 6144 */ 131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 22) /* 6400 */ 132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 23) /* 6656 */ 133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 24) /* 6912 */ 134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 25) /* 7168 */ 135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 26) /* 7424 */ 136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 27) /* 7680 */ 137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans S2B_256(S2B_SMIN + 28) /* 7936 */ 138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 139e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_1 141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_2 142e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_4 143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_8 144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_16 145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_32 146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_64 147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_128 148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_256 149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_QMIN 150e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_CMIN 151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_SMIN 152e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 154e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Function prototypes for non-inline static functions. */ 155e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 156e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, 157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool large, bool zero); 158e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t *arena_chunk_alloc(arena_t *arena); 159e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); 160e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, 161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool zero); 162e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_purge(arena_t *arena); 163e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty); 164e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, 165e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize); 166e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, 167e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); 168e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); 169e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); 170e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic size_t arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size); 171e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *arena_malloc_large(arena_t *arena, size_t size, bool zero); 172e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_is_large(const void *ptr); 173e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, arena_bin_t *bin); 175e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, 176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr, size_t size, size_t oldsize); 177e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, 178e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr, size_t size, size_t oldsize); 179e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_ralloc_large(void *ptr, size_t size, size_t oldsize); 180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 181e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic size_t pow2_ceil(size_t x); 182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 183e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool small_size2bin_init(void); 184e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 185e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void small_size2bin_validate(void); 186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 187e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool small_size2bin_init_hard(void); 188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 191e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 192e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_comp(arena_chunk_t *a, arena_chunk_t *b) 193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 194e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_chunk = (uintptr_t)a; 195e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t b_chunk = (uintptr_t)b; 196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(a != NULL); 198e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(b != NULL); 199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 200e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((a_chunk > b_chunk) - (a_chunk < b_chunk)); 201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 203f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 204f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evansrb_gen(static JEMALLOC_ATTR(unused), arena_chunk_tree_dirty_, 205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_t, arena_chunk_t, link_dirty, arena_chunk_comp) 206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 207e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 208e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 209e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm = (uintptr_t)a; 211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t b_mapelm = (uintptr_t)b; 212e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 213e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(a != NULL); 214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(b != NULL); 215e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 216e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 217e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 219f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 220f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evansrb_gen(static JEMALLOC_ATTR(unused), arena_run_tree_, arena_run_tree_t, 221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t, link, arena_run_comp) 222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 223e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 224e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 226e476f8a161d445211fd6e54fe370275196e66bcbJason Evans int ret; 227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t a_size = a->bits & ~PAGE_MASK; 228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t b_size = b->bits & ~PAGE_MASK; 229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_size > b_size) - (a_size < b_size); 231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == 0) { 232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm, b_mapelm; 233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((a->bits & CHUNK_MAP_KEY) != CHUNK_MAP_KEY) 235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = (uintptr_t)a; 236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Treat keys as though they are lower than anything 239e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * else. 240e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 241e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = 0; 242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans b_mapelm = (uintptr_t)b; 244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 251f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 252f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evansrb_gen(static JEMALLOC_ATTR(unused), arena_avail_tree_, arena_avail_tree_t, 253e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t, link, arena_avail_comp) 254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 255e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void 256e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_rc_incr(arena_run_t *run, arena_bin_t *bin, const void *ptr) 257e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 258e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 260e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pagebeg, pageend, i; 261e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 262e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 263e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 264e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pagebeg = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 265e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageend = ((uintptr_t)ptr + (uintptr_t)(bin->reg_size - 1) - 266e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)chunk) >> PAGE_SHIFT; 267e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = pagebeg; i <= pageend; i++) { 269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t mapbits = chunk->map[i].bits; 270e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapbits & CHUNK_MAP_DIRTY) { 272e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_RC_MASK) == 0); 273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty--; 274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty--; 275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits ^= CHUNK_MAP_DIRTY; 276e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 277e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_RC_MASK) != CHUNK_MAP_RC_MASK); 278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits += CHUNK_MAP_RC_ONE; 279e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[i].bits = mapbits; 280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 283e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void 284e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_rc_decr(arena_run_t *run, arena_bin_t *bin, const void *ptr) 285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 287e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pagebeg, pageend, mapbits, i; 289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool dirtier = false; 290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 293e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pagebeg = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 294e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageend = ((uintptr_t)ptr + (uintptr_t)(bin->reg_size - 1) - 295e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)chunk) >> PAGE_SHIFT; 296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* First page. */ 298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits = chunk->map[pagebeg].bits; 299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits -= CHUNK_MAP_RC_ONE; 300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((mapbits & CHUNK_MAP_RC_MASK) == 0) { 301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans dirtier = true; 302e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_DIRTY) == 0); 303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits |= CHUNK_MAP_DIRTY; 304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty++; 305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty++; 306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pagebeg].bits = mapbits; 308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (pageend - pagebeg >= 1) { 310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Interior pages are completely consumed by the object being 312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * deallocated, which means that the pages can be 313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * unconditionally marked dirty. 314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 315e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = pagebeg + 1; i < pageend; i++) { 316e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits = chunk->map[i].bits; 317e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits -= CHUNK_MAP_RC_ONE; 318e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_RC_MASK) == 0); 319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans dirtier = true; 320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_DIRTY) == 0); 321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits |= CHUNK_MAP_DIRTY; 322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty++; 323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty++; 324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[i].bits = mapbits; 325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Last page. */ 328e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits = chunk->map[pageend].bits; 329e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits -= CHUNK_MAP_RC_ONE; 330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((mapbits & CHUNK_MAP_RC_MASK) == 0) { 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans dirtier = true; 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_DIRTY) == 0); 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits |= CHUNK_MAP_DIRTY; 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty++; 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty++; 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageend].bits = mapbits; 338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (dirtier) { 341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->dirtied == false) { 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_dirty_insert(&arena->chunks_dirty, 343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk); 344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = true; 345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Enforce opt_lg_dirty_mult. */ 348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_dirty_mult >= 0 && (arena->nactive >> 349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans opt_lg_dirty_mult) < arena->ndirty) 350e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_purge(arena); 351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 354e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void * 355e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin) 356e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, mask, bit, regind; 359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 361e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->regs_minelm < bin->regs_mask_nelms); 362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Move the first check outside the loop, so that run->regs_minelm can 365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * be updated unconditionally, without the possibility of updating it 366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * multiple times. 367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 368e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = run->regs_minelm; 369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mask = run->regs_mask[i]; 370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mask != 0) { 371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Usable allocation found. */ 372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bit = ffs((int)mask) - 1; 373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans regind = ((i << (LG_SIZEOF_INT + 3)) + bit); 375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(regind < bin->nregs); 376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)(((uintptr_t)run) + bin->reg0_offset 377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans + (bin->reg_size * regind)); 378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 379e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Clear bit. */ 380e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mask ^= (1U << bit); 381e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_mask[i] = mask; 382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_rc_incr(run, bin, ret); 384e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 387e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i++; i < bin->regs_mask_nelms; i++) { 389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mask = run->regs_mask[i]; 390e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mask != 0) { 391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Usable allocation found. */ 392e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bit = ffs((int)mask) - 1; 393e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 394e476f8a161d445211fd6e54fe370275196e66bcbJason Evans regind = ((i << (LG_SIZEOF_INT + 3)) + bit); 395e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(regind < bin->nregs); 396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)(((uintptr_t)run) + bin->reg0_offset 397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans + (bin->reg_size * regind)); 398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Clear bit. */ 400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mask ^= (1U << bit); 401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_mask[i] = mask; 402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 403e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 404e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Make a note that nothing before this element 405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * contains a free region. 406e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 407e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_minelm = i; /* Low payoff: + (mask == 0); */ 408e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 409e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_rc_incr(run, bin, ret); 410e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 411e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 412e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 413e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 414e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Not reached. */ 415e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(0); 416e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 417e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 418e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4196109fe07a14b7a619365977d9523db9f8b333792Jason Evansstatic inline unsigned 4206109fe07a14b7a619365977d9523db9f8b333792Jason Evansarena_run_regind(arena_run_t *run, arena_bin_t *bin, const void *ptr, 4216109fe07a14b7a619365977d9523db9f8b333792Jason Evans size_t size) 422e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 4236109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned shift, diff, regind; 424e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 425e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 426e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 427e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Avoid doing division with a variable divisor if possible. Using 429e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * actual division here can reduce allocator throughput by over 20%! 430e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 431e476f8a161d445211fd6e54fe370275196e66bcbJason Evans diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset); 432e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 433e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Rescale (factor powers of 2 out of the numerator and denominator). */ 434e476f8a161d445211fd6e54fe370275196e66bcbJason Evans shift = ffs(size) - 1; 435e476f8a161d445211fd6e54fe370275196e66bcbJason Evans diff >>= shift; 436e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size >>= shift; 437e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 438e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size == 1) { 439e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* The divisor was a power of 2. */ 440e476f8a161d445211fd6e54fe370275196e66bcbJason Evans regind = diff; 441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 442e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 443e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * To divide by a number D that is not a power of two we 444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * multiply by (2^21 / D) and then right shift by 21 positions. 445e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * X / D 447e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 448e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * becomes 449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * (X * size_invs[D - 3]) >> SIZE_INV_SHIFT 451e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 452e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * We can omit the first three elements, because we never 453e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * divide by 0, and 1 and 2 are both powers of two, which are 454e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * handled above. 455e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 456e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define SIZE_INV_SHIFT 21 457e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) 458e476f8a161d445211fd6e54fe370275196e66bcbJason Evans static const unsigned size_invs[] = { 459e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(3), 460e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 462e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 466e476f8a161d445211fd6e54fe370275196e66bcbJason Evans SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 467e476f8a161d445211fd6e54fe370275196e66bcbJason Evans }; 468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= ((sizeof(size_invs) / sizeof(unsigned)) + 2)) 470e476f8a161d445211fd6e54fe370275196e66bcbJason Evans regind = (diff * size_invs[size - 3]) >> SIZE_INV_SHIFT; 471e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 472e476f8a161d445211fd6e54fe370275196e66bcbJason Evans regind = diff / size; 473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef SIZE_INV 474e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef SIZE_INV_SHIFT 475e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 476e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(diff == regind * size); 477e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(regind < bin->nregs); 478e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4796109fe07a14b7a619365977d9523db9f8b333792Jason Evans return (regind); 4806109fe07a14b7a619365977d9523db9f8b333792Jason Evans} 4816109fe07a14b7a619365977d9523db9f8b333792Jason Evans 4826109fe07a14b7a619365977d9523db9f8b333792Jason Evansstatic inline void 4836109fe07a14b7a619365977d9523db9f8b333792Jason Evansarena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size) 4846109fe07a14b7a619365977d9523db9f8b333792Jason Evans{ 4856109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned regind, elm, bit; 4866109fe07a14b7a619365977d9523db9f8b333792Jason Evans 4876109fe07a14b7a619365977d9523db9f8b333792Jason Evans regind = arena_run_regind(run, bin, ptr, size); 488e476f8a161d445211fd6e54fe370275196e66bcbJason Evans elm = regind >> (LG_SIZEOF_INT + 3); 489e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (elm < run->regs_minelm) 490e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_minelm = elm; 491e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bit = regind - (elm << (LG_SIZEOF_INT + 3)); 492e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((run->regs_mask[elm] & (1U << bit)) == 0); 493e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_mask[elm] |= (1U << bit); 494e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 495e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_rc_decr(run, bin, ptr); 496e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 497e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 498e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 499e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, 500e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool zero) 501e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 502e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 503e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i; 504e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 506e476f8a161d445211fd6e54fe370275196e66bcbJason Evans old_ndirty = chunk->ndirty; 507e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) 508e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> PAGE_SHIFT); 509e476f8a161d445211fd6e54fe370275196e66bcbJason Evans total_pages = (chunk->map[run_ind].bits & ~PAGE_MASK) >> 510e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT; 511e476f8a161d445211fd6e54fe370275196e66bcbJason Evans need_pages = (size >> PAGE_SHIFT); 512e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages > 0); 513e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages <= total_pages); 514e476f8a161d445211fd6e54fe370275196e66bcbJason Evans rem_pages = total_pages - need_pages; 515e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 516e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_remove(&arena->runs_avail, &chunk->map[run_ind]); 517e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive += need_pages; 518e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 519e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Keep track of trailing unused pages for later use. */ 520e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (rem_pages > 0) { 521e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+need_pages].bits = (rem_pages << 522e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT) | (chunk->map[run_ind+need_pages].bits & 523e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 524e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+total_pages-1].bits = (rem_pages << 525e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT) | (chunk->map[run_ind+total_pages-1].bits & 526e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 527e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_insert(&arena->runs_avail, 528e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_ind+need_pages]); 529e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 530e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 531e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < need_pages; i++) { 532e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Zero if necessary. */ 533e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero) { 534e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((chunk->map[run_ind + i].bits & CHUNK_MAP_ZEROED) 535e476f8a161d445211fd6e54fe370275196e66bcbJason Evans == 0) { 536e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)chunk + ((run_ind 537e476f8a161d445211fd6e54fe370275196e66bcbJason Evans + i) << PAGE_SHIFT)), 0, PAGE_SIZE); 538e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* CHUNK_MAP_ZEROED is cleared below. */ 539e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 540e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 541e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 542e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Update dirty page accounting. */ 543e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) { 544e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty--; 545e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty--; 546e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* CHUNK_MAP_DIRTY is cleared below. */ 547e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 548e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 549e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize the chunk map. */ 550e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (large) { 551e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind + i].bits = CHUNK_MAP_LARGE 552e476f8a161d445211fd6e54fe370275196e66bcbJason Evans | CHUNK_MAP_ALLOCATED; 553e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 554e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind + i].bits = (i << CHUNK_MAP_PG_SHIFT) 555e476f8a161d445211fd6e54fe370275196e66bcbJason Evans | CHUNK_MAP_ALLOCATED; 556e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 557e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 558e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 559e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (large) { 560e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 561e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Set the run size only in the first element for large runs. 562e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * This is primarily a debugging aid, since the lack of size 563e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * info for trailing pages only matters if the application 564e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * tries to operate on an interior pointer. 565e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 566e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits |= size; 567e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 568e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 569e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Initialize the first page's refcount to 1, so that the run 570e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * header is protected from dirty page purging. 571e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 572e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits += CHUNK_MAP_RC_ONE; 573e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 574e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 575e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 576e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t * 577e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_alloc(arena_t *arena) 578e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 579e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 580e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 581e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 582e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare != NULL) { 583e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena->spare; 584e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 585e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 58641631d00618d7262125e501c91d31b4d70e605faJason Evans bool zero; 58741631d00618d7262125e501c91d31b4d70e605faJason Evans size_t zeroed; 58841631d00618d7262125e501c91d31b4d70e605faJason Evans 58941631d00618d7262125e501c91d31b4d70e605faJason Evans zero = false; 59041631d00618d7262125e501c91d31b4d70e605faJason Evans chunk = (arena_chunk_t *)chunk_alloc(chunksize, &zero); 591e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk == NULL) 592e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 593e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 594e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.mapped += chunksize; 595e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 596e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 597e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->arena = arena; 598e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = false; 599e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 600e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 601e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Claim that no pages are in use, since the header is merely 602e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * overhead. 603e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 604e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty = 0; 605e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 606e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 607e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Initialize the map to contain one maximal free untouched run. 60841631d00618d7262125e501c91d31b4d70e605faJason Evans * Mark the pages as zeroed iff chunk_alloc() returned a zeroed 60941631d00618d7262125e501c91d31b4d70e605faJason Evans * chunk. 610e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 61141631d00618d7262125e501c91d31b4d70e605faJason Evans zeroed = zero ? CHUNK_MAP_ZEROED : 0; 612e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < arena_chunk_header_npages; i++) 613e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[i].bits = 0; 61441631d00618d7262125e501c91d31b4d70e605faJason Evans chunk->map[i].bits = arena_maxclass | zeroed; 61541631d00618d7262125e501c91d31b4d70e605faJason Evans for (i++; i < chunk_npages-1; i++) 61641631d00618d7262125e501c91d31b4d70e605faJason Evans chunk->map[i].bits = zeroed; 61741631d00618d7262125e501c91d31b4d70e605faJason Evans chunk->map[chunk_npages-1].bits = arena_maxclass | zeroed; 618e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 619e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 620e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert the run into the runs_avail tree. */ 621e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_insert(&arena->runs_avail, 622e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[arena_chunk_header_npages]); 623e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 624e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (chunk); 625e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 626e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 627e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 628e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) 629e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 630e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 631e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare != NULL) { 632e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare->dirtied) { 633e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_dirty_remove( 634e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->arena->chunks_dirty, arena->spare); 635e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty -= arena->spare->ndirty; 636e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 637e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk_dealloc((void *)arena->spare, chunksize); 638e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 639e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.mapped -= chunksize; 640e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 641e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 642e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 643e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 644e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove run from runs_avail, regardless of whether this chunk 645e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * will be cached, so that the arena does not use it. Dirty page 646e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * flushing only uses the chunks_dirty tree, so leaving this chunk in 647e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * the chunks_* trees is sufficient for that purpose. 648e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 649e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_remove(&arena->runs_avail, 650e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[arena_chunk_header_npages]); 651e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 652e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = chunk; 653e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 654e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 655e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 656e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) 657e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 658e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 659e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 660e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm, key; 661e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 662e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 663e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 664e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 665e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Search the arena's chunks for the lowest best fit. */ 666e476f8a161d445211fd6e54fe370275196e66bcbJason Evans key.bits = size | CHUNK_MAP_KEY; 667e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key); 668e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 669e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 670e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)mapelm - (uintptr_t)run_chunk->map) 671e476f8a161d445211fd6e54fe370275196e66bcbJason Evans / sizeof(arena_chunk_map_t); 672e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 673e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind 674e476f8a161d445211fd6e54fe370275196e66bcbJason Evans << PAGE_SHIFT)); 675e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, run, size, large, zero); 676e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 677e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 678e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 679e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 680e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * No usable runs. Create a new chunk from which to allocate the run. 681e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 682e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena_chunk_alloc(arena); 683e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk == NULL) 684e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 685e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)chunk + (arena_chunk_header_npages << 686e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)); 687e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Update page map. */ 688e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, run, size, large, zero); 689e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 690e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 691e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 692f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans#ifdef JEMALLOC_DEBUG 693f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evansstatic arena_chunk_t * 694f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evanschunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) 695f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans{ 696f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans size_t *ndirty = (size_t *)arg; 697f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans 698f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans assert(chunk->dirtied); 699f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans *ndirty += chunk->ndirty; 700f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans return (NULL); 701f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans} 702f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans#endif 703f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans 704e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 705e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_purge(arena_t *arena) 706e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 707e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 708e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i, npages; 709e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 710e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty = 0; 711e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 712f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans arena_chunk_tree_dirty_iter(&arena->chunks_dirty, NULL, 713f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans chunks_dirty_iter_cb, (void *)&ndirty); 714e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ndirty == arena->ndirty); 715e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 716e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((arena->nactive >> opt_lg_dirty_mult) < arena->ndirty); 717e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 718e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 719e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.npurge++; 720e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 721e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 722e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 723e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Iterate downward through chunks until enough dirty memory has been 724e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * purged. Terminate as soon as possible in order to minimize the 725e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * number of system calls, even if a chunk has only been partially 726e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * purged. 727e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 728e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 729e476f8a161d445211fd6e54fe370275196e66bcbJason Evans while ((arena->nactive >> (opt_lg_dirty_mult + 1)) < arena->ndirty) { 730e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena_chunk_tree_dirty_last(&arena->chunks_dirty); 731e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(chunk != NULL); 732e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 733e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = chunk_npages - 1; chunk->ndirty > 0; i--) { 734e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(i >= arena_chunk_header_npages); 735e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->map[i].bits & CHUNK_MAP_DIRTY) { 736e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[i].bits ^= CHUNK_MAP_DIRTY; 737e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Find adjacent dirty run(s). */ 738e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (npages = 1; i > arena_chunk_header_npages 739e476f8a161d445211fd6e54fe370275196e66bcbJason Evans && (chunk->map[i - 1].bits & 740e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_DIRTY); npages++) { 741e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i--; 742e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[i].bits ^= CHUNK_MAP_DIRTY; 743e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 744e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty -= npages; 745e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty -= npages; 746e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 747e476f8a161d445211fd6e54fe370275196e66bcbJason Evans madvise((void *)((uintptr_t)chunk + (i << 748e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)), (npages << PAGE_SHIFT), 749e476f8a161d445211fd6e54fe370275196e66bcbJason Evans MADV_DONTNEED); 750e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 751e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmadvise++; 752e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.purged += npages; 753e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 754e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((arena->nactive >> (opt_lg_dirty_mult + 1)) 755e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >= arena->ndirty) 756e476f8a161d445211fd6e54fe370275196e66bcbJason Evans break; 757e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 758e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 759e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 760e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->ndirty == 0) { 761e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_dirty_remove(&arena->chunks_dirty, 762e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk); 763e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = false; 764e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 765e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 766e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 767e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 768e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 769e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) 770e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 771e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 772e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size, run_ind, run_pages; 773e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 774e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 775e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) 776e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> PAGE_SHIFT); 777e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run_ind >= arena_chunk_header_npages); 778e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run_ind < chunk_npages); 779e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((chunk->map[run_ind].bits & CHUNK_MAP_LARGE) != 0) 780e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = chunk->map[run_ind].bits & ~PAGE_MASK; 781e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 782e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = run->bin->run_size; 783e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_pages = (size >> PAGE_SHIFT); 784e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive -= run_pages; 785e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 786e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Mark pages as unallocated in the chunk map. */ 787e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (dirty) { 788e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 789e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 790e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < run_pages; i++) { 791e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 792e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * When (dirty == true), *all* pages within the run 793e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * need to have their dirty bits set, because only 794e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * small runs can create a mixture of clean/dirty 795e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * pages, but such runs are passed to this function 796e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * with (dirty == false). 797e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 798e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) 799e476f8a161d445211fd6e54fe370275196e66bcbJason Evans == 0); 800e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty++; 801e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty++; 802e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind + i].bits = CHUNK_MAP_DIRTY; 803e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 804e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 805e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 806e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 807e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < run_pages; i++) { 808e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind + i].bits &= ~(CHUNK_MAP_LARGE | 809e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED); 810e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 811e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 812e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits & 813e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 814e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+run_pages-1].bits = size | 815e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (chunk->map[run_ind+run_pages-1].bits & CHUNK_MAP_FLAGS_MASK); 816e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 817e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce forward. */ 818e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run_ind + run_pages < chunk_npages && 819e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (chunk->map[run_ind+run_pages].bits & CHUNK_MAP_ALLOCATED) == 0) { 820e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t nrun_size = chunk->map[run_ind+run_pages].bits & 821e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ~PAGE_MASK; 822e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 823e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 824e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove successor from runs_avail; the coalesced run is 825e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 826e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 827e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_remove(&arena->runs_avail, 828e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_ind+run_pages]); 829e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 830e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += nrun_size; 831e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_pages = size >> PAGE_SHIFT; 832e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 833e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[run_ind+run_pages-1].bits & ~PAGE_MASK) 834e476f8a161d445211fd6e54fe370275196e66bcbJason Evans == nrun_size); 835e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits & 836e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 837e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+run_pages-1].bits = size | 838e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (chunk->map[run_ind+run_pages-1].bits & 839e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 840e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 841e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 842e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce backward. */ 843e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run_ind > arena_chunk_header_npages && (chunk->map[run_ind-1].bits & 844e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED) == 0) { 845e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t prun_size = chunk->map[run_ind-1].bits & ~PAGE_MASK; 846e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 847e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind -= prun_size >> PAGE_SHIFT; 848e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 849e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 850e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove predecessor from runs_avail; the coalesced run is 851e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 852e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 853e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_remove(&arena->runs_avail, 854e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_ind]); 855e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 856e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += prun_size; 857e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_pages = size >> PAGE_SHIFT; 858e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 859e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[run_ind].bits & ~PAGE_MASK) == prun_size); 860e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits & 861e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 862e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind+run_pages-1].bits = size | 863e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (chunk->map[run_ind+run_pages-1].bits & 864e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 865e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 866e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 867e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert into runs_avail, now that coalescing is complete. */ 868e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_insert(&arena->runs_avail, &chunk->map[run_ind]); 869e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 8704fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 8714fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * Deallocate chunk if it is now completely unused. The bit 8724fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * manipulation checks whether the first run is unallocated and extends 8734fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * to the end of the chunk. 8744fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 875e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((chunk->map[arena_chunk_header_npages].bits & (~PAGE_MASK | 876e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED)) == arena_maxclass) 877e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_dealloc(arena, chunk); 878e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 8794fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 8804fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * It is okay to do dirty page processing even if the chunk was 8814fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * deallocated above, since in that case it is the spare. Waiting 8824fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * until after possible chunk deallocation to do dirty processing 8834fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * allows for an old spare to be fully deallocated, thus decreasing the 8844fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * chances of spuriously crossing the dirty page purging threshold. 8854fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 886e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (dirty) { 887e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->dirtied == false) { 888e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_dirty_insert(&arena->chunks_dirty, 889e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk); 890e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = true; 891e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 892e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 893e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Enforce opt_lg_dirty_mult. */ 894e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_dirty_mult >= 0 && (arena->nactive >> 895e476f8a161d445211fd6e54fe370275196e66bcbJason Evans opt_lg_dirty_mult) < arena->ndirty) 896e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_purge(arena); 897e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 898e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 899e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 900e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 901e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 902e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize) 903e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 904e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; 905e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t head_npages = (oldsize - newsize) >> PAGE_SHIFT; 906e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 907e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 908e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 909e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 910e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 911e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * leading run as separately allocated. 912e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 913e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[pageind].bits & CHUNK_MAP_DIRTY) == 0); 914e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind].bits = (oldsize - newsize) | CHUNK_MAP_LARGE | 915e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 916e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[pageind+head_npages].bits & CHUNK_MAP_DIRTY) == 0); 917e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind+head_npages].bits = newsize | CHUNK_MAP_LARGE | 918e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 919e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 920e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, run, false); 921e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 922e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 923e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 924e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 925e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize, bool dirty) 926e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 927e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; 928e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t npages = newsize >> PAGE_SHIFT; 929e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 930e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 931e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 932e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 933e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 934e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * trailing run as separately allocated. 935e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 936e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[pageind].bits & CHUNK_MAP_DIRTY) == 0); 937e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind].bits = newsize | CHUNK_MAP_LARGE | 938e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 939e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[pageind+npages].bits & CHUNK_MAP_DIRTY) == 0); 940e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind+npages].bits = (oldsize - newsize) | CHUNK_MAP_LARGE 941e476f8a161d445211fd6e54fe370275196e66bcbJason Evans | CHUNK_MAP_ALLOCATED; 942e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 943e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 944e476f8a161d445211fd6e54fe370275196e66bcbJason Evans dirty); 945e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 946e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 947e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 948e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 949e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 950e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm; 951e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 952e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, remainder; 953e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 954e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Look for a usable run. */ 955e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapelm = arena_run_tree_first(&bin->runs); 956e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 957e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 958e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 959e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 960e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* run is guaranteed to have available space. */ 961e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_remove(&bin->runs, mapelm); 962e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 963e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 964e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 965e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sizeof(arena_chunk_map_t)); 966e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 967e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((mapelm->bits & CHUNK_MAP_PG_MASK) >> CHUNK_MAP_PG_SHIFT)) 968e476f8a161d445211fd6e54fe370275196e66bcbJason Evans << PAGE_SHIFT)); 969e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 970e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.reruns++; 971e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 972e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 973e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 974e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* No existing runs have any space available. */ 975e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 976e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Allocate a new run. */ 977e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = arena_run_alloc(arena, bin->run_size, false, false); 978e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run == NULL) 979e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 980e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 981e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize run internals. */ 982e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->bin = bin; 983e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 984e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0; i < bin->regs_mask_nelms - 1; i++) 985e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_mask[i] = UINT_MAX; 986e476f8a161d445211fd6e54fe370275196e66bcbJason Evans remainder = bin->nregs & ((1U << (LG_SIZEOF_INT + 3)) - 1); 987e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (remainder == 0) 988e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_mask[i] = UINT_MAX; 989e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 990e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* The last element has spare bits that need to be unset. */ 991e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_mask[i] = (UINT_MAX >> ((1U << (LG_SIZEOF_INT + 3)) 992e476f8a161d445211fd6e54fe370275196e66bcbJason Evans - remainder)); 993e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 994e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 995e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->regs_minelm = 0; 996e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 997e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->nfree = bin->nregs; 998e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 999e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->magic = ARENA_RUN_MAGIC; 1000e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1001e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1002e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1003e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nruns++; 1004e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.curruns++; 1005e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->stats.curruns > bin->stats.highruns) 1006e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.highruns = bin->stats.curruns; 1007e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1008e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 1009e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1010e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1011e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* bin->runcur must have space available before this function is called. */ 1012e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void * 1013e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_malloc_easy(arena_t *arena, arena_bin_t *bin, arena_run_t *run) 1014e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1015e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1016e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1017e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 1018e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->nfree > 0); 1019e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1020e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_run_reg_alloc(run, bin); 1021e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ret != NULL); 1022e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->nfree--; 1023e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1024e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1025e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1026e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1027e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Re-fill bin->runcur, then call arena_bin_malloc_easy(). */ 1028e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void * 1029e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1030e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1031e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1032e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = arena_bin_nonfull_run_get(arena, bin); 1033e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->runcur == NULL) 1034e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1035e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->runcur->magic == ARENA_RUN_MAGIC); 1036e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->runcur->nfree > 0); 1037e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1038e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (arena_bin_malloc_easy(arena, bin, bin->runcur)); 1039e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1040e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1041e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 1042e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1043d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evansarena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind 1044d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans# ifdef JEMALLOC_PROF 1045d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , uint64_t prof_accumbytes 1046d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans# endif 1047d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ) 1048e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1049e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, nfill; 1050e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1051e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1052e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr; 1053e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1054e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tbin->ncached == 0); 1055e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1056e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 1057e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1058d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 1059d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, prof_accumbytes); 1060d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 1061e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (i = 0, nfill = (tcache_nslots >> 1); i < nfill; i++) { 1062e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 1063e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ptr = arena_bin_malloc_easy(arena, bin, run); 1064e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1065e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ptr = arena_bin_malloc_hard(arena, bin); 1066e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ptr == NULL) { 1067e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (i > 0) { 1068e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1069e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Move valid pointers to the base of 1070e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * tbin->slots. 1071e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1072e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memmove(&tbin->slots[0], 1073e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &tbin->slots[nfill - i], 1074e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i * sizeof(void *)); 1075e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1076e476f8a161d445211fd6e54fe370275196e66bcbJason Evans break; 1077e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1078e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1079e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Fill slots such that the objects lowest in memory come last. 1080e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * This causes tcache to use low objects first. 1081e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1082e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->slots[nfill - 1 - i] = ptr; 1083e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1084e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1085e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nfills++; 1086e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 1087e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->reg_size <= small_maxclass) { 1088e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_small += (i - tbin->ncached) * 1089e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size; 1090e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_small += tbin->tstats.nrequests; 1091e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1092e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_medium += (i - tbin->ncached) * 1093e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size; 1094e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_medium += tbin->tstats.nrequests; 1095e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1096e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->tstats.nrequests = 0; 1097e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1098e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1099e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->ncached = i; 1100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (tbin->ncached > tbin->high_water) 1101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->high_water = tbin->ncached; 1102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1105d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 1106d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evansvoid 1107d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evansarena_prof_accum(arena_t *arena, uint64_t accumbytes) 1108d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans{ 1109d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 111059e9be0f5f8006c4531f647f718683e2ec016b21Jason Evans if (prof_interval != 0) { 111159e9be0f5f8006c4531f647f718683e2ec016b21Jason Evans arena->prof_accumbytes += accumbytes; 111259e9be0f5f8006c4531f647f718683e2ec016b21Jason Evans if (arena->prof_accumbytes >= prof_interval) { 111359e9be0f5f8006c4531f647f718683e2ec016b21Jason Evans prof_idump(); 111459e9be0f5f8006c4531f647f718683e2ec016b21Jason Evans arena->prof_accumbytes -= prof_interval; 111559e9be0f5f8006c4531f647f718683e2ec016b21Jason Evans } 1116d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans } 1117d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans} 1118d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 1119d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 1120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 1121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Calculate bin->run_size such that it meets the following constraints: 1122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 1123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) bin->run_size >= min_run_size 1124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) bin->run_size <= arena_maxclass 1125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) bin->run_size <= RUN_MAX_SMALL 1126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 1127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * *) run header size < PAGE_SIZE 1128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 1129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * bin->nregs, bin->regs_mask_nelms, and bin->reg0_offset are 1130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * also calculated here, since these settings are all interdependent. 1131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1132e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic size_t 1133e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size) 1134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t try_run_size, good_run_size; 11366109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint32_t try_nregs, good_nregs; 11376109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint32_t try_mask_nelms, good_mask_nelms; 11386109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint32_t try_hdr_size, good_hdr_size; 11396109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 11406109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint32_t try_cnt0_offset, good_cnt0_offset; 11416109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 11426109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint32_t try_reg0_offset, good_reg0_offset; 1143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(min_run_size >= PAGE_SIZE); 1145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(min_run_size <= arena_maxclass); 1146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(min_run_size <= RUN_MAX_SMALL); 1147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Calculate known-valid settings before entering the run_size 1150e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * expansion loop, so that the first part of the loop always copies 1151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * valid settings. 1152e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 1153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The do..while loop iteratively reduces the number of regions until 1154e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * the run header and the regions no longer overlap. A closed formula 1155e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * would be quite messy, since there is an interdependency between the 1156e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * header's mask length and the number of regions. 1157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1158e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_run_size = min_run_size; 1159e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size) 1160e476f8a161d445211fd6e54fe370275196e66bcbJason Evans + 1; /* Counter-act try_nregs-- in loop. */ 1161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans do { 1162e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_nregs--; 1163e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_mask_nelms = (try_nregs >> (LG_SIZEOF_INT + 3)) + 1164e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((try_nregs & ((1U << (LG_SIZEOF_INT + 3)) - 1)) ? 1 : 0); 11656109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_hdr_size = sizeof(arena_run_t) + (sizeof(unsigned) * 11666109fe07a14b7a619365977d9523db9f8b333792Jason Evans (try_mask_nelms - 1)); 11676109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 11686109fe07a14b7a619365977d9523db9f8b333792Jason Evans if (opt_prof) { 11696109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Pad to a quantum boundary. */ 11706109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 11716109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_cnt0_offset = try_hdr_size; 11726109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Add space for one (prof_thr_cnt_t *) per region. */ 11736109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_hdr_size += try_nregs * sizeof(prof_thr_cnt_t *); 11746109fe07a14b7a619365977d9523db9f8b333792Jason Evans } else 11756109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_cnt0_offset = 0; 11766109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 1177e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_reg0_offset = try_run_size - (try_nregs * bin->reg_size); 11786109fe07a14b7a619365977d9523db9f8b333792Jason Evans } while (try_hdr_size > try_reg0_offset); 1179e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* run_size expansion loop. */ 1181e476f8a161d445211fd6e54fe370275196e66bcbJason Evans do { 1182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1183e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Copy valid settings before trying more aggressive settings. 1184e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1185e476f8a161d445211fd6e54fe370275196e66bcbJason Evans good_run_size = try_run_size; 1186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans good_nregs = try_nregs; 1187e476f8a161d445211fd6e54fe370275196e66bcbJason Evans good_mask_nelms = try_mask_nelms; 11886109fe07a14b7a619365977d9523db9f8b333792Jason Evans good_hdr_size = try_hdr_size; 11896109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 11906109fe07a14b7a619365977d9523db9f8b333792Jason Evans good_cnt0_offset = try_cnt0_offset; 11916109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 1192e476f8a161d445211fd6e54fe370275196e66bcbJason Evans good_reg0_offset = try_reg0_offset; 1193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1194e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try more aggressive settings. */ 1195e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_run_size += PAGE_SIZE; 1196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / 1197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */ 1198e476f8a161d445211fd6e54fe370275196e66bcbJason Evans do { 1199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_nregs--; 1200e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_mask_nelms = (try_nregs >> (LG_SIZEOF_INT + 3)) + 1201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((try_nregs & ((1U << (LG_SIZEOF_INT + 3)) - 1)) ? 1202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1 : 0); 12036109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_hdr_size = sizeof(arena_run_t) + (sizeof(unsigned) * 12046109fe07a14b7a619365977d9523db9f8b333792Jason Evans (try_mask_nelms - 1)); 12056109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 12066109fe07a14b7a619365977d9523db9f8b333792Jason Evans if (opt_prof) { 12076109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Pad to a quantum boundary. */ 12086109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 12096109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_cnt0_offset = try_hdr_size; 12106109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* 12116109fe07a14b7a619365977d9523db9f8b333792Jason Evans * Add space for one (prof_thr_cnt_t *) per 12126109fe07a14b7a619365977d9523db9f8b333792Jason Evans * region. 12136109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 12146109fe07a14b7a619365977d9523db9f8b333792Jason Evans try_hdr_size += try_nregs * 12156109fe07a14b7a619365977d9523db9f8b333792Jason Evans sizeof(prof_thr_cnt_t *); 12166109fe07a14b7a619365977d9523db9f8b333792Jason Evans } 12176109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 1218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans try_reg0_offset = try_run_size - (try_nregs * 1219e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size); 12206109fe07a14b7a619365977d9523db9f8b333792Jason Evans } while (try_hdr_size > try_reg0_offset); 1221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } while (try_run_size <= arena_maxclass && try_run_size <= RUN_MAX_SMALL 1222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX 1223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size 12246109fe07a14b7a619365977d9523db9f8b333792Jason Evans && try_hdr_size < PAGE_SIZE); 1225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 12266109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(good_hdr_size <= good_reg0_offset); 1227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((good_mask_nelms << (LG_SIZEOF_INT + 3)) >= good_nregs); 1228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Copy final settings. */ 1230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->run_size = good_run_size; 1231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->nregs = good_nregs; 1232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->regs_mask_nelms = good_mask_nelms; 12336109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 12346109fe07a14b7a619365977d9523db9f8b333792Jason Evans bin->cnt0_offset = good_cnt0_offset; 12356109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 1236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg0_offset = good_reg0_offset; 1237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (good_run_size); 1239e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1240e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1241e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1242e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_small(arena_t *arena, size_t size, bool zero) 1243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 1248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = small_size2bin[size]; 1250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(binind < mbin0); 1251e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 1252e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = bin->reg_size; 1253e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1255e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 1256e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_easy(arena, bin, run); 1257e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1258e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_hard(arena, bin); 1259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1260e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1261e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1262e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1263e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1264e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1265e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1266e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 1267e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (isthreaded == false) { 1268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 1269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nrequests++; 1270e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_small++; 1271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 1272e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 1274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_small += size; 1275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1276d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 1277d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans if (isthreaded == false) 1278d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, size); 1279d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 1280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 1283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1287e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 1290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1293e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1294e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1295e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1296e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_medium(arena_t *arena, size_t size, bool zero) 1297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 1302e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = MEDIUM_CEILING(size); 1304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = mbin0 + ((size - medium_min) >> lg_mspace); 1305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(binind < nbins); 1306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 1307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->reg_size == size); 1308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 1311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_easy(arena, bin, run); 1312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_hard(arena, bin); 1314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1315e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1316e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1317e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1318e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 1322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (isthreaded == false) { 1323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 1324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.nrequests++; 1325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_medium++; 1326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 1327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1328e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 1329e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_medium += size; 1330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1331d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 1332d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans if (isthreaded == false) 1333d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, size); 1334d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 1335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 1338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 1345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1350e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void * 1351e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_large(arena_t *arena, size_t size, bool zero) 1352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1355e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1356e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = PAGE_CEILING(size); 1357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)arena_run_alloc(arena, size, true, zero); 1359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1361e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_large++; 1365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large += size; 1366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1368e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1374d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 1375d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, size); 1376d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 1377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1379e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 1380e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1381e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1384e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1387e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1390e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1391e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1392e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc(size_t size, bool zero) 1393e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1394e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1395e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size != 0); 1396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(QUANTUM_CEILING(size) <= arena_maxclass); 1397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= bin_maxclass) { 1399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 1400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache; 1401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((tcache = tcache_get()) != NULL) 1403e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (tcache_alloc(tcache, size, zero)); 1404e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= small_maxclass) 1406e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (arena_malloc_small(choose_arena(), size, zero)); 1407e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 1408e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (arena_malloc_medium(choose_arena(), size, 1409e476f8a161d445211fd6e54fe370275196e66bcbJason Evans zero)); 1410e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1411e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 1412e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (arena_malloc_large(choose_arena(), size, zero)); 1413e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1414e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1415e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Only handles large allocations that require more than page alignment. */ 1416e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1417e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size) 1418e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1419e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1420e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t offset; 1421e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1422e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1423e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 1424e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((alignment & PAGE_MASK) == 0); 1425e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1426e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1427e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)arena_run_alloc(arena, alloc_size, true, false); 1428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1429e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1430e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1431e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1432e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1433e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); 1434e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1435e476f8a161d445211fd6e54fe370275196e66bcbJason Evans offset = (uintptr_t)ret & (alignment - 1); 1436e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((offset & PAGE_MASK) == 0); 1437e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(offset < alloc_size); 1438e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (offset == 0) 1439e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, ret, alloc_size, size, false); 1440e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 1441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t leadsize, trailsize; 1442e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1443e476f8a161d445211fd6e54fe370275196e66bcbJason Evans leadsize = alignment - offset; 1444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (leadsize > 0) { 1445e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_head(arena, chunk, ret, alloc_size, 1446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans alloc_size - leadsize); 1447e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)((uintptr_t)ret + leadsize); 1448e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans trailsize = alloc_size - leadsize - size; 1451e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (trailsize != 0) { 1452e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Trim trailing space. */ 1453e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(trailsize < alloc_size); 1454e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, ret, size + trailsize, 1455e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size, false); 1456e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1457e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1458e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1459e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1460e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.nmalloc_large++; 1461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large += size; 1462e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1466e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1467e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1470e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1471e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1472e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1474e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0xa5, size); 1475e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero) 1476e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1477e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1478e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1479e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1480e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1481e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1482e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_is_large(const void *ptr) 1483e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1484e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1485e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind, mapbits; 1486e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1487e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1488e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1489e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1490e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1491e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 1492e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits = chunk->map[pageind].bits; 1493e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 1494e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((mapbits & CHUNK_MAP_LARGE) != 0); 1495e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1496e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1497e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Return the size of the allocation pointed to by ptr. */ 1498e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t 1499e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_salloc(const void *ptr) 1500e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1501e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ret; 1502e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1503e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind, mapbits; 1504e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1506e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1507e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1508e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1509e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 1510e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapbits = chunk->map[pageind].bits; 1511e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 1512e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 1513e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 1514e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)((pageind - ((mapbits & CHUNK_MAP_PG_MASK) >> 1515e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT)); 1516e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 1517e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = run->bin->reg_size; 1518e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1519e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = mapbits & ~PAGE_MASK; 1520e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ret != 0); 1521e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1522e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1523e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1524e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1525e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 15266109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 15276109fe07a14b7a619365977d9523db9f8b333792Jason Evansprof_thr_cnt_t * 15286109fe07a14b7a619365977d9523db9f8b333792Jason Evansarena_prof_cnt_get(const void *ptr) 15296109fe07a14b7a619365977d9523db9f8b333792Jason Evans{ 15306109fe07a14b7a619365977d9523db9f8b333792Jason Evans prof_thr_cnt_t *ret; 15316109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena_chunk_t *chunk; 15326109fe07a14b7a619365977d9523db9f8b333792Jason Evans size_t pageind, mapbits; 15336109fe07a14b7a619365977d9523db9f8b333792Jason Evans 15346109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(ptr != NULL); 15356109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 15366109fe07a14b7a619365977d9523db9f8b333792Jason Evans 15376109fe07a14b7a619365977d9523db9f8b333792Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 15386109fe07a14b7a619365977d9523db9f8b333792Jason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 15396109fe07a14b7a619365977d9523db9f8b333792Jason Evans mapbits = chunk->map[pageind].bits; 15406109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 15416109fe07a14b7a619365977d9523db9f8b333792Jason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 15426109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 15436109fe07a14b7a619365977d9523db9f8b333792Jason Evans (uintptr_t)((pageind - ((mapbits & CHUNK_MAP_PG_MASK) >> 15446109fe07a14b7a619365977d9523db9f8b333792Jason Evans CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT)); 15456109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena_bin_t *bin = run->bin; 15466109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned regind; 15476109fe07a14b7a619365977d9523db9f8b333792Jason Evans 15486109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(run->magic == ARENA_RUN_MAGIC); 15496109fe07a14b7a619365977d9523db9f8b333792Jason Evans regind = arena_run_regind(run, bin, ptr, bin->reg_size); 15506109fe07a14b7a619365977d9523db9f8b333792Jason Evans ret = *(prof_thr_cnt_t **)((uintptr_t)run + bin->cnt0_offset + 15516109fe07a14b7a619365977d9523db9f8b333792Jason Evans (regind * sizeof(prof_thr_cnt_t *))); 15526109fe07a14b7a619365977d9523db9f8b333792Jason Evans } else { 15536109fe07a14b7a619365977d9523db9f8b333792Jason Evans ret = chunk->map[pageind].prof_cnt; 15546109fe07a14b7a619365977d9523db9f8b333792Jason Evans } 15556109fe07a14b7a619365977d9523db9f8b333792Jason Evans 15566109fe07a14b7a619365977d9523db9f8b333792Jason Evans return (ret); 15576109fe07a14b7a619365977d9523db9f8b333792Jason Evans} 15586109fe07a14b7a619365977d9523db9f8b333792Jason Evans 15596109fe07a14b7a619365977d9523db9f8b333792Jason Evansvoid 15606109fe07a14b7a619365977d9523db9f8b333792Jason Evansarena_prof_cnt_set(const void *ptr, prof_thr_cnt_t *cnt) 15616109fe07a14b7a619365977d9523db9f8b333792Jason Evans{ 15626109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena_chunk_t *chunk; 15636109fe07a14b7a619365977d9523db9f8b333792Jason Evans size_t pageind, mapbits; 15646109fe07a14b7a619365977d9523db9f8b333792Jason Evans 15656109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(ptr != NULL); 15666109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 15676109fe07a14b7a619365977d9523db9f8b333792Jason Evans 15686109fe07a14b7a619365977d9523db9f8b333792Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 15696109fe07a14b7a619365977d9523db9f8b333792Jason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 15706109fe07a14b7a619365977d9523db9f8b333792Jason Evans mapbits = chunk->map[pageind].bits; 15716109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 15726109fe07a14b7a619365977d9523db9f8b333792Jason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 15736109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 15746109fe07a14b7a619365977d9523db9f8b333792Jason Evans (uintptr_t)((pageind - ((mapbits & CHUNK_MAP_PG_MASK) >> 15756109fe07a14b7a619365977d9523db9f8b333792Jason Evans CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT)); 15766109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena_bin_t *bin = run->bin; 15776109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned regind; 15786109fe07a14b7a619365977d9523db9f8b333792Jason Evans 15796109fe07a14b7a619365977d9523db9f8b333792Jason Evans assert(run->magic == ARENA_RUN_MAGIC); 15806109fe07a14b7a619365977d9523db9f8b333792Jason Evans regind = arena_run_regind(run, bin, ptr, bin->reg_size); 15816109fe07a14b7a619365977d9523db9f8b333792Jason Evans 15826109fe07a14b7a619365977d9523db9f8b333792Jason Evans *((prof_thr_cnt_t **)((uintptr_t)run + bin->cnt0_offset + 15836109fe07a14b7a619365977d9523db9f8b333792Jason Evans (regind * sizeof(prof_thr_cnt_t *)))) = cnt; 15846109fe07a14b7a619365977d9523db9f8b333792Jason Evans } else { 15856109fe07a14b7a619365977d9523db9f8b333792Jason Evans chunk->map[pageind].prof_cnt = cnt; 15866109fe07a14b7a619365977d9523db9f8b333792Jason Evans } 15876109fe07a14b7a619365977d9523db9f8b333792Jason Evans} 15886109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 15896109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1590e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1591e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1592e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin) 1593e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1594e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t run_ind; 1595e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1596e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Deallocate run. */ 1597e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run == bin->runcur) 1598e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1599e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (bin->nregs != 1) { 1600e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t run_pageind = (((uintptr_t)run - 1601e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)chunk)) >> PAGE_SHIFT; 1602e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *run_mapelm = 1603e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_pageind]; 1604e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1605e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * This block's conditional is necessary because if the 1606e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * run only contains one region, then it never gets 1607e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted into the non-full runs tree. 1608e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1609e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_remove(&bin->runs, run_mapelm); 1610e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1611e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1612e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Mark the first page as dirty. The dirty bit for every other page in 1613e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * the run is already properly set, which means we can call 1614e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * arena_run_dalloc(..., false), thus potentially avoiding the needless 1615e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * creation of many dirty pages. 1616e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1617e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT); 1618e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((chunk->map[run_ind].bits & CHUNK_MAP_DIRTY) == 0); 1619e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[run_ind].bits |= CHUNK_MAP_DIRTY; 1620e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty++; 1621e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty++; 1622e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1623e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 1624e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->magic = 0; 1625e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1626e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, run, false); 1627e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1628e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->stats.curruns--; 1629e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1630e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1631e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk->dirtied == false) { 1632e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_dirty_insert(&arena->chunks_dirty, chunk); 1633e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = true; 1634e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1635e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Enforce opt_lg_dirty_mult. */ 1636e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_dirty_mult >= 0 && (arena->nactive >> opt_lg_dirty_mult) < 1637e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty) 1638e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_purge(arena); 1639e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1640e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1641e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1642e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1643e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm) 1644e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1645e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 1646e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1647e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1648e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size; 1649e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1650e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 1651e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1652e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((mapelm->bits & CHUNK_MAP_PG_MASK) >> CHUNK_MAP_PG_SHIFT)) << 1653e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)); 1654e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run->magic == ARENA_RUN_MAGIC); 1655e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = run->bin; 1656e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = bin->reg_size; 1657e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1658e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1659e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1660e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ptr, 0x5a, size); 1661e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1662e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1663e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_reg_dalloc(run, bin, ptr, size); 1664e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run->nfree++; 1665e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1666e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run->nfree == bin->nregs) 1667e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 1668e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (run->nfree == 1 && run != bin->runcur) { 1669e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1670e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Make sure that bin->runcur always refers to the lowest 1671e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * non-full run, if one exists. 1672e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1673e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->runcur == NULL) 1674e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = run; 1675e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if ((uintptr_t)run < (uintptr_t)bin->runcur) { 1676e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Switch runcur. */ 1677e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin->runcur->nfree > 0) { 1678e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *runcur_chunk = 1679e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_ADDR2BASE(bin->runcur); 1680e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t runcur_pageind = 1681e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (((uintptr_t)bin->runcur - 1682e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)runcur_chunk)) >> PAGE_SHIFT; 1683e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *runcur_mapelm = 1684e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &runcur_chunk->map[runcur_pageind]; 1685e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1686e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert runcur. */ 1687e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_insert(&bin->runs, 1688e476f8a161d445211fd6e54fe370275196e66bcbJason Evans runcur_mapelm); 1689e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1690e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = run; 1691e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1692e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t run_pageind = (((uintptr_t)run - 1693e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (uintptr_t)chunk)) >> PAGE_SHIFT; 1694e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *run_mapelm = 1695e476f8a161d445211fd6e54fe370275196e66bcbJason Evans &chunk->map[run_pageind]; 1696e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1697e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(arena_run_tree_search(&bin->runs, run_mapelm) == 1698e476f8a161d445211fd6e54fe370275196e66bcbJason Evans NULL); 1699e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_insert(&bin->runs, run_mapelm); 1700e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1701e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1702e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1703e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1704e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= small_maxclass) { 1705e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_small -= size; 1706e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.ndalloc_small++; 1707e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1708e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_medium -= size; 1709e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.ndalloc_medium++; 1710e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1711e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1712e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1713e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1714e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1715e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1716b34e8684ec025aa780997c11f847c19fb269755bJason Evansarena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, 1717b34e8684ec025aa780997c11f847c19fb269755bJason Evans arena_stats_t *astats, malloc_bin_stats_t *bstats, 1718b34e8684ec025aa780997c11f847c19fb269755bJason Evans malloc_large_stats_t *lstats) 1719b34e8684ec025aa780997c11f847c19fb269755bJason Evans{ 17203c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans unsigned i; 1721b34e8684ec025aa780997c11f847c19fb269755bJason Evans 1722b34e8684ec025aa780997c11f847c19fb269755bJason Evans *nactive += arena->nactive; 1723b34e8684ec025aa780997c11f847c19fb269755bJason Evans *ndirty += arena->ndirty; 1724b34e8684ec025aa780997c11f847c19fb269755bJason Evans 17254201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans astats->mapped += arena->stats.mapped; 1726b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->npurge += arena->stats.npurge; 1727b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmadvise += arena->stats.nmadvise; 1728b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->purged += arena->stats.purged; 1729b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->allocated_small += arena->stats.allocated_small; 1730b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmalloc_small += arena->stats.nmalloc_small; 1731b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->ndalloc_small += arena->stats.ndalloc_small; 1732b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->allocated_medium += arena->stats.allocated_medium; 1733b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmalloc_medium += arena->stats.nmalloc_medium; 1734b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->ndalloc_medium += arena->stats.ndalloc_medium; 1735b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->allocated_large += arena->stats.allocated_large; 1736b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmalloc_large += arena->stats.nmalloc_large; 1737b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->ndalloc_large += arena->stats.ndalloc_large; 1738b34e8684ec025aa780997c11f847c19fb269755bJason Evans 1739b34e8684ec025aa780997c11f847c19fb269755bJason Evans for (i = 0; i < nbins; i++) { 1740b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].nrequests += arena->bins[i].stats.nrequests; 1741b34e8684ec025aa780997c11f847c19fb269755bJason Evans#ifdef JEMALLOC_TCACHE 1742b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].nfills += arena->bins[i].stats.nfills; 1743b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].nflushes += arena->bins[i].stats.nflushes; 1744b34e8684ec025aa780997c11f847c19fb269755bJason Evans#endif 1745b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].nruns += arena->bins[i].stats.nruns; 1746b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].reruns += arena->bins[i].stats.reruns; 1747b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].highruns += arena->bins[i].stats.highruns; 1748b34e8684ec025aa780997c11f847c19fb269755bJason Evans bstats[i].curruns += arena->bins[i].stats.curruns; 1749b34e8684ec025aa780997c11f847c19fb269755bJason Evans } 1750b34e8684ec025aa780997c11f847c19fb269755bJason Evans 17513c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans for (i = 0; i < nlclasses; i++) { 1752b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].nrequests += arena->stats.lstats[i].nrequests; 1753b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].highruns += arena->stats.lstats[i].highruns; 1754b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].curruns += arena->stats.lstats[i].curruns; 1755b34e8684ec025aa780997c11f847c19fb269755bJason Evans } 1756b34e8684ec025aa780997c11f847c19fb269755bJason Evans} 1757e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1758e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1759e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1760e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1761e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 176213668262d17fb5950e2441bc9d56a15db9c93877Jason Evans 1763e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1764e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1765e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1766e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1767990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans# ifndef JEMALLOC_STATS 1768e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1769990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans# endif 1770e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1771e476f8a161d445211fd6e54fe370275196e66bcbJason Evans { 1772e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) 1773e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 1774e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT; 1775e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size = chunk->map[pageind].bits & ~PAGE_MASK; 1776e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1777e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1778e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1779990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans# ifdef JEMALLOC_STATS 1780e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) 1781990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans# endif 1782e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ptr, 0x5a, size); 1783e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1784e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1785990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.ndalloc_large++; 1786e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.allocated_large -= size; 1787e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns--; 1788e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1789e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1790e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1791e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)ptr, true); 1792e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1793e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1794e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1795e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1796e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1797e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size, size_t oldsize) 1798e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1799e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1800e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size < oldsize); 1801e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1802e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1803e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Shrink the run, and make trailing pages available for other 1804e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocations. 1805e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1806e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1807e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 1808e476f8a161d445211fd6e54fe370275196e66bcbJason Evans true); 1809e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1810990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.ndalloc_large++; 1811990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.allocated_large -= oldsize; 1812990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; 1813990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans 1814990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.nmalloc_large++; 1815990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.allocated_large += size; 1816990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1817990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1818990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1819990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1820990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1821990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1822e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1823e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1824e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1825e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1826e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1827e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1828e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1829e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t size, size_t oldsize) 1830e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1831e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 1832e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t npages = oldsize >> PAGE_SHIFT; 1833e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1834e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize == (chunk->map[pageind].bits & ~PAGE_MASK)); 1835e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1836e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to extend the run. */ 1837e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size > oldsize); 1838e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1839e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (pageind + npages < chunk_npages && (chunk->map[pageind+npages].bits 1840e476f8a161d445211fd6e54fe370275196e66bcbJason Evans & CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[pageind+npages].bits & 1841e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ~PAGE_MASK) >= size - oldsize) { 1842e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1843e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The next run is available and sufficiently large. Split the 1844e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * following run, then merge the first part with the existing 1845e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocation. 1846e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1847e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + 1848e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((pageind+npages) << PAGE_SHIFT)), size - oldsize, true, 1849e476f8a161d445211fd6e54fe370275196e66bcbJason Evans false); 1850e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1851e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind].bits = size | CHUNK_MAP_LARGE | 1852e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 1853e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->map[pageind+npages].bits = CHUNK_MAP_LARGE | 1854e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_ALLOCATED; 1855e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1856e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 1857990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.ndalloc_large++; 1858990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.allocated_large -= oldsize; 1859990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; 1860990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans 1861990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.nmalloc_large++; 1862990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.allocated_large += size; 1863990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 1864990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1865990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > 1866990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { 1867990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = 1868990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; 1869990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans } 1870e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1871e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1872e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1873e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1874e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1875e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1876e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 1877e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1878e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1879e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 1880e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Try to resize a large allocation, in order to avoid copying. This will 1881e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * always fail if growing an object, and the following run is already in use. 1882e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1883e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1884e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large(void *ptr, size_t size, size_t oldsize) 1885e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1886e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t psize; 1887e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1888e476f8a161d445211fd6e54fe370275196e66bcbJason Evans psize = PAGE_CEILING(size); 1889e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize == oldsize) { 1890e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Same size class. */ 1891e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1892e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk && size < oldsize) { 1893e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - 1894e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size); 1895e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1896e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1897e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1898e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1899e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1900e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 1901e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1902e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1903e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 1904e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(arena->magic == ARENA_MAGIC); 1905e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1906e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize < oldsize) { 1907e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1908e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Fill before shrinking in order avoid a race. */ 1909e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk) { 1910e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, 1911e476f8a161d445211fd6e54fe370275196e66bcbJason Evans oldsize - size); 1912e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1913e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1914e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_ralloc_large_shrink(arena, chunk, ptr, psize, 1915e476f8a161d445211fd6e54fe370275196e66bcbJason Evans oldsize); 1916e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1917e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1918e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 1919e476f8a161d445211fd6e54fe370275196e66bcbJason Evans psize, oldsize); 1920e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 1921e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == false && opt_zero) { 1922e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + oldsize), 0, 1923e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size - oldsize); 1924e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1925e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1926e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1927e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1928e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1929e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1930e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1931e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1932e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc(void *ptr, size_t size, size_t oldsize) 1933e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1934e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1935e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t copysize; 1936e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1937e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1938e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Try to avoid moving the allocation. 1939e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 1940e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * posix_memalign() can cause allocation of "large" objects that are 1941e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * smaller than bin_maxclass (in order to meet alignment requirements). 1942e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Therefore, do not assume that (oldsize <= bin_maxclass) indicates 1943e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * ptr refers to a bin-allocated object. 1944e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1945e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= arena_maxclass) { 1946e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena_is_large(ptr) == false ) { 1947e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= small_maxclass) { 1948e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= small_maxclass && 1949e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin[size] == 1950e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin[oldsize]) 1951e476f8a161d445211fd6e54fe370275196e66bcbJason Evans goto IN_PLACE; 1952e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else if (size <= bin_maxclass) { 1953e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (small_maxclass < oldsize && oldsize <= 1954e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin_maxclass && MEDIUM_CEILING(size) == 1955e476f8a161d445211fd6e54fe370275196e66bcbJason Evans MEDIUM_CEILING(oldsize)) 1956e476f8a161d445211fd6e54fe370275196e66bcbJason Evans goto IN_PLACE; 1957e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1958e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1959e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 1960e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size > bin_maxclass) { 1961e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena_ralloc_large(ptr, size, oldsize) == 1962e476f8a161d445211fd6e54fe370275196e66bcbJason Evans false) 1963e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 1964e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1965e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1966e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1967e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1968e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to avoid moving the allocation. */ 1969e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (size <= small_maxclass) { 1970e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= small_maxclass && small_size2bin[size] == 1971e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin[oldsize]) 1972e476f8a161d445211fd6e54fe370275196e66bcbJason Evans goto IN_PLACE; 1973e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else if (size <= bin_maxclass) { 1974e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (small_maxclass < oldsize && oldsize <= bin_maxclass && 1975e476f8a161d445211fd6e54fe370275196e66bcbJason Evans MEDIUM_CEILING(size) == MEDIUM_CEILING(oldsize)) 1976e476f8a161d445211fd6e54fe370275196e66bcbJason Evans goto IN_PLACE; 1977e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1978e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (bin_maxclass < oldsize && oldsize <= arena_maxclass) { 1979e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size > bin_maxclass); 1980e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena_ralloc_large(ptr, size, oldsize) == false) 1981e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 1982e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1983e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1984e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1985e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1986e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * If we get here, then size and oldsize are different enough that we 1987e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * need to move the object. In that case, fall back to allocating new 1988e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * space and copying. 1989e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1990e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_malloc(size, false); 1991e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) 1992e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1993e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1994e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Junk/zero-filling were already done by arena_malloc(). */ 1995e476f8a161d445211fd6e54fe370275196e66bcbJason Evans copysize = (size < oldsize) ? size : oldsize; 1996e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(ret, ptr, copysize); 1997e476f8a161d445211fd6e54fe370275196e66bcbJason Evans idalloc(ptr); 1998e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1999e476f8a161d445211fd6e54fe370275196e66bcbJason EvansIN_PLACE: 2000e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_FILL 2001e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_junk && size < oldsize) 2002e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size); 2003e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else if (opt_zero && size > oldsize) 2004e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize); 2005e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2006e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 2007e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2008e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2009e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 2010e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_new(arena_t *arena, unsigned ind) 2011e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2012e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 2013e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 2014e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t prev_run_size; 2015e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 20166109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena->ind = ind; 20176109fe07a14b7a619365977d9523db9f8b333792Jason Evans 2018e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (malloc_mutex_init(&arena->lock)) 2019e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2020e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2021e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 2022e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&arena->stats, 0, sizeof(arena_stats_t)); 2023e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->stats.lstats = (malloc_large_stats_t *)base_alloc( 2024e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sizeof(malloc_large_stats_t) * ((chunksize - PAGE_SIZE) >> 2025e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT)); 2026e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->stats.lstats == NULL) 2027e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2028e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(arena->stats.lstats, 0, sizeof(malloc_large_stats_t) * 2029e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((chunksize - PAGE_SIZE) >> PAGE_SHIFT)); 2030e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 2031e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_new(&arena->tcache_ql); 2032e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 2033e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2034e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2035d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 2036d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena->prof_accumbytes = 0; 2037d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 2038d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 2039e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize chunks. */ 2040e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_tree_dirty_new(&arena->chunks_dirty); 2041e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 2042e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2043e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive = 0; 2044e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty = 0; 2045e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2046e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_avail_tree_new(&arena->runs_avail); 2047e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2048e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize bins. */ 2049e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = PAGE_SIZE; 2050e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2051e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 0; 2052e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 2053e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* (2^n)-spaced tiny bins. */ 2054e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins; i++) { 2055e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 2056e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 2057e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 2058e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2059e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = (1U << (LG_TINY_MIN + i)); 2060e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2061e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 2062e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2063e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 2064e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2065e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2066e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2067e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2068e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2069e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Quantum-spaced bins. */ 2070e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins + nqbins; i++) { 2071e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 2072e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 2073e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 2074e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2075e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = (i - ntbins + 1) << LG_QUANTUM; 2076e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2077e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 2078e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2079e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 2080e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2081e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2082e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2083e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2084e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Cacheline-spaced bins. */ 2085e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins + nqbins + ncbins; i++) { 2086e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 2087e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 2088e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 2089e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2090e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = cspace_min + ((i - (ntbins + nqbins)) << 2091e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_CACHELINE); 2092e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2093e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 2094e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2095e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 2096e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2097e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2098e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2099e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Subpage-spaced bins. */ 2101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < ntbins + nqbins + ncbins + nsbins; i++) { 2102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 2103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 2104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 2105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = sspace_min + ((i - (ntbins + nqbins + ncbins)) 2107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans << LG_SUBPAGE); 2108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 2110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 2112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Medium bins. */ 2117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < nbins; i++) { 2118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 2119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 2120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 2121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->reg_size = medium_min + ((i - (ntbins + nqbins + ncbins + 2123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nsbins)) << lg_mspace); 2124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans prev_run_size = arena_bin_run_size_calc(bin, prev_run_size); 2126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 2128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->magic = ARENA_MAGIC; 2134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2139e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 2140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Compute the smallest power of 2 that is >= x. */ 2141e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic size_t 2142e476f8a161d445211fd6e54fe370275196e66bcbJason Evanspow2_ceil(size_t x) 2143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x--; 2146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 1; 2147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 2; 2148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 4; 2149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 8; 2150e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 16; 2151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (SIZEOF_PTR == 8) 2152e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x |= x >> 32; 2153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2154e476f8a161d445211fd6e54fe370275196e66bcbJason Evans x++; 2155e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (x); 2156e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2158e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2159e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2160e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 2161e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssmall_size2bin_validate(void) 2162e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2163e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i, size, binind; 2164e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2165e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[0] == 0xffU); 2166e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 1; 2167e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TINY 2168e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Tiny. */ 2169e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < (1U << LG_TINY_MIN); i++) { 2170e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(1U << LG_TINY_MIN); 2171e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 2172e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 2173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < qspace_min; i++) { 2175e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(i); 2176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 2177e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 2178e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2179e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 2180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Quantum-spaced. */ 2181e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= qspace_max; i++) { 2182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = QUANTUM_CEILING(i); 2183e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + (size >> LG_QUANTUM) - 1; 2184e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 2185e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Cacheline-spaced. */ 2187e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= cspace_max; i++) { 2188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = CACHELINE_CEILING(i); 2189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ((size - cspace_min) >> 2190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_CACHELINE); 2191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 2192e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Sub-page. */ 2194e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= sspace_max; i++) { 2195e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = SUBPAGE_CEILING(i); 2196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ncbins + ((size - sspace_min) 2197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> LG_SUBPAGE); 2198e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(small_size2bin[i] == binind); 2199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2200e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2203e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 2204e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssmall_size2bin_init(void) 2205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT 2208e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT 2209e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || sizeof(const_small_size2bin) != small_maxclass + 1) 2210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (small_size2bin_init_hard()); 2211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2212e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin = const_small_size2bin; 2213e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(sizeof(const_small_size2bin) == small_maxclass + 1); 2215e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin_validate(); 2216e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2217e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2219e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2220e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 2221e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssmall_size2bin_init_hard(void) 2222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i, size, binind; 2224e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uint8_t *custom_small_size2bin; 2225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2226e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT 2227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT 2228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans || sizeof(const_small_size2bin) != small_maxclass + 1); 2229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin = (uint8_t *)base_alloc(small_maxclass + 1); 2231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (custom_small_size2bin == NULL) 2232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[0] = 0xffU; 2235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans i = 1; 2236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 2237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Tiny. */ 2238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < (1U << LG_TINY_MIN); i++) { 2239e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(1U << LG_TINY_MIN); 2240e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 2241e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i < qspace_min; i++) { 2244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = pow2_ceil(i); 2245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ffs((int)(size >> (LG_TINY_MIN + 1))); 2246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Quantum-spaced. */ 2250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= qspace_max; i++) { 2251e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = QUANTUM_CEILING(i); 2252e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + (size >> LG_QUANTUM) - 1; 2253e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2255e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Cacheline-spaced. */ 2256e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= cspace_max; i++) { 2257e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = CACHELINE_CEILING(i); 2258e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ((size - cspace_min) >> 2259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_CACHELINE); 2260e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2261e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2262e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Sub-page. */ 2263e476f8a161d445211fd6e54fe370275196e66bcbJason Evans for (; i <= sspace_max; i++) { 2264e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = SUBPAGE_CEILING(i); 2265e476f8a161d445211fd6e54fe370275196e66bcbJason Evans binind = ntbins + nqbins + ncbins + ((size - sspace_min) >> 2266e476f8a161d445211fd6e54fe370275196e66bcbJason Evans LG_SUBPAGE); 2267e476f8a161d445211fd6e54fe370275196e66bcbJason Evans custom_small_size2bin[i] = binind; 2268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2270e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin = custom_small_size2bin; 2271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 2272e476f8a161d445211fd6e54fe370275196e66bcbJason Evans small_size2bin_validate(); 2273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2276e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2277e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 2278a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evansarena_boot(void) 2279e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2280a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans size_t header_size; 2281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Set variables according to the value of opt_lg_[qc]space_max. */ 2283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans qspace_max = (1U << opt_lg_qspace_max); 2284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans cspace_min = CACHELINE_CEILING(qspace_max); 2285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (cspace_min == qspace_max) 2286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans cspace_min += CACHELINE; 2287e476f8a161d445211fd6e54fe370275196e66bcbJason Evans cspace_max = (1U << opt_lg_cspace_max); 2288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sspace_min = SUBPAGE_CEILING(cspace_max); 2289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (sspace_min == cspace_max) 2290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sspace_min += SUBPAGE; 2291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(sspace_min < PAGE_SIZE); 2292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans sspace_max = PAGE_SIZE - SUBPAGE; 2293e476f8a161d445211fd6e54fe370275196e66bcbJason Evans medium_max = (1U << opt_lg_medium_max); 2294e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2295e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 2296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(LG_QUANTUM >= LG_TINY_MIN); 2297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 2298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ntbins <= LG_QUANTUM); 2299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nqbins = qspace_max >> LG_QUANTUM; 2300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ncbins = ((cspace_max - cspace_min) >> LG_CACHELINE) + 1; 2301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nsbins = ((sspace_max - sspace_min) >> LG_SUBPAGE) + 1; 2302e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Compute medium size class spacing and the number of medium size 2305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * classes. Limit spacing to no more than pagesize, but if possible 2306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * use the smallest spacing that does not exceed NMBINS_MAX medium size 2307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * classes. 2308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans lg_mspace = LG_SUBPAGE; 2310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nmbins = ((medium_max - medium_min) >> lg_mspace) + 1; 2311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans while (lg_mspace < PAGE_SHIFT && nmbins > NMBINS_MAX) { 2312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans lg_mspace = lg_mspace + 1; 2313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nmbins = ((medium_max - medium_min) >> lg_mspace) + 1; 2314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2315e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mspace_mask = (1U << lg_mspace) - 1U; 2316e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2317e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mbin0 = ntbins + nqbins + ncbins + nsbins; 2318e476f8a161d445211fd6e54fe370275196e66bcbJason Evans nbins = mbin0 + nmbins; 2319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The small_size2bin lookup table uses uint8_t to encode each bin 2321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * index, so we cannot support more than 256 small size classes. This 2322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * limit is difficult to exceed (not even possible with 16B quantum and 2323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 4KiB pages), and such configurations are impractical, but 2324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * nonetheless we need to protect against this case in order to avoid 2325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * undefined behavior. 2326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mbin0 > 256) { 2328e476f8a161d445211fd6e54fe370275196e66bcbJason Evans char line_buf[UMAX2S_BUFSIZE]; 2329e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_write4("<jemalloc>: Too many small size classes (", 2330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans umax2s(mbin0, 10, line_buf), " > max 256)\n", ""); 2331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans abort(); 2332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (small_size2bin_init()) 2335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Compute the header size such that it is large enough to contain the 2339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * page map. 2340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans header_size = sizeof(arena_chunk_t) + 2342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (sizeof(arena_chunk_map_t) * (chunk_npages - 1)); 2343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_header_npages = (header_size >> PAGE_SHIFT) + 2344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ((header_size & PAGE_MASK) != 0); 2345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_maxclass = chunksize - (arena_chunk_header_npages << PAGE_SHIFT); 2346a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans 2347a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans return (false); 2348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2349