arena.c revision e3d13060c8a04f08764b16b003169eb205fa09eb
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_ARENA_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evansssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 8b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansarena_bin_info_t arena_bin_info[NBINS]; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 10da99e31105eb709ef4ec8a120b115c32a6b9723aMike HommeyJEMALLOC_ALIGNED(CACHELINE) 11b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansconst uint8_t small_size2bin[] = { 1241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#define S2B_8(i) i, 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_16(i) S2B_8(i) S2B_8(i) 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_32(i) S2B_16(i) S2B_16(i) 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_64(i) S2B_32(i) S2B_32(i) 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_128(i) S2B_64(i) S2B_64(i) 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_256(i) S2B_128(i) S2B_128(i) 18b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_512(i) S2B_256(i) S2B_256(i) 19b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_1024(i) S2B_512(i) S2B_512(i) 20b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_2048(i) S2B_1024(i) S2B_1024(i) 21b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_4096(i) S2B_2048(i) S2B_2048(i) 22b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_8192(i) S2B_4096(i) S2B_4096(i) 23b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define SIZE_CLASS(bin, delta, size) \ 24b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans S2B_##delta(bin) 25b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans SIZE_CLASSES 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_8 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_16 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_32 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_64 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_128 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_256 32b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_512 33b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_1024 34b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_2048 35b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_4096 36b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_8192 37b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef SIZE_CLASS 38b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans}; 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 41e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Function prototypes for non-inline static functions. */ 42e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 43e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic void arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, 44e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t pageind, size_t npages, bool maybe_adjac_pred, 45e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans bool maybe_adjac_succ); 46e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic void arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, 47e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t pageind, size_t npages, bool maybe_adjac_pred, 48e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans bool maybe_adjac_succ); 49e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, 50203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans bool large, size_t binind, bool zero); 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t *arena_chunk_alloc(arena_t *arena); 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); 535b0c99649fa71674daadf4dd53b1ab05428483fbJason Evansstatic arena_run_t *arena_run_alloc_helper(arena_t *arena, size_t size, 545b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans bool large, size_t binind, bool zero); 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, 56203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind, bool zero); 57e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic arena_chunk_t *chunks_dirty_iter_cb(arena_chunk_tree_t *tree, 58e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_t *chunk, void *arg); 596005f0710cf07d60659d91b20b7ff5592d310027Jason Evansstatic void arena_purge(arena_t *arena, bool all); 60e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, 61e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans bool cleaned); 62e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, 63e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize); 64e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, 65e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); 66e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t *arena_bin_runs_first(arena_bin_t *bin); 67e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run); 68e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run); 69e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t *arena_bin_nonfull_run_tryget(arena_bin_t *bin); 70e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); 71e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); 72088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansstatic void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 73088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_bin_t *bin); 74e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 75e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, arena_bin_t *bin); 76940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansstatic void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 77940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_t *run, arena_bin_t *bin); 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, 798e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ptr, size_t oldsize, size_t size); 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, 818e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); 828e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansstatic bool arena_ralloc_large(void *ptr, size_t oldsize, size_t size, 838e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t extra, bool zero); 8449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstatic size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, 8549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t min_run_size); 86b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansstatic void bin_info_init(void); 87e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 89e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 90e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 91e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 92e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 93e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm = (uintptr_t)a; 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t b_mapelm = (uintptr_t)b; 95e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(a != NULL); 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(b != NULL); 98e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 99e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 102f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 1037372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansrb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, 1047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans u.rb_link, arena_run_comp) 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans int ret; 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t a_size = a->bits & ~PAGE_MASK; 111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t b_size = b->bits & ~PAGE_MASK; 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_size > b_size) - (a_size < b_size); 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == 0) { 115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm, b_mapelm; 116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((a->bits & CHUNK_MAP_KEY) != CHUNK_MAP_KEY) 118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = (uintptr_t)a; 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Treat keys as though they are lower than anything 122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * else. 123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = 0; 125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans b_mapelm = (uintptr_t)b; 127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 134f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 1357372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansrb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, 1367372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans u.rb_link, arena_avail_comp) 137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 138e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline int 139e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_chunk_dirty_comp(arena_chunk_t *a, arena_chunk_t *b) 140e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 141e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t a_val, b_val; 142e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 143e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(a != NULL); 144e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(b != NULL); 145e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 146e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 147e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Order such that chunks with higher fragmentation are "less than" 148e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * those with lower fragmentation. Fragmentation is measured as: 149e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 150e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * mean current avail run size 151e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * -------------------------------- 152e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * mean defragmented avail run size 153e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 154e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * navail 155e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * ----------- 156e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * nruns_avail nruns_avail-nruns_adjac 157e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * = ========================= = ----------------------- 158e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * navail nruns_avail 159e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * ----------------------- 160e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * nruns_avail-nruns_adjac 161e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 162e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * The following code multiplies away the denominator prior to 163e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * comparison, in order to avoid division. 164e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 165e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 166e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans a_val = (a->nruns_avail - a->nruns_adjac) * b->nruns_avail; 167e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans b_val = (b->nruns_avail - b->nruns_adjac) * a->nruns_avail; 168e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (a_val < b_val) 169e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (1); 170e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (a_val > b_val) 171e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (-1); 172e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Break ties by chunk address. */ 173e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans { 174e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans uintptr_t a_chunk = (uintptr_t)a; 175e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans uintptr_t b_chunk = (uintptr_t)b; 176e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 177e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return ((a_chunk > b_chunk) - (a_chunk < b_chunk)); 178e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 179e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 180e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 181e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans/* Generate red-black tree functions. */ 182e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansrb_gen(static UNUSED, arena_chunk_dirty_, arena_chunk_tree_t, arena_chunk_t, 183e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans dirty_link, arena_chunk_dirty_comp) 184e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 185e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline bool 186e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind) 187e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 188e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans bool ret; 189e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 190e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (pageind-1 < map_bias) 191e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans ret = false; 192e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans else { 193e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0); 194e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(ret == false || arena_mapbits_dirty_get(chunk, 195e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans pageind-1) != arena_mapbits_dirty_get(chunk, pageind)); 196e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 197e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (ret); 198e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 199e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 200e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline bool 201e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages) 202e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 203e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans bool ret; 204e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 205e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (pageind+npages == chunk_npages) 206e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans ret = false; 207e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans else { 208e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(pageind+npages < chunk_npages); 209e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0); 210e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(ret == false || arena_mapbits_dirty_get(chunk, pageind) 211e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans != arena_mapbits_dirty_get(chunk, pageind+npages)); 212e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 213e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (ret); 214e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 215e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 216e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline bool 217e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages) 218e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 219e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 220e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (arena_avail_adjac_pred(chunk, pageind) || 221e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_adjac_succ(chunk, pageind, npages)); 222e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 223e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 224e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic void 225e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 226e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 227e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 228e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 229e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 230e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans LG_PAGE)); 231e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 232e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 233e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 234e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * removed and reinserted even if the run to be inserted is clean. 235e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 236e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->ndirty != 0) 237e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 238e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 239e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 240e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac++; 241e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 242e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac++; 243e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_avail++; 244e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->nruns_avail > chunk->nruns_adjac); 245e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 246e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 247e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->ndirty += npages; 248e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->ndirty += npages; 249e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 250e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->ndirty != 0) 251e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 252e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 253e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, 254e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans pageind)); 255e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 256e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 257e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic void 258e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 259e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 260e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 261e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 262e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 263e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans LG_PAGE)); 264e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 265e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 266e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 267e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * removed and reinserted even if the run to be removed is clean. 268e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 269e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->ndirty != 0) 270e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 271e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 272e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 273e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac--; 274e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 275e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac--; 276e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_avail--; 277e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->nruns_avail > chunk->nruns_adjac || (chunk->nruns_avail 278e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans == 0 && chunk->nruns_adjac == 0)); 279e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 280e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 281e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->ndirty -= npages; 282e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->ndirty -= npages; 283e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 284e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->ndirty != 0) 285e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 286e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 287e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, 288e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans pageind)); 289e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 290e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 291e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void * 29249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 293e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 294e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 29584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned regind; 29684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 29784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2991e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(run->nfree > 0); 30084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); 301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 30284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); 30384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + 304122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (uintptr_t)(bin_info->reg_interval * regind)); 3051e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree--; 30684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans if (regind == run->nextind) 30784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans run->nextind++; 30884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(regind < run->nextind); 3091e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans return (ret); 3106109fe07a14b7a619365977d9523db9f8b333792Jason Evans} 3116109fe07a14b7a619365977d9523db9f8b333792Jason Evans 3126109fe07a14b7a619365977d9523db9f8b333792Jason Evansstatic inline void 3131e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evansarena_run_reg_dalloc(arena_run_t *run, void *ptr) 3146109fe07a14b7a619365977d9523db9f8b333792Jason Evans{ 31549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 316203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 317203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits = arena_mapbits_get(chunk, pageind); 31880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t binind = arena_ptr_small_binind_get(ptr, mapbits); 31949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 32084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned regind = arena_run_regind(run, bin_info, ptr); 32184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 32284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 32384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 32449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(run->nfree < bin_info->nregs); 3251e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Freeing an interior pointer can cause assertion failure. */ 3261e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 327122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (uintptr_t)bin_info->reg0_offset)) % 328122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (uintptr_t)bin_info->reg_interval == 0); 32921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans assert((uintptr_t)ptr >= (uintptr_t)run + 33049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset); 33184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Freeing an unallocated pointer can cause assertion failure. */ 33284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 33484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_unset(bitmap, &bin_info->bitmap_info, regind); 3351e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree++; 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 33821fb95bba6ea922e0523f269c0d9a32640047a29Jason Evansstatic inline void 33921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evansarena_chunk_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 34021fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans{ 341d4bab21756279db540866998099522dbd39c05f7Jason Evans size_t i; 342ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); 343d4bab21756279db540866998099522dbd39c05f7Jason Evans 344ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans for (i = 0; i < PAGE / sizeof(size_t); i++) 34521fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans assert(p[i] == 0); 34621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans} 34721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans 348e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 349e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, 350203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind, bool zero) 351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 353bdcadf41e961a3c6bbb37d8d24e4b68a27f2b952Jason Evans size_t run_ind, total_pages, need_pages, rem_pages, i; 35419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t flag_dirty; 355e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 356203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((large && binind == BININD_INVALID) || (large == false && binind 357203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans != BININD_INVALID)); 358203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 360ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 361203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 362203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> 363ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE; 364203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == 365203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans flag_dirty); 366ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans need_pages = (size >> LG_PAGE); 367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages > 0); 368e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages <= total_pages); 369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans rem_pages = total_pages - need_pages; 370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 371e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); 3727372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 3737372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans /* 3747372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * Update stats_cactive if nactive is crossing a chunk 3757372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * multiple. 3767372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans */ 3777372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t cactive_diff = CHUNK_CEILING((arena->nactive + 378ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans need_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive << 379ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE); 3807372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (cactive_diff != 0) 3817372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans stats_cactive_add(cactive_diff); 3827372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive += need_pages; 384e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Keep track of trailing unused pages for later use. */ 386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (rem_pages > 0) { 38719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (flag_dirty != 0) { 388203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 389203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (rem_pages << LG_PAGE), CHUNK_MAP_DIRTY); 390203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_set(chunk, 391203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+total_pages-1, (rem_pages << LG_PAGE), 392203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans CHUNK_MAP_DIRTY); 39319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 394203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 395203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (rem_pages << LG_PAGE), 396203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unzeroed_get(chunk, 397203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+need_pages)); 398203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_set(chunk, 399203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+total_pages-1, (rem_pages << LG_PAGE), 400203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unzeroed_get(chunk, 401203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+total_pages-1)); 40219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 403e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages, 404e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false, true); 40519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 40619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 40719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 40819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Update the page map separately for large vs. small runs, since it is 40919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * possible to avoid iteration for large mallocs. 41019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 41119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (large) { 412e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero) { 41319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (flag_dirty == 0) { 41419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 41519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * The run is clean, so some pages may be 41619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * zeroed (i.e. never before touched). 41719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 41819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans for (i = 0; i < need_pages; i++) { 419203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (arena_mapbits_unzeroed_get(chunk, 420203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+i) != 0) { 421122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans VALGRIND_MAKE_MEM_UNDEFINED( 422122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (void *)((uintptr_t) 423122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans chunk + ((run_ind+i) << 424122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans LG_PAGE)), PAGE); 42519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans memset((void *)((uintptr_t) 42621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans chunk + ((run_ind+i) << 427ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)), 0, PAGE); 4287372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } else if (config_debug) { 429122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans VALGRIND_MAKE_MEM_DEFINED( 430122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (void *)((uintptr_t) 431122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans chunk + ((run_ind+i) << 432122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans LG_PAGE)), PAGE); 43321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed( 43421fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans chunk, run_ind+i); 435940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 43619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 43719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 43819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 43919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * The run is dirty, so all pages must be 44019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * zeroed. 44119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 442122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans VALGRIND_MAKE_MEM_UNDEFINED((void 443122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans *)((uintptr_t)chunk + (run_ind << 444122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans LG_PAGE)), (need_pages << LG_PAGE)); 44519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans memset((void *)((uintptr_t)chunk + (run_ind << 446ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)), 0, (need_pages << LG_PAGE)); 447e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 448e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 45119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Set the last element first, in case the run only contains one 45219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * page (i.e. both statements set the same element). 453e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 454203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, 455203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans flag_dirty); 456203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); 45719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 45819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans assert(zero == false); 45919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 460940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * Propagate the dirty and unzeroed flags to the allocated 461940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * small run, so that arena_dalloc_bin_run() has the ability to 462940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * conditionally trim clean pages. 46319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 464d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); 46521fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans /* 46621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans * The first page will always be dirtied during small run 46721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans * initialization, so a validation failure here would not 46821fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans * actually cause an observable failure. 46921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans */ 4707372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug && flag_dirty == 0 && 471203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unzeroed_get(chunk, run_ind) == 0) 47221fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed(chunk, run_ind); 47319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans for (i = 1; i < need_pages - 1; i++) { 474d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); 4757372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug && flag_dirty == 0 && 476203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) 47721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed(chunk, run_ind+i); 47819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 479203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_small_set(chunk, run_ind+need_pages-1, 480d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans need_pages-1, binind, flag_dirty); 4817372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug && flag_dirty == 0 && 482203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unzeroed_get(chunk, run_ind+need_pages-1) == 483203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 0) { 48421fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed(chunk, 48521fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans run_ind+need_pages-1); 48621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans } 487e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 488e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 489e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 490e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t * 491e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_alloc(arena_t *arena) 492e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 493e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 494e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 495e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 496e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare != NULL) { 497e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena->spare; 498e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 49919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 50030fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 50130fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 502203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 503203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_maxclass); 504203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_unallocated_size_get(chunk, 505203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans chunk_npages-1) == arena_maxclass); 506203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, map_bias) == 507203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_dirty_get(chunk, chunk_npages-1)); 508e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 50941631d00618d7262125e501c91d31b4d70e605faJason Evans bool zero; 5103377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans size_t unzeroed; 51141631d00618d7262125e501c91d31b4d70e605faJason Evans 51241631d00618d7262125e501c91d31b4d70e605faJason Evans zero = false; 513e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&arena->lock); 514eae269036c9f702d9fa9be497a1a2aa1be13a29eMike Hommey chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize, 515609ae595f0358157b19311b0f9f9591db7cee705Jason Evans false, &zero, arena->dss_prec); 516e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&arena->lock); 517e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk == NULL) 518e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 5197372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 5207372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.mapped += chunksize; 521e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 522e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->arena = arena; 523e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 524e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 525e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Claim that no pages are in use, since the header is merely 526e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * overhead. 527e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 528e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty = 0; 529e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 530e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_avail = 0; 531e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac = 0; 532e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 533e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 534e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Initialize the map to contain one maximal free untouched run. 53541631d00618d7262125e501c91d31b4d70e605faJason Evans * Mark the pages as zeroed iff chunk_alloc() returned a zeroed 53641631d00618d7262125e501c91d31b4d70e605faJason Evans * chunk. 537e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 5383377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; 539203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, 540203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans unzeroed); 5413377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans /* 5423377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * There is no need to initialize the internal page map entries 5433377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * unless the chunk is not zeroed. 5443377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans */ 5453377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans if (zero == false) { 5463377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans for (i = map_bias+1; i < chunk_npages-1; i++) 547203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unzeroed_set(chunk, i, unzeroed); 5487372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } else if (config_debug) { 549203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans for (i = map_bias+1; i < chunk_npages-1; i++) { 550203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_unzeroed_get(chunk, i) == 551203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans unzeroed); 552203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 553940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 554203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_set(chunk, chunk_npages-1, 555203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_maxclass, unzeroed); 55619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 557e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 558e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Insert the run into the runs_avail tree. */ 559e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias, 560e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false, false); 561e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 562e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (chunk); 563e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 564e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 565e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 566e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) 567e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 56830fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 56930fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 57030fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 57130fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_maxclass); 57230fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 57330fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_maxclass); 57430fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_dirty_get(chunk, map_bias) == 57530fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_mapbits_dirty_get(chunk, chunk_npages-1)); 57630fe12b866edbc2cf9aaef299063b392ea125aacJason Evans 5778d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans /* 578e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Remove run from the runs_avail tree, so that the arena does not use 579e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * it. 5808d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans */ 581e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias, 582e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false, false); 5838d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans 5848d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (arena->spare != NULL) { 585e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_chunk_t *spare = arena->spare; 586e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 5878d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans arena->spare = chunk; 588e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&arena->lock); 58912a488782681cbd740a5f54e0b7e74ea84858e21Jason Evans chunk_dealloc((void *)spare, chunksize, true); 590e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&arena->lock); 5917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 5927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.mapped -= chunksize; 5938d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans } else 5948d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans arena->spare = chunk; 595e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 596e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 597e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 5985b0c99649fa71674daadf4dd53b1ab05428483fbJason Evansarena_run_alloc_helper(arena_t *arena, size_t size, bool large, size_t binind, 599203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans bool zero) 600e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 601e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 602e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm, key; 603e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 604e476f8a161d445211fd6e54fe370275196e66bcbJason Evans key.bits = size | CHUNK_MAP_KEY; 605e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key); 606e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 607e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 6087393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 6097393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 6107393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 611e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 612e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 613ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 614203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_run_split(arena, run, size, large, binind, zero); 615e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 616e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 617e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 6185b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans return (NULL); 6195b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans} 6205b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans 6215b0c99649fa71674daadf4dd53b1ab05428483fbJason Evansstatic arena_run_t * 6225b0c99649fa71674daadf4dd53b1ab05428483fbJason Evansarena_run_alloc(arena_t *arena, size_t size, bool large, size_t binind, 6235b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans bool zero) 6245b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans{ 6255b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans arena_chunk_t *chunk; 6265b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans arena_run_t *run; 6275b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans 6285b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans assert(size <= arena_maxclass); 6295b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans assert((size & PAGE_MASK) == 0); 6305b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans assert((large && binind == BININD_INVALID) || (large == false && binind 6315b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans != BININD_INVALID)); 6325b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans 6335b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans /* Search the arena's chunks for the lowest best fit. */ 6345b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans run = arena_run_alloc_helper(arena, size, large, binind, zero); 6355b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans if (run != NULL) 6365b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans return (run); 6375b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans 638e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 639e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * No usable runs. Create a new chunk from which to allocate the run. 640e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 641e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena_chunk_alloc(arena); 642e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (chunk != NULL) { 643ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); 644203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_run_split(arena, run, size, large, binind, zero); 645e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 646e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 647e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 648e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 649e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_chunk_alloc() failed, but another thread may have made 650e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * sufficient memory available while this one dropped arena->lock in 651e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_chunk_alloc(), so search one more time. 652e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 6535b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans return (arena_run_alloc_helper(arena, size, large, binind, zero)); 654e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 655e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 65605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansstatic inline void 65705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansarena_maybe_purge(arena_t *arena) 65805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans{ 659e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npurgeable, threshold; 660e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 661e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Don't purge if the option is disabled. */ 662e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (opt_lg_dirty_mult < 0) 663e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return; 664e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Don't purge if all dirty pages are already being purged. */ 665e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena->ndirty <= arena->npurgatory) 666e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return; 667e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgeable = arena->ndirty - arena->npurgatory; 668e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans threshold = (arena->nactive >> opt_lg_dirty_mult); 669e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 670e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Don't purge unless the number of purgeable pages exceeds the 671e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * threshold. 672e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 673e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (npurgeable <= threshold) 674e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return; 67505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 676e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_purge(arena, false); 67705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans} 67805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 679e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline size_t 680e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) 68105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans{ 682e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npurged; 68305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_head(arena_chunk_map_t) mapelms; 68405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_map_t *mapelm; 685e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t pageind, npages; 68605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t nmadvise; 68705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 68805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_new(&mapelms); 68905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 69005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 69105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * If chunk is the spare, temporarily re-allocate it, 1) so that its 692e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * run is reinserted into runs_avail, and 2) so that it cannot be 69305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * completely discarded by another thread while arena->lock is dropped 69405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * by this thread. Note that the arena_run_dalloc() call will 69505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * implicitly deallocate the chunk, so no explicit action is required 69605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * in this function to deallocate the chunk. 69719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 69819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Note that once a chunk contains dirty pages, it cannot again contain 69919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * a single run unless 1) it is a dirty run, or 2) this function purges 70019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * dirty pages and causes the transition to a single clean run. Thus 70119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * (chunk == arena->spare) is possible, but it is not possible for 70219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * this function to be called on the spare unless it contains a dirty 70319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run. 70405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 70519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (chunk == arena->spare) { 706203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); 70730fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); 70830fe12b866edbc2cf9aaef299063b392ea125aacJason Evans 70905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_alloc(arena); 71019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 71105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 712e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (config_stats) 713e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->stats.purged += chunk->ndirty; 714e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 715e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 716e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Operate on all dirty runs if there is no clean/dirty run 717e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * fragmentation. 718e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 719e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->nruns_adjac == 0) 720e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans all = true; 721e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 722e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 723e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Temporarily allocate free dirty runs within chunk. If all is false, 724e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * only operate on dirty runs that are fragments; otherwise operate on 725e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * all dirty runs. 726e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 727e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { 728203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapelm = arena_mapp_get(chunk, pageind); 729203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (arena_mapbits_allocated_get(chunk, pageind) == 0) { 730e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t run_size = 731e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_mapbits_unallocated_size_get(chunk, pageind); 73205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 733e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages = run_size >> LG_PAGE; 734e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind + npages <= chunk_npages); 73530fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_dirty_get(chunk, pageind) == 73630fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_mapbits_dirty_get(chunk, pageind+npages-1)); 737c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 738e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena_mapbits_dirty_get(chunk, pageind) != 0 && 739e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans (all || arena_avail_adjac(chunk, pageind, 740e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages))) { 741e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t) 742e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk + (uintptr_t)(pageind << LG_PAGE)); 743e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 744e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_split(arena, run, run_size, true, 745e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans BININD_INVALID, false); 74619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Append to list for later processing. */ 74719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans ql_elm_new(mapelm, u.ql_link); 74819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans ql_tail_insert(&mapelms, mapelm, u.ql_link); 74905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 75005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } else { 751e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Skip run. */ 752e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena_mapbits_large_get(chunk, pageind) != 0) { 753e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages = arena_mapbits_large_size_get(chunk, 754203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind) >> LG_PAGE; 755e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } else { 7568b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey size_t binind; 7578b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey arena_bin_info_t *bin_info; 75805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t) 759ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans chunk + (uintptr_t)(pageind << LG_PAGE)); 760e69bee01de62b56d3e585042d341743239568043Jason Evans 761203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_small_runind_get(chunk, 762203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind) == 0); 7638b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey binind = arena_bin_index(arena, run->bin); 7648b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey bin_info = &arena_bin_info[binind]; 765e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages = bin_info->run_size >> LG_PAGE; 76605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 76705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 76805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 769e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind == chunk_npages); 770e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->ndirty == 0 || all == false); 771e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->nruns_adjac == 0); 77205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 77305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans malloc_mutex_unlock(&arena->lock); 7747372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 7757372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans nmadvise = 0; 776e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurged = 0; 77705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_foreach(mapelm, &mapelms, u.ql_link) { 7787de92767c20cb72c94609b9c78985526fb84a679Jason Evans bool unzeroed; 7797de92767c20cb72c94609b9c78985526fb84a679Jason Evans size_t flag_unzeroed, i; 78005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 781e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 782e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans sizeof(arena_chunk_map_t)) + map_bias; 783e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages = arena_mapbits_large_size_get(chunk, pageind) >> 784e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans LG_PAGE; 785e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind + npages <= chunk_npages); 7867de92767c20cb72c94609b9c78985526fb84a679Jason Evans unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind << 7877de92767c20cb72c94609b9c78985526fb84a679Jason Evans LG_PAGE)), (npages << LG_PAGE)); 7887de92767c20cb72c94609b9c78985526fb84a679Jason Evans flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; 7897de92767c20cb72c94609b9c78985526fb84a679Jason Evans /* 7907de92767c20cb72c94609b9c78985526fb84a679Jason Evans * Set the unzeroed flag for all pages, now that pages_purge() 7917de92767c20cb72c94609b9c78985526fb84a679Jason Evans * has returned whether the pages were zeroed as a side effect 7927de92767c20cb72c94609b9c78985526fb84a679Jason Evans * of purging. This chunk map modification is safe even though 7937de92767c20cb72c94609b9c78985526fb84a679Jason Evans * the arena mutex isn't currently owned by this thread, 7947de92767c20cb72c94609b9c78985526fb84a679Jason Evans * because the run is marked as allocated, thus protecting it 7957de92767c20cb72c94609b9c78985526fb84a679Jason Evans * from being modified by any other thread. As long as these 7967de92767c20cb72c94609b9c78985526fb84a679Jason Evans * writes don't perturb the first and last elements' 7977de92767c20cb72c94609b9c78985526fb84a679Jason Evans * CHUNK_MAP_ALLOCATED bits, behavior is well defined. 7987de92767c20cb72c94609b9c78985526fb84a679Jason Evans */ 7997de92767c20cb72c94609b9c78985526fb84a679Jason Evans for (i = 0; i < npages; i++) { 8007de92767c20cb72c94609b9c78985526fb84a679Jason Evans arena_mapbits_unzeroed_set(chunk, pageind+i, 8017de92767c20cb72c94609b9c78985526fb84a679Jason Evans flag_unzeroed); 8027de92767c20cb72c94609b9c78985526fb84a679Jason Evans } 803e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurged += npages; 8047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 8057372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans nmadvise++; 80605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 80705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans malloc_mutex_lock(&arena->lock); 8087372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 8097372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmadvise += nmadvise; 81005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 81105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Deallocate runs. */ 81205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans for (mapelm = ql_first(&mapelms); mapelm != NULL; 81305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans mapelm = ql_first(&mapelms)) { 814e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_t *run; 81505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 816e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 817e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans sizeof(arena_chunk_map_t)) + map_bias; 818e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << 819e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans LG_PAGE)); 82005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_remove(&mapelms, mapelm, u.ql_link); 821e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_dalloc(arena, run, false, true); 82205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 823e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 824e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (npurged); 825e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 826e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 827e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic arena_chunk_t * 828e3d13060c8a04f08764b16b003169eb205fa09ebJason Evanschunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) 829e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 830e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t *ndirty = (size_t *)arg; 831e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 832e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->ndirty != 0); 833e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans *ndirty += chunk->ndirty; 834e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (NULL); 83505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans} 83605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 837e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 8386005f0710cf07d60659d91b20b7ff5592d310027Jason Evansarena_purge(arena_t *arena, bool all) 839e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 840e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 84105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t npurgatory; 8427372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) { 8437372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t ndirty = 0; 844e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 845e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_iter(&arena->chunks_dirty, NULL, 846e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunks_dirty_iter_cb, (void *)&ndirty); 8477372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans assert(ndirty == arena->ndirty); 8482caa4715ed4f787f263239ff97dd824636289286Jason Evans } 849af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans assert(arena->ndirty > arena->npurgatory || all); 850af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - 851f9a8edbb50f8cbcaf8ed62b36e8d7191ed223d2aJason Evans arena->npurgatory) || all); 852e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 8537372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 8547372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.npurge++; 855e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 856e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 85705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * Compute the minimum number of pages that this thread should try to 858799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * purge, and add the result to arena->npurgatory. This will keep 859799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * multiple threads from racing to reduce ndirty below the threshold. 860e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 861e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans { 862e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npurgeable = arena->ndirty - arena->npurgatory; 863e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 864e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (all == false) { 865e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t threshold = (arena->nactive >> 866e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans opt_lg_dirty_mult); 867e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 868e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgatory = npurgeable - threshold; 869e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } else 870e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgatory = npurgeable; 871af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans } 872799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory += npurgatory; 873799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans 87405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans while (npurgatory > 0) { 875e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npurgeable, npurged, nunpurged; 876e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 87705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Get next chunk with dirty pages. */ 878e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk = arena_chunk_dirty_first(&arena->chunks_dirty); 87905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans if (chunk == NULL) { 88005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 88105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * This thread was unable to purge as many pages as 88205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * originally intended, due to races with other threads 883799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * that either did some of the purging work, or re-used 884799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * dirty pages. 88505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 886799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory -= npurgatory; 887799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans return; 888e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 889e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgeable = chunk->ndirty; 890e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(npurgeable != 0); 89105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 892e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (npurgeable > npurgatory && chunk->nruns_adjac == 0) { 893799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans /* 894e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * This thread will purge all the dirty pages in chunk, 895e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * so set npurgatory to reflect this thread's intent to 896e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * purge the pages. This tends to reduce the chances 897e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * of the following scenario: 898799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 899799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 1) This thread sets arena->npurgatory such that 900799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * (arena->ndirty - arena->npurgatory) is at the 901799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * threshold. 902799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 2) This thread drops arena->lock. 903799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 3) Another thread causes one or more pages to be 904799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * dirtied, and immediately determines that it must 905799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * purge dirty pages. 906799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 907799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * If this scenario *does* play out, that's okay, 908799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * because all of the purging work being done really 909799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * needs to happen. 910799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans */ 911e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->npurgatory += npurgeable - npurgatory; 912e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgatory = npurgeable; 913799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans } 914799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans 915e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 916e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Keep track of how many pages are purgeable, versus how many 917e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * actually get purged, and adjust counters accordingly. 918e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 919e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->npurgatory -= npurgeable; 920e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgatory -= npurgeable; 921e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurged = arena_chunk_purge(arena, chunk, all); 922e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans nunpurged = npurgeable - npurged; 923e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->npurgatory += nunpurged; 924e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgatory += nunpurged; 925e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 926e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 927e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 9286005f0710cf07d60659d91b20b7ff5592d310027Jason Evansvoid 9296005f0710cf07d60659d91b20b7ff5592d310027Jason Evansarena_purge_all(arena_t *arena) 9306005f0710cf07d60659d91b20b7ff5592d310027Jason Evans{ 9316005f0710cf07d60659d91b20b7ff5592d310027Jason Evans 9326005f0710cf07d60659d91b20b7ff5592d310027Jason Evans malloc_mutex_lock(&arena->lock); 9336005f0710cf07d60659d91b20b7ff5592d310027Jason Evans arena_purge(arena, true); 9346005f0710cf07d60659d91b20b7ff5592d310027Jason Evans malloc_mutex_unlock(&arena->lock); 9356005f0710cf07d60659d91b20b7ff5592d310027Jason Evans} 9366005f0710cf07d60659d91b20b7ff5592d310027Jason Evans 937e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 938e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) 939e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 940e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 94119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t size, run_ind, run_pages, flag_dirty; 942e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 943e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 944ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 9457393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(run_ind >= map_bias); 946e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run_ind < chunk_npages); 947203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (arena_mapbits_large_get(chunk, run_ind) != 0) { 948203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size = arena_mapbits_large_size_get(chunk, run_ind); 949ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans assert(size == PAGE || 950203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_size_get(chunk, 951203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+(size>>LG_PAGE)-1) == 0); 95249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } else { 95349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(arena, run->bin); 95449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 95549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size = bin_info->run_size; 95649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 957ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run_pages = (size >> LG_PAGE); 9587372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 9597372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans /* 9607372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * Update stats_cactive if nactive is crossing a chunk 9617372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * multiple. 9627372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans */ 963ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t cactive_diff = CHUNK_CEILING(arena->nactive << LG_PAGE) - 964ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans CHUNK_CEILING((arena->nactive - run_pages) << LG_PAGE); 9657372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (cactive_diff != 0) 9667372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans stats_cactive_sub(cactive_diff); 9677372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 968e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive -= run_pages; 969e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 97019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 97119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * The run is dirty if the caller claims to have dirtied it, as well as 972e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * if it was already dirty before being allocated and the caller 973e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * doesn't claim to have cleaned it. 97419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 97530fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_dirty_get(chunk, run_ind) == 97630fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 977e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) 97819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans dirty = true; 97919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 98019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 981e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Mark pages as unallocated in the chunk map. */ 982e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (dirty) { 983203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_set(chunk, run_ind, size, 984203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans CHUNK_MAP_DIRTY); 985203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 986203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans CHUNK_MAP_DIRTY); 987e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 988203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_set(chunk, run_ind, size, 989203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unzeroed_get(chunk, run_ind)); 990203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 991203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); 992e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 993e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 994e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce forward. */ 995e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run_ind + run_pages < chunk_npages && 996203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && 997203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) { 998203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, 999203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+run_pages); 1000ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t nrun_pages = nrun_size >> LG_PAGE; 1001e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1002e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1003e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove successor from runs_avail; the coalesced run is 1004e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 1005e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1006203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_unallocated_size_get(chunk, 1007203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+run_pages+nrun_pages-1) == nrun_size); 1008203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, 1009203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+run_pages+nrun_pages-1) == flag_dirty); 1010e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages, 1011e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false, true); 1012e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1013e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += nrun_size; 101412ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_pages += nrun_pages; 1015e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1016203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1017203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1018203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size); 1019e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1020e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1021e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce backward. */ 1022203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, run_ind-1) 1023203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == flag_dirty) { 1024203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t prun_size = arena_mapbits_unallocated_size_get(chunk, 1025203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind-1); 1026ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t prun_pages = prun_size >> LG_PAGE; 1027e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 102812ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_ind -= prun_pages; 1029e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1030e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1031e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove predecessor from runs_avail; the coalesced run is 1032e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 1033e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1034203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1035203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans prun_size); 1036203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); 1037e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_remove(arena, chunk, run_ind, prun_pages, true, 1038e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false); 1039e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1040e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += prun_size; 104112ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_pages += prun_pages; 1042e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1043203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1044203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1045203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size); 1046e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1047e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1048e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert into runs_avail, now that coalescing is complete. */ 1049203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1050203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); 1051203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, run_ind) == 1052203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1053e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_insert(arena, chunk, run_ind, run_pages, true, true); 10548d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans 1055203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans /* Deallocate chunk if it is now completely unused. */ 1056203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (size == arena_maxclass) { 1057203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(run_ind == map_bias); 1058203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(run_pages == (arena_maxclass >> LG_PAGE)); 1059e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_dealloc(arena, chunk); 1060203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 1061e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 10624fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 10638d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * It is okay to do dirty page processing here even if the chunk was 10644fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * deallocated above, since in that case it is the spare. Waiting 10654fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * until after possible chunk deallocation to do dirty processing 10664fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * allows for an old spare to be fully deallocated, thus decreasing the 10674fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * chances of spuriously crossing the dirty page purging threshold. 10684fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 10698d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (dirty) 107005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_maybe_purge(arena); 1071e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1072e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1073e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1074e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1075e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize) 1076e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1077ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1078ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t head_npages = (oldsize - newsize) >> LG_PAGE; 1079203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1080e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1081e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 1082e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1083e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1084e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 1085940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * leading run as separately allocated. Set the last element of each 1086940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * run first, in case of single-page runs. 1087e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1088203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1089d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1090d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty); 1091940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 10927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) { 1093ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans UNUSED size_t tail_npages = newsize >> LG_PAGE; 1094203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, 1095203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+head_npages+tail_npages-1) == 0); 1096203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, 1097203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+head_npages+tail_npages-1) == flag_dirty); 1098940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 1099d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind+head_npages, newsize, 1100d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans flag_dirty); 1101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1102e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_dalloc(arena, run, false, false); 1103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1105e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1106e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize, bool dirty) 1108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1109ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1110ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t head_npages = newsize >> LG_PAGE; 1111203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 1114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 1117940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * trailing run as separately allocated. Set the last element of each 1118940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * run first, in case of single-page runs. 1119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1120203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1121d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1122d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty); 1123203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1124203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (config_debug) { 1125203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; 1126203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, 1127203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+head_npages+tail_npages-1) == 0); 1128203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, 1129203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+head_npages+tail_npages-1) == flag_dirty); 1130203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 1131203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, 1132d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans flag_dirty); 1133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 1135e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans dirty, false); 1136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1138e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 1139e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_first(arena_bin_t *bin) 1140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1141e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); 1142e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 1143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 11458b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey arena_run_t *run; 1146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 11487393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / 11497393f44ff025ca67716fc53b68003fd65122fd97Jason Evans sizeof(arena_chunk_map_t))) + map_bias; 1150203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1151203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_small_runind_get(chunk, pageind)) << 1152ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 1153e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1154e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans } 1155e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1156e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (NULL); 1157e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1158e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1159e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void 1160e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 1161e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1162e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); 1163ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1164203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1165e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1166e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); 1167e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1168e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_tree_insert(&bin->runs, mapelm); 1169e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1170e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1171e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void 1172e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) 1173e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1174e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1175ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1176203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1177e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1178e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); 1179e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1180e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_tree_remove(&bin->runs, mapelm); 1181e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1182e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1183e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t * 1184e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_nonfull_run_tryget(arena_bin_t *bin) 1185e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1186e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_t *run = arena_bin_runs_first(bin); 1187e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) { 1188e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_remove(bin, run); 11897372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 11907372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.reruns++; 1191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1192e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1193e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1194e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1195e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t * 1196e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 1197e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1198e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_t *run; 1199e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans size_t binind; 1200e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_info_t *bin_info; 1201e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1202e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans /* Look for a usable run. */ 1203e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans run = arena_bin_nonfull_run_tryget(bin); 1204e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) 1205e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* No existing runs have any space available. */ 1207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 120849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(arena, bin); 120949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 121049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 1211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Allocate a new run. */ 1212e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&bin->lock); 1213e69bee01de62b56d3e585042d341743239568043Jason Evans /******************************/ 121486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1215203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run = arena_run_alloc(arena, bin_info->run_size, false, binind, false); 1216e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run != NULL) { 121784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 121884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 121984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 1220e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* Initialize run internals. */ 1221f54166e7ef5313c3b5c773cbb0ca2af95f5a15aeJason Evans VALGRIND_MAKE_MEM_UNDEFINED(run, bin_info->reg0_offset - 1222f54166e7ef5313c3b5c773cbb0ca2af95f5a15aeJason Evans bin_info->redzone_size); 1223e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run->bin = bin; 122484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans run->nextind = 0; 122549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans run->nfree = bin_info->nregs; 122684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_init(bitmap, &bin_info->bitmap_info); 1227e69bee01de62b56d3e585042d341743239568043Jason Evans } 1228e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_unlock(&arena->lock); 1229e69bee01de62b56d3e585042d341743239568043Jason Evans /********************************/ 1230e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 1231e69bee01de62b56d3e585042d341743239568043Jason Evans if (run != NULL) { 12327372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 12337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nruns++; 12347372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.curruns++; 12357372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1236e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 1237e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1238e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1239e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 1240e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_run_alloc() failed, but another thread may have made 1241940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * sufficient memory available while this one dropped bin->lock above, 1242e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * so search one more time. 1243e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 1244e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans run = arena_bin_nonfull_run_tryget(bin); 1245e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) 1246e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 1247e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1248e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (NULL); 1249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 12511e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 1252e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void * 1253e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1255e00572b384c81bd2aba57fac32f7077a34388915Jason Evans void *ret; 125649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 125749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1258e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_t *run; 1259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 126049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(arena, bin); 126149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 1262e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->runcur = NULL; 1263e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = arena_bin_nonfull_run_get(arena, bin); 1264e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (bin->runcur != NULL && bin->runcur->nfree > 0) { 1265e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 1266e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * Another thread updated runcur while this one ran without the 1267e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * bin lock in arena_bin_nonfull_run_get(). 1268e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 1269e00572b384c81bd2aba57fac32f7077a34388915Jason Evans assert(bin->runcur->nfree > 0); 127049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_run_reg_alloc(bin->runcur, bin_info); 1271e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run != NULL) { 1272940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_chunk_t *chunk; 1273940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1274940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* 1275940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * arena_run_alloc() may have allocated run, or it may 127684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * have pulled run from the bin's run tree. Therefore 1277940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * it is unsafe to make any assumptions about how run 1278940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * has previously been used, and arena_bin_lower_run() 1279940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * must be called, as if a region were just deallocated 1280940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * from the run. 1281940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans */ 1282940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 128349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (run->nfree == bin_info->nregs) 12848de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 12858de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans else 12868de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_bin_lower_run(arena, chunk, run, bin); 1287e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1288e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (ret); 1289e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1290e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1291e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run == NULL) 1292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1293e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1294e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->runcur = run; 1295e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->runcur->nfree > 0); 1297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 129849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (arena_run_reg_alloc(bin->runcur, bin_info)); 1299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 130186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evansvoid 130286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evansarena_prof_accum(arena_t *arena, uint64_t accumbytes) 130386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans{ 130486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 130578f7352259768f670f8e1f9b000388dd32b62493Jason Evans cassert(config_prof); 130678f7352259768f670f8e1f9b000388dd32b62493Jason Evans 130778f7352259768f670f8e1f9b000388dd32b62493Jason Evans if (config_prof && prof_interval != 0) { 130886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena->prof_accumbytes += accumbytes; 130986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (arena->prof_accumbytes >= prof_interval) { 131086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans prof_idump(); 131186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena->prof_accumbytes -= prof_interval; 131286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 131386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 131486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans} 131586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 1316e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 13177372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansarena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, 13187372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans uint64_t prof_accumbytes) 1319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, nfill; 1321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr; 1324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tbin->ncached == 0); 1326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 13277372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) { 13287372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans malloc_mutex_lock(&arena->lock); 13297372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena_prof_accum(arena, prof_accumbytes); 13307372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans malloc_mutex_unlock(&arena->lock); 13317372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1332e69bee01de62b56d3e585042d341743239568043Jason Evans bin = &arena->bins[binind]; 1333e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 13341dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 13351dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans tbin->lg_fill_div); i < nfill; i++) { 1336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 133749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ptr = arena_bin_malloc_hard(arena, bin); 13403fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (ptr == NULL) 1341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans break; 1342122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (config_fill && opt_junk) { 1343122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_alloc_junk_small(ptr, &arena_bin_info[binind], 1344122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans true); 1345122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 13469c43c13a35220c10d97a886616899189daceb359Jason Evans /* Insert such that low regions get used first. */ 13479c43c13a35220c10d97a886616899189daceb359Jason Evans tbin->avail[nfill - 1 - i] = ptr; 1348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 13497372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 13507372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated += i * arena_bin_info[binind].reg_size; 13517372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nmalloc += i; 13527372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 13537372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nfills++; 13547372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tbin->tstats.nrequests = 0; 13557372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 135686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->ncached = i; 1358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1360122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid 1361122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansarena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) 1362122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans{ 1363122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 1364122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (zero) { 1365122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t redzone_size = bin_info->redzone_size; 1366122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, 1367122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans redzone_size); 1368122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, 1369122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans redzone_size); 1370122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 1371122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, 1372122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval); 1373122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1374122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans} 1375122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 1376122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid 1377122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansarena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) 1378122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans{ 1379122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t size = bin_info->reg_size; 1380122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t redzone_size = bin_info->redzone_size; 1381122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t i; 1382122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bool error = false; 1383122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 1384122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans for (i = 1; i <= redzone_size; i++) { 1385122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans unsigned byte; 1386122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if ((byte = *(uint8_t *)((uintptr_t)ptr - i)) != 0xa5) { 1387122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans error = true; 1388122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans malloc_printf("<jemalloc>: Corrupt redzone " 1389122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans "%zu byte%s before %p (size %zu), byte=%#x\n", i, 1390122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (i == 1) ? "" : "s", ptr, size, byte); 1391122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1392122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1393122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans for (i = 0; i < redzone_size; i++) { 1394122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans unsigned byte; 1395122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if ((byte = *(uint8_t *)((uintptr_t)ptr + size + i)) != 0xa5) { 1396122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans error = true; 1397122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans malloc_printf("<jemalloc>: Corrupt redzone " 1398122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans "%zu byte%s after end of %p (size %zu), byte=%#x\n", 1399122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans i, (i == 1) ? "" : "s", ptr, size, byte); 1400122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1401122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1402122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (opt_abort && error) 1403122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans abort(); 1404122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 1405122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, 1406122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval); 1407122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans} 1408122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 1409e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1410e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_small(arena_t *arena, size_t size, bool zero) 1411e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1412e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1413e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1414e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1415e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 1416e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 141741ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans binind = SMALL_SIZE2BIN(size); 1418b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 1419e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 142049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size = arena_bin_info[binind].reg_size; 1421e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 142286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 1423e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 142449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1425e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1426e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_hard(arena, bin); 1427e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 142986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1430e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1431e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1432e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 14337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 14347372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated += size; 14357372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nmalloc++; 14367372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nrequests++; 14377372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 143886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 14397372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof && isthreaded == false) { 144086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1441d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, size); 144286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 144386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 1444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1445e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 14467372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill) { 1447122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (opt_junk) { 1448122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_alloc_junk_small(ret, 1449122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans &arena_bin_info[binind], false); 1450122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else if (opt_zero) 14517372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0, size); 14527372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1453122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 1454122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (config_fill && opt_junk) { 1455122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_alloc_junk_small(ret, &arena_bin_info[binind], 1456122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans true); 1457122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1458122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1459e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1460122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1462e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1465e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1466e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_large(arena_t *arena, size_t size, bool zero) 1467e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1470e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1471e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = PAGE_CEILING(size); 1472e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1473203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans ret = (void *)arena_run_alloc(arena, size, true, BININD_INVALID, zero); 1474e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1475e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1476e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1477e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 14787372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 14797372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 14807372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 14817372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 1482ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1483ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1484ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1485e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 14867372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) 14877372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena_prof_accum(arena, size); 1488e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1489e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1490e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 14917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill) { 14927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (opt_junk) 14937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0xa5, size); 14947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans else if (opt_zero) 14957372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0, size); 14967372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1497e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1498e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1499e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1500e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1501e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1502e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Only handles large allocations that require more than page alignment. */ 1503e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 15045ff709c264e52651de25b788692c62ff1f6f389cJason Evansarena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) 1505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1506e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 15075ff709c264e52651de25b788692c62ff1f6f389cJason Evans size_t alloc_size, leadsize, trailsize; 15085ff709c264e52651de25b788692c62ff1f6f389cJason Evans arena_run_t *run; 1509e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1510e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1511e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 151293443689a4367cc6fe3de1c9e918adc13d8f9100Jason Evans 151393443689a4367cc6fe3de1c9e918adc13d8f9100Jason Evans alignment = PAGE_CEILING(alignment); 15145ff709c264e52651de25b788692c62ff1f6f389cJason Evans alloc_size = size + alignment - PAGE; 1515e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1516e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1517203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run = arena_run_alloc(arena, alloc_size, true, BININD_INVALID, zero); 15185ff709c264e52651de25b788692c62ff1f6f389cJason Evans if (run == NULL) { 1519e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1520e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1521e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 15225ff709c264e52651de25b788692c62ff1f6f389cJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1523e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 15245ff709c264e52651de25b788692c62ff1f6f389cJason Evans leadsize = ALIGNMENT_CEILING((uintptr_t)run, alignment) - 15255ff709c264e52651de25b788692c62ff1f6f389cJason Evans (uintptr_t)run; 15265ff709c264e52651de25b788692c62ff1f6f389cJason Evans assert(alloc_size >= leadsize + size); 15275ff709c264e52651de25b788692c62ff1f6f389cJason Evans trailsize = alloc_size - leadsize - size; 15285ff709c264e52651de25b788692c62ff1f6f389cJason Evans ret = (void *)((uintptr_t)run + leadsize); 15295ff709c264e52651de25b788692c62ff1f6f389cJason Evans if (leadsize != 0) { 15305ff709c264e52651de25b788692c62ff1f6f389cJason Evans arena_run_trim_head(arena, chunk, run, alloc_size, alloc_size - 15315ff709c264e52651de25b788692c62ff1f6f389cJason Evans leadsize); 15325ff709c264e52651de25b788692c62ff1f6f389cJason Evans } 15335ff709c264e52651de25b788692c62ff1f6f389cJason Evans if (trailsize != 0) { 15345ff709c264e52651de25b788692c62ff1f6f389cJason Evans arena_run_trim_tail(arena, chunk, ret, size + trailsize, size, 15355ff709c264e52651de25b788692c62ff1f6f389cJason Evans false); 1536e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1537e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 15387372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 15397372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 15407372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 15417372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 1542ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1543ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1544ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1545e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1546e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1547e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 15487372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && zero == false) { 15498e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (opt_junk) 15508e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset(ret, 0xa5, size); 15518e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans else if (opt_zero) 15528e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset(ret, 0, size); 15538e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 1554e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1555e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1556e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 15570b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansvoid 15580b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansarena_prof_promoted(const void *ptr, size_t size) 15590b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans{ 15600b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans arena_chunk_t *chunk; 15610b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t pageind, binind; 15620b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 156378f7352259768f670f8e1f9b000388dd32b62493Jason Evans cassert(config_prof); 15640b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(ptr != NULL); 15650b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1566122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(isalloc(ptr, false) == PAGE); 1567122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(isalloc(ptr, true) == PAGE); 1568b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(size <= SMALL_MAXCLASS); 15690b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 15700b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1571ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 157241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans binind = SMALL_SIZE2BIN(size); 1573b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 1574203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_binind_set(chunk, pageind, binind); 15750b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 1576122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(isalloc(ptr, false) == PAGE); 1577122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(isalloc(ptr, true) == size); 15780b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans} 15796109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1580e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1581088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansarena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 1582e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin) 1583e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1584e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 158519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Dissociate run from bin. */ 1586e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run == bin->runcur) 1587e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 158849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans else { 158949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, bin); 159049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 159149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 159249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (bin_info->nregs != 1) { 159349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 159449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * This block's conditional is necessary because if the 159549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * run only contains one region, then it never gets 159649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * inserted into the non-full runs tree. 159749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 1598e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_remove(bin, run); 159949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 1600e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1601088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans} 1602088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans 1603088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansstatic void 1604088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansarena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1605088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_bin_t *bin) 1606088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans{ 160749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 160849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1609088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans size_t npages, run_ind, past; 1610088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans 1611088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans assert(run != bin->runcur); 1612203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_run_tree_search(&bin->runs, 1613203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapp_get(chunk, ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) 1614203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans == NULL); 161586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 161649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(chunk->arena, run->bin); 161749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 161849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 1619e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&bin->lock); 1620e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /******************************/ 1621ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans npages = bin_info->run_size >> LG_PAGE; 1622ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 162384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans past = (size_t)(PAGE_CEILING((uintptr_t)run + 162484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * 1625122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval - bin_info->redzone_size) - 1626122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (uintptr_t)chunk) >> LG_PAGE); 162786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 162819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 162919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 163019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * If the run was originally clean, and some pages were never touched, 163119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * trim the clean pages before deallocating the dirty portion of the 163219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run. 163319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 163430fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_dirty_get(chunk, run_ind) == 163530fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_mapbits_dirty_get(chunk, run_ind+npages-1)); 1636203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind < 1637203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans npages) { 163830fe12b866edbc2cf9aaef299063b392ea125aacJason Evans /* Trim clean pages. Convert to large run beforehand. */ 163930fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(npages > 0); 1640d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, run_ind, bin_info->run_size, 0); 1641d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); 1642ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), 1643ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans ((past - run_ind) << LG_PAGE), false); 1644940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* npages = past - run_ind; */ 16451e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans } 1646e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_dalloc(arena, run, true, false); 164786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 1648e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /****************************/ 1649e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&bin->lock); 16507372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 16517372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.curruns--; 1652e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1653e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1654940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansstatic void 1655940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansarena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1656940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_bin_t *bin) 1657e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1658e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 16598de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* 1660e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * Make sure that if bin->runcur is non-NULL, it refers to the lowest 1661e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * non-full run. It is okay to NULL runcur out rather than proactively 1662e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * keeping it pointing at the lowest non-full run. 16638de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans */ 1664e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if ((uintptr_t)run < (uintptr_t)bin->runcur) { 16658de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* Switch runcur. */ 1666e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (bin->runcur->nfree > 0) 1667e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_insert(bin, bin->runcur); 16688de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans bin->runcur = run; 1669e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (config_stats) 1670e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans bin->stats.reruns++; 1671e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans } else 1672e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_insert(bin, run); 1673940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans} 1674940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1675940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansvoid 1676203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1677940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_chunk_map_t *mapelm) 1678940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans{ 1679940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t pageind; 1680940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_t *run; 1681940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_bin_t *bin; 16828b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey arena_bin_info_t *bin_info; 16838b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey size_t size, binind; 1684940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1685ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1686940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1687203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1688940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans bin = run->bin; 168980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = arena_ptr_small_binind_get(ptr, mapelm->bits); 16908b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey bin_info = &arena_bin_info[binind]; 16917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill || config_stats) 16927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size = bin_info->reg_size; 1693940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 16947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk) 1695122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_dalloc_junk_small(ptr, bin_info); 1696940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1697940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_reg_dalloc(run, ptr); 169849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (run->nfree == bin_info->nregs) { 1699088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_dissociate_bin_run(chunk, run, bin); 17008de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 1701088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans } else if (run->nfree == 1 && run != bin->runcur) 17028de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_bin_lower_run(arena, chunk, run, bin); 1703e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 17057372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated -= size; 17067372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.ndalloc++; 17077372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1708e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1709e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1710e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1711203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1712203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind, arena_chunk_map_t *mapelm) 1713203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 1714203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_run_t *run; 1715203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_bin_t *bin; 1716203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1717203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1718203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1719203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans bin = run->bin; 1720203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans malloc_mutex_lock(&bin->lock); 1721203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_bin_locked(arena, chunk, ptr, mapelm); 1722203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans malloc_mutex_unlock(&bin->lock); 1723203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 1724203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1725203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid 1726203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1727203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind) 1728203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 1729203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_chunk_map_t *mapelm; 1730203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1731203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (config_debug) { 173280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans /* arena_ptr_small_binind_get() does extra sanity checking. */ 173380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 173480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind)) != BININD_INVALID); 1735203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 1736203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapelm = arena_mapp_get(chunk, pageind); 1737203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); 1738203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 1739e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1740e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1741203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1742e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 174313668262d17fb5950e2441bc9d56a15db9c93877Jason Evans 17447372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill || config_stats) { 1745ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1746203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size = arena_mapbits_large_size_get(chunk, pageind); 1747e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17487372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && config_stats && opt_junk) 1749e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ptr, 0x5a, size); 17507372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 17517372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 17527372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= size; 1753ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].ndalloc++; 1754ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns--; 17557372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1756e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1757e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1758e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_dalloc(arena, (arena_run_t *)ptr, true, false); 1759e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1760e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1761203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid 1762203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1763203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 1764203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1765203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans malloc_mutex_lock(&arena->lock); 1766203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_large_locked(arena, chunk, ptr); 1767203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans malloc_mutex_unlock(&arena->lock); 1768203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 1769203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1770e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1771e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 17728e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t oldsize, size_t size) 1773e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1774e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1775e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size < oldsize); 1776e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1777e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1778e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Shrink the run, and make trailing pages available for other 1779e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocations. 1780e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1781e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1782e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 1783e476f8a161d445211fd6e54fe370275196e66bcbJason Evans true); 17847372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 17857372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 17867372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= oldsize; 1787ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 1788ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 17897372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 17907372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 17917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 17927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 1793ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1794ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1795ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1796e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1797e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1798e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1799e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1800e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1801e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 18028e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t oldsize, size_t size, size_t extra, bool zero) 1803e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1804ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1805ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t npages = oldsize >> LG_PAGE; 18068e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t followsize; 1807e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1808203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(oldsize == arena_mapbits_large_size_get(chunk, pageind)); 1809e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1810e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to extend the run. */ 18118e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans assert(size + extra > oldsize); 1812e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 18137393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if (pageind + npages < chunk_npages && 1814203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && 1815203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (followsize = arena_mapbits_unallocated_size_get(chunk, 1816203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+npages)) >= size - oldsize) { 1817e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1818e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The next run is available and sufficiently large. Split the 1819e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * following run, then merge the first part with the existing 1820e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocation. 1821e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1822940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t flag_dirty; 18238e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t splitsize = (oldsize + followsize <= size + extra) 18248e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ? followsize : size + extra - oldsize; 1825e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + 1826203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans ((pageind+npages) << LG_PAGE)), splitsize, true, 1827203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans BININD_INVALID, zero); 1828e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1829088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans size = oldsize + splitsize; 1830ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans npages = size >> LG_PAGE; 1831e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1832940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* 1833940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * Mark the extended run as dirty if either portion of the run 1834940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * was dirty before allocation. This is rather pedantic, 1835940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * because there's not actually any sequence of events that 1836940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * could cause the resulting run to be passed to 1837940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * arena_run_dalloc() with the dirty argument set to false 1838940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * (which is when dirty flag consistency would really matter). 1839940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans */ 1840203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | 1841203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_dirty_get(chunk, pageind+npages-1); 1842203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_set(chunk, pageind, size, flag_dirty); 1843203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty); 1844990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans 18457372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 18467372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 18477372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= oldsize; 1848203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 1849203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 18507372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 18517372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 18527372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 18537372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 1854ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1855203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1856ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1857940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 1858e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1859e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1860e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1861e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1862e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1863e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 1864e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1865e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1866e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 1867e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Try to resize a large allocation, in order to avoid copying. This will 1868e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * always fail if growing an object, and the following run is already in use. 1869e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1870e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 18718e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, 18728e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 1873e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1874e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t psize; 1875e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18768e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize = PAGE_CEILING(size + extra); 1877e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize == oldsize) { 1878e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Same size class. */ 18797372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk && size < oldsize) { 1880e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - 1881e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size); 1882e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1883e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1884e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1885e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1886e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 1887e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1888e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1889e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 1890e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1891e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize < oldsize) { 1892e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Fill before shrinking in order avoid a race. */ 18937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk) { 1894e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, 1895e476f8a161d445211fd6e54fe370275196e66bcbJason Evans oldsize - size); 1896e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 18978e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, 18988e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize); 1899e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1900e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1901e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 19028e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans oldsize, PAGE_CEILING(size), 19038e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize - PAGE_CEILING(size), zero); 19047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && ret == false && zero == false && 19057372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans opt_zero) { 1906e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + oldsize), 0, 1907e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size - oldsize); 1908e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1909e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1910e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1911e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1912e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1913e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1914e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 19158e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, 19168e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 1917e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1918e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19198e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* 19208e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * Avoid moving the allocation if the size class can be left the same. 19218e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans */ 1922e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= arena_maxclass) { 1923b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (oldsize <= SMALL_MAXCLASS) { 192449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size 192549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans == oldsize); 1926b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if ((size + extra <= SMALL_MAXCLASS && 192741ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans SMALL_SIZE2BIN(size + extra) == 192841ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && 19298e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size + extra >= oldsize)) { 19307372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk && size < oldsize) { 19318e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset((void *)((uintptr_t)ptr + size), 19328e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 0x5a, oldsize - size); 19338e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 19348e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (ptr); 19358e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 1936e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1937e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 1938b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (size + extra > SMALL_MAXCLASS) { 19398e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (arena_ralloc_large(ptr, oldsize, size, 19408e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans extra, zero) == false) 1941e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 1942e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1943e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1944e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1945e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19468e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Reallocation would require a move. */ 19478e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 19488e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans} 19498e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 19508e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansvoid * 1951609ae595f0358157b19311b0f9f9591db7cee705Jason Evansarena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, 1952609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, 1953609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bool try_tcache_dalloc) 19548e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans{ 19558e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ret; 19568e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t copysize; 19578e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 19588e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Try to avoid moving the allocation. */ 19598e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = arena_ralloc_no_move(ptr, oldsize, size, extra, zero); 19608e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret != NULL) 19618e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (ret); 19628e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 1963e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 19648e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * size and oldsize are different enough that we need to move the 19658e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * object. In that case, fall back to allocating new space and 19668e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * copying. 1967e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 196838d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (alignment != 0) { 19695ff709c264e52651de25b788692c62ff1f6f389cJason Evans size_t usize = sa2u(size + extra, alignment); 197038d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (usize == 0) 197138d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans return (NULL); 1972609ae595f0358157b19311b0f9f9591db7cee705Jason Evans ret = ipallocx(usize, alignment, zero, try_tcache_alloc, arena); 197338d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans } else 1974609ae595f0358157b19311b0f9f9591db7cee705Jason Evans ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc); 1975e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19768e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == NULL) { 19778e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (extra == 0) 19788e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 19798e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Try again, this time without extra. */ 198038d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (alignment != 0) { 19815ff709c264e52651de25b788692c62ff1f6f389cJason Evans size_t usize = sa2u(size, alignment); 198238d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (usize == 0) 198338d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans return (NULL); 1984609ae595f0358157b19311b0f9f9591db7cee705Jason Evans ret = ipallocx(usize, alignment, zero, try_tcache_alloc, 1985609ae595f0358157b19311b0f9f9591db7cee705Jason Evans arena); 198638d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans } else 1987609ae595f0358157b19311b0f9f9591db7cee705Jason Evans ret = arena_malloc(arena, size, zero, try_tcache_alloc); 19888e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 19898e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == NULL) 19908e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 19918e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 19928e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 19938e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ 19948e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 19958e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* 19968e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * Copy at most size bytes (not size+extra), since the caller has no 19978e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * expectation that the extra bytes will be reliably preserved. 19988e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans */ 1999e476f8a161d445211fd6e54fe370275196e66bcbJason Evans copysize = (size < oldsize) ? size : oldsize; 2000f54166e7ef5313c3b5c773cbb0ca2af95f5a15aeJason Evans VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); 2001e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(ret, ptr, copysize); 2002609ae595f0358157b19311b0f9f9591db7cee705Jason Evans iqallocx(ptr, try_tcache_dalloc); 2003e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 2004e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2005e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2006609ae595f0358157b19311b0f9f9591db7cee705Jason Evansdss_prec_t 2007609ae595f0358157b19311b0f9f9591db7cee705Jason Evansarena_dss_prec_get(arena_t *arena) 2008609ae595f0358157b19311b0f9f9591db7cee705Jason Evans{ 2009609ae595f0358157b19311b0f9f9591db7cee705Jason Evans dss_prec_t ret; 2010609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2011609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&arena->lock); 2012609ae595f0358157b19311b0f9f9591db7cee705Jason Evans ret = arena->dss_prec; 2013609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&arena->lock); 2014609ae595f0358157b19311b0f9f9591db7cee705Jason Evans return (ret); 2015609ae595f0358157b19311b0f9f9591db7cee705Jason Evans} 2016609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2017609ae595f0358157b19311b0f9f9591db7cee705Jason Evansvoid 2018609ae595f0358157b19311b0f9f9591db7cee705Jason Evansarena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) 2019609ae595f0358157b19311b0f9f9591db7cee705Jason Evans{ 2020609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2021609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&arena->lock); 2022609ae595f0358157b19311b0f9f9591db7cee705Jason Evans arena->dss_prec = dss_prec; 2023609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&arena->lock); 2024609ae595f0358157b19311b0f9f9591db7cee705Jason Evans} 2025609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2026609ae595f0358157b19311b0f9f9591db7cee705Jason Evansvoid 2027609ae595f0358157b19311b0f9f9591db7cee705Jason Evansarena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, 2028609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 2029609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_large_stats_t *lstats) 2030609ae595f0358157b19311b0f9f9591db7cee705Jason Evans{ 2031609ae595f0358157b19311b0f9f9591db7cee705Jason Evans unsigned i; 2032609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2033609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&arena->lock); 2034609ae595f0358157b19311b0f9f9591db7cee705Jason Evans *dss = dss_prec_names[arena->dss_prec]; 2035609ae595f0358157b19311b0f9f9591db7cee705Jason Evans *nactive += arena->nactive; 2036609ae595f0358157b19311b0f9f9591db7cee705Jason Evans *ndirty += arena->ndirty; 2037609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2038609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->mapped += arena->stats.mapped; 2039609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->npurge += arena->stats.npurge; 2040609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->nmadvise += arena->stats.nmadvise; 2041609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->purged += arena->stats.purged; 2042609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->allocated_large += arena->stats.allocated_large; 2043609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->nmalloc_large += arena->stats.nmalloc_large; 2044609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->ndalloc_large += arena->stats.ndalloc_large; 2045609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->nrequests_large += arena->stats.nrequests_large; 2046609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2047609ae595f0358157b19311b0f9f9591db7cee705Jason Evans for (i = 0; i < nlclasses; i++) { 2048609ae595f0358157b19311b0f9f9591db7cee705Jason Evans lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 2049609ae595f0358157b19311b0f9f9591db7cee705Jason Evans lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 2050609ae595f0358157b19311b0f9f9591db7cee705Jason Evans lstats[i].nrequests += arena->stats.lstats[i].nrequests; 2051609ae595f0358157b19311b0f9f9591db7cee705Jason Evans lstats[i].curruns += arena->stats.lstats[i].curruns; 2052609ae595f0358157b19311b0f9f9591db7cee705Jason Evans } 2053609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&arena->lock); 2054609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2055609ae595f0358157b19311b0f9f9591db7cee705Jason Evans for (i = 0; i < NBINS; i++) { 2056609ae595f0358157b19311b0f9f9591db7cee705Jason Evans arena_bin_t *bin = &arena->bins[i]; 2057609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2058609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&bin->lock); 2059609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].allocated += bin->stats.allocated; 2060609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nmalloc += bin->stats.nmalloc; 2061609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].ndalloc += bin->stats.ndalloc; 2062609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nrequests += bin->stats.nrequests; 2063609ae595f0358157b19311b0f9f9591db7cee705Jason Evans if (config_tcache) { 2064609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nfills += bin->stats.nfills; 2065609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nflushes += bin->stats.nflushes; 2066609ae595f0358157b19311b0f9f9591db7cee705Jason Evans } 2067609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nruns += bin->stats.nruns; 2068609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].reruns += bin->stats.reruns; 2069609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].curruns += bin->stats.curruns; 2070609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&bin->lock); 2071609ae595f0358157b19311b0f9f9591db7cee705Jason Evans } 2072609ae595f0358157b19311b0f9f9591db7cee705Jason Evans} 2073609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2074e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 2075e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_new(arena_t *arena, unsigned ind) 2076e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2077e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 2078e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 2079e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 20806109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena->ind = ind; 2081597632be188d2bcc135dad2145cc46ef44897aadJason Evans arena->nthreads = 0; 20826109fe07a14b7a619365977d9523db9f8b333792Jason Evans 2083e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (malloc_mutex_init(&arena->lock)) 2084e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2085e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 20867372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 20877372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(&arena->stats, 0, sizeof(arena_stats_t)); 20887372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats = 20897372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans (malloc_large_stats_t *)base_alloc(nlclasses * 20907372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans sizeof(malloc_large_stats_t)); 20917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (arena->stats.lstats == NULL) 20927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans return (true); 20937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(arena->stats.lstats, 0, nlclasses * 20947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans sizeof(malloc_large_stats_t)); 20957372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_tcache) 20967372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ql_new(&arena->tcache_ql); 20977372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 2098e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 20997372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) 21007372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->prof_accumbytes = 0; 2101d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 2102609ae595f0358157b19311b0f9f9591db7cee705Jason Evans arena->dss_prec = chunk_dss_prec_get(); 2103609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize chunks. */ 2105e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_new(&arena->chunks_dirty); 2106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 2107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive = 0; 2109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty = 0; 2110799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory = 0; 2111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2112e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_new(&arena->runs_avail); 2113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize bins. */ 2115b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans for (i = 0; i < NBINS; i++) { 2116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 211786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 211886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 2119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 2120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 21217372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 21227372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 212849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 212949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Calculate bin_info->run_size such that it meets the following constraints: 213049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 213149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) bin_info->run_size >= min_run_size 213249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) bin_info->run_size <= arena_maxclass 213349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 213447e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans * *) bin_info->nregs <= RUN_MAXREGS 213549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 213684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also 213784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * calculated here, since these settings are all interdependent. 213849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 213949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstatic size_t 214049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansbin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) 214149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 2142122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t pad_size; 214349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t try_run_size, good_run_size; 214449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_nregs, good_nregs; 214549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_hdr_size, good_hdr_size; 214684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans uint32_t try_bitmap_offset, good_bitmap_offset; 214749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_ctx0_offset, good_ctx0_offset; 2148122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans uint32_t try_redzone0_offset, good_redzone0_offset; 214949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2150ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans assert(min_run_size >= PAGE); 215149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(min_run_size <= arena_maxclass); 215249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 215349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 2154122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Determine redzone size based on minimum alignment and minimum 2155122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * redzone size. Add padding to the end of the run if it is needed to 2156122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * align the regions. The padding allows each redzone to be half the 2157122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * minimum alignment; without the padding, each redzone would have to 2158122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * be twice as large in order to maintain alignment. 2159122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans */ 2160122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (config_fill && opt_redzone) { 2161122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t align_min = ZU(1) << (ffs(bin_info->reg_size) - 1); 2162122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (align_min <= REDZONE_MINSIZE) { 2163122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->redzone_size = REDZONE_MINSIZE; 2164122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans pad_size = 0; 2165122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 2166122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->redzone_size = align_min >> 1; 2167122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans pad_size = bin_info->redzone_size; 2168122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 2169122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 2170122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->redzone_size = 0; 2171122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans pad_size = 0; 2172122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 2173122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval = bin_info->reg_size + 2174122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (bin_info->redzone_size << 1); 2175122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 2176122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* 217749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Calculate known-valid settings before entering the run_size 217849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * expansion loop, so that the first part of the loop always copies 217949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * valid settings. 218049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 218149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * The do..while loop iteratively reduces the number of regions until 218249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * the run header and the regions no longer overlap. A closed formula 218349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * would be quite messy, since there is an interdependency between the 218449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * header's mask length and the number of regions. 218549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 218649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_run_size = min_run_size; 2187122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / 2188122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval) 218949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 219047e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans if (try_nregs > RUN_MAXREGS) { 219147e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans try_nregs = RUN_MAXREGS 219247e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 219347e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans } 219449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 219549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs--; 219649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = sizeof(arena_run_t); 219784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Pad to a long boundary. */ 219884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size = LONG_CEILING(try_hdr_size); 219984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_bitmap_offset = try_hdr_size; 220084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Add space for bitmap. */ 220184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size += bitmap_size(try_nregs); 22027372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof && opt_prof && prof_promote == false) { 220349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Pad to a quantum boundary. */ 220449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 220549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = try_hdr_size; 220649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Add space for one (prof_ctx_t *) per region. */ 220749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size += try_nregs * sizeof(prof_ctx_t *); 220849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } else 220949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = 0; 2210122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans try_redzone0_offset = try_run_size - (try_nregs * 2211122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval) - pad_size; 2212122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } while (try_hdr_size > try_redzone0_offset); 221349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 221449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* run_size expansion loop. */ 221549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 221649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 221749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Copy valid settings before trying more aggressive settings. 221849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 221949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_run_size = try_run_size; 222049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_nregs = try_nregs; 222149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_hdr_size = try_hdr_size; 222284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans good_bitmap_offset = try_bitmap_offset; 222349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_ctx0_offset = try_ctx0_offset; 2224122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans good_redzone0_offset = try_redzone0_offset; 222549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 222649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Try more aggressive settings. */ 2227ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans try_run_size += PAGE; 2228122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) / 2229122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval) 223049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 223147e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans if (try_nregs > RUN_MAXREGS) { 223247e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans try_nregs = RUN_MAXREGS 223347e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 223447e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans } 223549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 223649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs--; 223749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = sizeof(arena_run_t); 223884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Pad to a long boundary. */ 223984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size = LONG_CEILING(try_hdr_size); 224084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_bitmap_offset = try_hdr_size; 224184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Add space for bitmap. */ 224284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size += bitmap_size(try_nregs); 22437372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof && opt_prof && prof_promote == false) { 224449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Pad to a quantum boundary. */ 224549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 224649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = try_hdr_size; 224749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 224849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Add space for one (prof_ctx_t *) per region. 224949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 225049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size += try_nregs * 225149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans sizeof(prof_ctx_t *); 225249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 2253122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans try_redzone0_offset = try_run_size - (try_nregs * 2254122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval) - pad_size; 2255122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } while (try_hdr_size > try_redzone0_offset); 225649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } while (try_run_size <= arena_maxclass 225749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans && try_run_size <= arena_maxclass 2258122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > 2259122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans RUN_MAX_OVRHD_RELAX 2260122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size 226147e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans && try_nregs < RUN_MAXREGS); 226249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2263122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(good_hdr_size <= good_redzone0_offset); 226449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 226549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Copy final settings. */ 226649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->run_size = good_run_size; 226749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->nregs = good_nregs; 226884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bin_info->bitmap_offset = good_bitmap_offset; 226949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->ctx0_offset = good_ctx0_offset; 2270122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; 2271122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 2272122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs 2273122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * bin_info->reg_interval) + pad_size == bin_info->run_size); 227449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 227549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (good_run_size); 227649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 227749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2278b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansstatic void 227949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansbin_info_init(void) 228049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 228149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 2282ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t prev_run_size = PAGE; 2283b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans 2284b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define SIZE_CLASS(bin, delta, size) \ 2285b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info = &arena_bin_info[bin]; \ 2286b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info->reg_size = size; \ 2287b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ 2288b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 2289b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans SIZE_CLASSES 2290b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef SIZE_CLASS 229149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 229249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2293b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansvoid 2294a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evansarena_boot(void) 2295e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2296a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans size_t header_size; 22977393f44ff025ca67716fc53b68003fd65122fd97Jason Evans unsigned i; 2298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Compute the header size such that it is large enough to contain the 23017393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * page map. The page map is biased to omit entries for the header 23027393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * itself, so some iteration is necessary to compute the map bias. 23037393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 23047393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 1) Compute safe header_size and map_bias values that include enough 23057393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * space for an unbiased page map. 23067393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 2) Refine map_bias based on (1) to omit the header pages in the page 23077393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * map. The resulting map_bias may be one too small. 23087393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 3) Refine map_bias based on (2). The result will be >= the result 23097393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * from (2), and will always be correct. 2310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 23117393f44ff025ca67716fc53b68003fd65122fd97Jason Evans map_bias = 0; 23127393f44ff025ca67716fc53b68003fd65122fd97Jason Evans for (i = 0; i < 3; i++) { 2313ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans header_size = offsetof(arena_chunk_t, map) + 2314ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); 2315ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) 2316ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans != 0); 23177393f44ff025ca67716fc53b68003fd65122fd97Jason Evans } 23187393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(map_bias > 0); 23197393f44ff025ca67716fc53b68003fd65122fd97Jason Evans 2320ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena_maxclass = chunksize - (map_bias << LG_PAGE); 2321a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans 2322b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info_init(); 2323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 23244e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 23254e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 23264e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansarena_prefork(arena_t *arena) 23274e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 23284e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans unsigned i; 23294e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 23304e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_prefork(&arena->lock); 23314e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans for (i = 0; i < NBINS; i++) 23324e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_prefork(&arena->bins[i].lock); 23334e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 23344e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 23354e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 23364e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansarena_postfork_parent(arena_t *arena) 23374e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 23384e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans unsigned i; 23394e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 23404e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans for (i = 0; i < NBINS; i++) 23414e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_parent(&arena->bins[i].lock); 23424e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_parent(&arena->lock); 23434e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 23444e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 23454e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 23464e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansarena_postfork_child(arena_t *arena) 23474e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 23484e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans unsigned i; 23494e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 23504e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans for (i = 0; i < NBINS; i++) 23514e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_child(&arena->bins[i].lock); 23524e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_child(&arena->lock); 23534e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 2354