arena.c revision f9ff60346d7c25ad653ea062e496a5d0864233b2
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_ARENA_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evansssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 8b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansarena_bin_info_t arena_bin_info[NBINS]; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 10da99e31105eb709ef4ec8a120b115c32a6b9723aMike HommeyJEMALLOC_ALIGNED(CACHELINE) 11b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansconst uint8_t small_size2bin[] = { 1241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#define S2B_8(i) i, 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_16(i) S2B_8(i) S2B_8(i) 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_32(i) S2B_16(i) S2B_16(i) 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_64(i) S2B_32(i) S2B_32(i) 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_128(i) S2B_64(i) S2B_64(i) 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_256(i) S2B_128(i) S2B_128(i) 18b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_512(i) S2B_256(i) S2B_256(i) 19b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_1024(i) S2B_512(i) S2B_512(i) 20b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_2048(i) S2B_1024(i) S2B_1024(i) 21b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_4096(i) S2B_2048(i) S2B_2048(i) 22b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_8192(i) S2B_4096(i) S2B_4096(i) 23b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define SIZE_CLASS(bin, delta, size) \ 24b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans S2B_##delta(bin) 25b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans SIZE_CLASSES 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_8 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_16 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_32 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_64 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_128 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_256 32b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_512 33b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_1024 34b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_2048 35b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_4096 36b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_8192 37b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef SIZE_CLASS 38b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans}; 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 41aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans/* 42aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Function prototypes for static functions that are referenced prior to 43aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * definition. 44aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 45aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 466005f0710cf07d60659d91b20b7ff5592d310027Jason Evansstatic void arena_purge(arena_t *arena, bool all); 47e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, 48e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans bool cleaned); 49e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, arena_bin_t *bin); 51940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansstatic void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 52940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_t *run, arena_bin_t *bin); 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 56f9ff60346d7c25ad653ea062e496a5d0864233b2Ben MaurerJEMALLOC_INLINE_C size_t 57f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurerarena_mapelm_to_pageind(arena_chunk_map_t *mapelm) 58f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer{ 59f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer uintptr_t map_offset = 60f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer CHUNK_ADDR2OFFSET(mapelm) - offsetof(arena_chunk_t, map); 61f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer 62f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer return ((map_offset / sizeof(arena_chunk_map_t)) + map_bias); 63f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer} 64f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer 65f9ff60346d7c25ad653ea062e496a5d0864233b2Ben MaurerJEMALLOC_INLINE_C size_t 66f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurerarena_mapelm_to_bits(arena_chunk_map_t *mapelm) 67f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer{ 68f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer 69f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer return (mapelm->bits); 70f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer} 71f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer 72e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 73e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 74e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 75e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm = (uintptr_t)a; 76e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t b_mapelm = (uintptr_t)b; 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(a != NULL); 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(b != NULL); 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 83e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 84f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 857372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansrb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, 867372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans u.rb_link, arena_run_comp) 87e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 89e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 90e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 91e476f8a161d445211fd6e54fe370275196e66bcbJason Evans int ret; 92f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer size_t a_size; 93f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer size_t b_size = arena_mapelm_to_bits(b) & ~PAGE_MASK; 94f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer uintptr_t a_mapelm = (uintptr_t)a; 95f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer uintptr_t b_mapelm = (uintptr_t)b; 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 97f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer if (a_mapelm & CHUNK_MAP_KEY) 98f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer a_size = a_mapelm & ~PAGE_MASK; 99f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer else 100f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer a_size = arena_mapelm_to_bits(a) & ~PAGE_MASK; 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 102f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer ret = (a_size > b_size) - (a_size < b_size); 103f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer if (ret == 0 && (!(a_mapelm & CHUNK_MAP_KEY))) 104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 109f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 1107372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansrb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, 1117372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans u.rb_link, arena_avail_comp) 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 113e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline int 114e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_chunk_dirty_comp(arena_chunk_t *a, arena_chunk_t *b) 115e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 116e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 117e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(a != NULL); 118e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(b != NULL); 119e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 120e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 121abf6739317742ca4677bf885178984a8757ee14aJason Evans * Short-circuit for self comparison. The following comparison code 122abf6739317742ca4677bf885178984a8757ee14aJason Evans * would come to the same result, but at the cost of executing the slow 123abf6739317742ca4677bf885178984a8757ee14aJason Evans * path. 124abf6739317742ca4677bf885178984a8757ee14aJason Evans */ 125abf6739317742ca4677bf885178984a8757ee14aJason Evans if (a == b) 126abf6739317742ca4677bf885178984a8757ee14aJason Evans return (0); 127abf6739317742ca4677bf885178984a8757ee14aJason Evans 128abf6739317742ca4677bf885178984a8757ee14aJason Evans /* 129e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Order such that chunks with higher fragmentation are "less than" 130abf6739317742ca4677bf885178984a8757ee14aJason Evans * those with lower fragmentation -- purging order is from "least" to 131abf6739317742ca4677bf885178984a8757ee14aJason Evans * "greatest". Fragmentation is measured as: 132e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 133e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * mean current avail run size 134e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * -------------------------------- 135e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * mean defragmented avail run size 136e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 137e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * navail 138e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * ----------- 139e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * nruns_avail nruns_avail-nruns_adjac 140e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * = ========================= = ----------------------- 141e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * navail nruns_avail 142e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * ----------------------- 143e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * nruns_avail-nruns_adjac 144e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 145e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * The following code multiplies away the denominator prior to 146e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * comparison, in order to avoid division. 147e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 148e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 149abf6739317742ca4677bf885178984a8757ee14aJason Evans { 150abf6739317742ca4677bf885178984a8757ee14aJason Evans size_t a_val = (a->nruns_avail - a->nruns_adjac) * 151abf6739317742ca4677bf885178984a8757ee14aJason Evans b->nruns_avail; 152abf6739317742ca4677bf885178984a8757ee14aJason Evans size_t b_val = (b->nruns_avail - b->nruns_adjac) * 153abf6739317742ca4677bf885178984a8757ee14aJason Evans a->nruns_avail; 154abf6739317742ca4677bf885178984a8757ee14aJason Evans 155abf6739317742ca4677bf885178984a8757ee14aJason Evans if (a_val < b_val) 156abf6739317742ca4677bf885178984a8757ee14aJason Evans return (1); 157abf6739317742ca4677bf885178984a8757ee14aJason Evans if (a_val > b_val) 158abf6739317742ca4677bf885178984a8757ee14aJason Evans return (-1); 159abf6739317742ca4677bf885178984a8757ee14aJason Evans } 160abf6739317742ca4677bf885178984a8757ee14aJason Evans /* 161abf6739317742ca4677bf885178984a8757ee14aJason Evans * Break ties by chunk address. For fragmented chunks, report lower 162abf6739317742ca4677bf885178984a8757ee14aJason Evans * addresses as "lower", so that fragmentation reduction happens first 163abf6739317742ca4677bf885178984a8757ee14aJason Evans * at lower addresses. However, use the opposite ordering for 164abf6739317742ca4677bf885178984a8757ee14aJason Evans * unfragmented chunks, in order to increase the chances of 165abf6739317742ca4677bf885178984a8757ee14aJason Evans * re-allocating dirty runs. 166abf6739317742ca4677bf885178984a8757ee14aJason Evans */ 167e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans { 168e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans uintptr_t a_chunk = (uintptr_t)a; 169e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans uintptr_t b_chunk = (uintptr_t)b; 170abf6739317742ca4677bf885178984a8757ee14aJason Evans int ret = ((a_chunk > b_chunk) - (a_chunk < b_chunk)); 171abf6739317742ca4677bf885178984a8757ee14aJason Evans if (a->nruns_adjac == 0) { 172abf6739317742ca4677bf885178984a8757ee14aJason Evans assert(b->nruns_adjac == 0); 173abf6739317742ca4677bf885178984a8757ee14aJason Evans ret = -ret; 174abf6739317742ca4677bf885178984a8757ee14aJason Evans } 175abf6739317742ca4677bf885178984a8757ee14aJason Evans return (ret); 176e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 177e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 178e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 179e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans/* Generate red-black tree functions. */ 180e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansrb_gen(static UNUSED, arena_chunk_dirty_, arena_chunk_tree_t, arena_chunk_t, 181e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans dirty_link, arena_chunk_dirty_comp) 182e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 183e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline bool 184e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind) 185e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 186e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans bool ret; 187e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 188e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (pageind-1 < map_bias) 189e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans ret = false; 190e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans else { 191e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0); 192e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(ret == false || arena_mapbits_dirty_get(chunk, 193e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans pageind-1) != arena_mapbits_dirty_get(chunk, pageind)); 194e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 195e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (ret); 196e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 197e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 198e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline bool 199e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages) 200e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 201e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans bool ret; 202e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 203e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (pageind+npages == chunk_npages) 204e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans ret = false; 205e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans else { 206e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(pageind+npages < chunk_npages); 207e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0); 208e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(ret == false || arena_mapbits_dirty_get(chunk, pageind) 209e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans != arena_mapbits_dirty_get(chunk, pageind+npages)); 210e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 211e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (ret); 212e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 213e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 214e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline bool 215e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages) 216e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 217e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 218e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (arena_avail_adjac_pred(chunk, pageind) || 219e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_adjac_succ(chunk, pageind, npages)); 220e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 221e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 222e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic void 223e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 224e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 225e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 226e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 227e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 228e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans LG_PAGE)); 229e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 230e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 231e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 232e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * removed and reinserted even if the run to be inserted is clean. 233e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 234e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->ndirty != 0) 235e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 236e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 237e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 238e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac++; 239e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 240e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac++; 241e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_avail++; 242e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->nruns_avail > chunk->nruns_adjac); 243e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 244e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 245e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->ndirty += npages; 246e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->ndirty += npages; 247e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 248e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->ndirty != 0) 249e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 250e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 251e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, 252e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans pageind)); 253e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 254e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 255e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic void 256e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 257e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 258e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 259e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 260e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 261e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans LG_PAGE)); 262e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 263e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 264e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 265e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * removed and reinserted even if the run to be removed is clean. 266e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 267e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->ndirty != 0) 268e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 269e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 270e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 271e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac--; 272e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 273e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac--; 274e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_avail--; 275e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->nruns_avail > chunk->nruns_adjac || (chunk->nruns_avail 276e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans == 0 && chunk->nruns_adjac == 0)); 277e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 278e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 279e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->ndirty -= npages; 280e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->ndirty -= npages; 281e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 282e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->ndirty != 0) 283e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 284e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 285e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, 286e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans pageind)); 287e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 288e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 289e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void * 29049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 29384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned regind; 29484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 29584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2971e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(run->nfree > 0); 29884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); 299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 30084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); 30184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + 302122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (uintptr_t)(bin_info->reg_interval * regind)); 3031e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree--; 30484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans if (regind == run->nextind) 30584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans run->nextind++; 30684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(regind < run->nextind); 3071e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans return (ret); 3086109fe07a14b7a619365977d9523db9f8b333792Jason Evans} 3096109fe07a14b7a619365977d9523db9f8b333792Jason Evans 3106109fe07a14b7a619365977d9523db9f8b333792Jason Evansstatic inline void 3111e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evansarena_run_reg_dalloc(arena_run_t *run, void *ptr) 3126109fe07a14b7a619365977d9523db9f8b333792Jason Evans{ 31349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 314203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 315203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits = arena_mapbits_get(chunk, pageind); 31680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t binind = arena_ptr_small_binind_get(ptr, mapbits); 31749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 31884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned regind = arena_run_regind(run, bin_info, ptr); 31984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 32084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 32184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 32249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(run->nfree < bin_info->nregs); 3231e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Freeing an interior pointer can cause assertion failure. */ 3241e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 325122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (uintptr_t)bin_info->reg0_offset)) % 326122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (uintptr_t)bin_info->reg_interval == 0); 32721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans assert((uintptr_t)ptr >= (uintptr_t)run + 32849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset); 32984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Freeing an unallocated pointer can cause assertion failure. */ 33084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 33284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_unset(bitmap, &bin_info->bitmap_info, regind); 3331e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree++; 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 33621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evansstatic inline void 33738067483c542adfe092644d1ecc103c6bc74add0Jason Evansarena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) 33838067483c542adfe092644d1ecc103c6bc74add0Jason Evans{ 33938067483c542adfe092644d1ecc103c6bc74add0Jason Evans 34038067483c542adfe092644d1ecc103c6bc74add0Jason Evans VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << 34138067483c542adfe092644d1ecc103c6bc74add0Jason Evans LG_PAGE)), (npages << LG_PAGE)); 34238067483c542adfe092644d1ecc103c6bc74add0Jason Evans memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, 34338067483c542adfe092644d1ecc103c6bc74add0Jason Evans (npages << LG_PAGE)); 34438067483c542adfe092644d1ecc103c6bc74add0Jason Evans} 34538067483c542adfe092644d1ecc103c6bc74add0Jason Evans 34638067483c542adfe092644d1ecc103c6bc74add0Jason Evansstatic inline void 347dda90f59e2b67903668a2799970f64df163e9ccfJason Evansarena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) 348dda90f59e2b67903668a2799970f64df163e9ccfJason Evans{ 349dda90f59e2b67903668a2799970f64df163e9ccfJason Evans 350dda90f59e2b67903668a2799970f64df163e9ccfJason Evans VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind << 351dda90f59e2b67903668a2799970f64df163e9ccfJason Evans LG_PAGE)), PAGE); 352dda90f59e2b67903668a2799970f64df163e9ccfJason Evans} 353dda90f59e2b67903668a2799970f64df163e9ccfJason Evans 354dda90f59e2b67903668a2799970f64df163e9ccfJason Evansstatic inline void 35538067483c542adfe092644d1ecc103c6bc74add0Jason Evansarena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 35621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans{ 357d4bab21756279db540866998099522dbd39c05f7Jason Evans size_t i; 358ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); 359d4bab21756279db540866998099522dbd39c05f7Jason Evans 360dda90f59e2b67903668a2799970f64df163e9ccfJason Evans arena_run_page_mark_zeroed(chunk, run_ind); 361ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans for (i = 0; i < PAGE / sizeof(size_t); i++) 36221fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans assert(p[i] == 0); 36321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans} 36421fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans 365e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 366aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_cactive_update(arena_t *arena, size_t add_pages, size_t sub_pages) 367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 368e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 369aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_stats) { 370aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ssize_t cactive_diff = CHUNK_CEILING((arena->nactive + 371aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans add_pages) << LG_PAGE) - CHUNK_CEILING((arena->nactive - 372aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans sub_pages) << LG_PAGE); 373aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (cactive_diff != 0) 374aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans stats_cactive_add(cactive_diff); 375aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 376aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 377aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 378aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic void 379aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, 380aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t flag_dirty, size_t need_pages) 381aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 382aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t total_pages, rem_pages; 383aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 384aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> 385aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans LG_PAGE; 386aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == 387aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty); 388aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(need_pages <= total_pages); 389aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans rem_pages = total_pages - need_pages; 390aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 391aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); 392aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_cactive_update(arena, need_pages, 0); 393aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena->nactive += need_pages; 394aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 395aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* Keep track of trailing unused pages for later use. */ 396aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (rem_pages > 0) { 397aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (flag_dirty != 0) { 398aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, 399aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+need_pages, (rem_pages << LG_PAGE), 400aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty); 401aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, 402aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+total_pages-1, (rem_pages << LG_PAGE), 403aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty); 404aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } else { 405aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 406aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans (rem_pages << LG_PAGE), 407aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unzeroed_get(chunk, 408aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+need_pages)); 409aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, 410aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+total_pages-1, (rem_pages << LG_PAGE), 411aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unzeroed_get(chunk, 412aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+total_pages-1)); 413aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 414aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages, 415aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans false, true); 416aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 417aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 418aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 419aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic void 420aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, 421aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans bool remove, bool zero) 422aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 423aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *chunk; 424aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t flag_dirty, run_ind, need_pages, i; 425203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 426e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 427ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 428203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 429ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans need_pages = (size >> LG_PAGE); 430e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages > 0); 431e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 432c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans if (remove) { 433aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_remove(arena, chunk, run_ind, flag_dirty, 434aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans need_pages); 435aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 436c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans 437aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (zero) { 438aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (flag_dirty == 0) { 439c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans /* 440aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * The run is clean, so some pages may be zeroed (i.e. 441aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * never before touched). 442c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans */ 443aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans for (i = 0; i < need_pages; i++) { 444aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (arena_mapbits_unzeroed_get(chunk, run_ind+i) 445aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans != 0) 446aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_zero(chunk, run_ind+i, 1); 447aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans else if (config_debug) { 448aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_page_validate_zeroed(chunk, 449aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+i); 450aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } else { 451aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_page_mark_zeroed(chunk, 452aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+i); 45319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 454e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 455dda90f59e2b67903668a2799970f64df163e9ccfJason Evans } else { 456aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* The run is dirty, so all pages must be zeroed. */ 457aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_zero(chunk, run_ind, need_pages); 458e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 45919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 460dda90f59e2b67903668a2799970f64df163e9ccfJason Evans VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 461dda90f59e2b67903668a2799970f64df163e9ccfJason Evans (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 462e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 463aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 464aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 465aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Set the last element first, in case the run only contains one page 466aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * (i.e. both statements set the same element). 467aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 468aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty); 469aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); 470e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 471e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 472c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evansstatic void 473aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 474c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans{ 475c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans 476aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_large_helper(arena, run, size, true, zero); 477c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans} 478c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans 479c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evansstatic void 480aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 481c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans{ 482c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans 483aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_large_helper(arena, run, size, false, zero); 484aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 485aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 486aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic void 487aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, 488aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t binind) 489aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 490aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *chunk; 491aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t flag_dirty, run_ind, need_pages, i; 492aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 493aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(binind != BININD_INVALID); 494aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 495aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 496aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 497aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 498aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans need_pages = (size >> LG_PAGE); 499aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(need_pages > 0); 500aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 501aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_remove(arena, chunk, run_ind, flag_dirty, need_pages); 502aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 503aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 504aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Propagate the dirty and unzeroed flags to the allocated small run, 505aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * so that arena_dalloc_bin_run() has the ability to conditionally trim 506aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * clean pages. 507aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 508aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); 509aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 510aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * The first page will always be dirtied during small run 511aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * initialization, so a validation failure here would not actually 512aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * cause an observable failure. 513aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 514aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, 515aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind) == 0) 516aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_page_validate_zeroed(chunk, run_ind); 517aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans for (i = 1; i < need_pages - 1; i++) { 518aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); 519aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_debug && flag_dirty == 0 && 520aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) 521aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_page_validate_zeroed(chunk, run_ind+i); 522aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 523aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_small_set(chunk, run_ind+need_pages-1, need_pages-1, 524aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans binind, flag_dirty); 525aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, 526aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+need_pages-1) == 0) 527aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_page_validate_zeroed(chunk, run_ind+need_pages-1); 528aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 529aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 530c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans} 531c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans 532e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t * 533aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_init_spare(arena_t *arena) 534e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 535e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 536e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 537aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena->spare != NULL); 53819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 539aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = arena->spare; 540aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena->spare = NULL; 54141631d00618d7262125e501c91d31b4d70e605faJason Evans 542aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 543aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 544aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 545aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_maxclass); 546aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 547aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_maxclass); 548aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_dirty_get(chunk, map_bias) == 549aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_dirty_get(chunk, chunk_npages-1)); 550e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 551aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (chunk); 552aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 553e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 554aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic arena_chunk_t * 555aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_init_hard(arena_t *arena) 556aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 557aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *chunk; 558aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans bool zero; 559aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t unzeroed, i; 560e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 561aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena->spare == NULL); 562e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 563aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans zero = false; 564aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans malloc_mutex_unlock(&arena->lock); 565aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize, false, 566aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans &zero, arena->dss_prec); 567aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans malloc_mutex_lock(&arena->lock); 568aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (chunk == NULL) 569aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (NULL); 570aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_stats) 571aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena->stats.mapped += chunksize; 572aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 573aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk->arena = arena; 574aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 575aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 576aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Claim that no pages are in use, since the header is merely overhead. 577aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 578aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk->ndirty = 0; 579aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 580aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk->nruns_avail = 0; 581aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk->nruns_adjac = 0; 582aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 583aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 584aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Initialize the map to contain one maximal free untouched run. Mark 585aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * the pages as zeroed iff chunk_alloc() returned a zeroed chunk. 586aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 587aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; 588aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, 589aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans unzeroed); 590aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 591aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * There is no need to initialize the internal page map entries unless 592aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * the chunk is not zeroed. 593aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 594aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (zero == false) { 595aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans VALGRIND_MAKE_MEM_UNDEFINED((void *)arena_mapp_get(chunk, 596aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, 597aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, 598aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans map_bias+1))); 599aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans for (i = map_bias+1; i < chunk_npages-1; i++) 600aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unzeroed_set(chunk, i, unzeroed); 601aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } else { 602aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans VALGRIND_MAKE_MEM_DEFINED((void *)arena_mapp_get(chunk, 603aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, 604aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, 605aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans map_bias+1))); 606aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_debug) { 607aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans for (i = map_bias+1; i < chunk_npages-1; i++) { 608aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_unzeroed_get(chunk, i) == 609aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans unzeroed); 610203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 611940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 61219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 613aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxclass, 614aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans unzeroed); 615aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 616aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (chunk); 617aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 618aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 619aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic arena_chunk_t * 620aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_alloc(arena_t *arena) 621aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 622aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *chunk; 623aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 624aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (arena->spare != NULL) 625aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = arena_chunk_init_spare(arena); 62620a8c78bfe3310e0f0f72b596d4e10ca7336063bChris Pride else { 627aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = arena_chunk_init_hard(arena); 62820a8c78bfe3310e0f0f72b596d4e10ca7336063bChris Pride if (chunk == NULL) 62920a8c78bfe3310e0f0f72b596d4e10ca7336063bChris Pride return (NULL); 63020a8c78bfe3310e0f0f72b596d4e10ca7336063bChris Pride } 631e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 632e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Insert the run into the runs_avail tree. */ 633e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias, 634e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false, false); 635e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 636e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (chunk); 637e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 638e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 639e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 640e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) 641e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 64230fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 64330fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 64430fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 64530fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_maxclass); 64630fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 64730fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_maxclass); 64830fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_dirty_get(chunk, map_bias) == 64930fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_mapbits_dirty_get(chunk, chunk_npages-1)); 65030fe12b866edbc2cf9aaef299063b392ea125aacJason Evans 6518d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans /* 652e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Remove run from the runs_avail tree, so that the arena does not use 653e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * it. 6548d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans */ 655e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias, 656e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false, false); 6578d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans 6588d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (arena->spare != NULL) { 659e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_chunk_t *spare = arena->spare; 660e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 6618d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans arena->spare = chunk; 662e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&arena->lock); 66312a488782681cbd740a5f54e0b7e74ea84858e21Jason Evans chunk_dealloc((void *)spare, chunksize, true); 664e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&arena->lock); 6657372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 6667372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.mapped -= chunksize; 6678d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans } else 6688d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans arena->spare = chunk; 669e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 670e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 671e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 672aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) 673aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 674aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_t *run; 675f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer arena_chunk_map_t *mapelm; 676f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer arena_chunk_map_t *key; 677aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 678f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); 679f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); 680aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (mapelm != NULL) { 681aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 682f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer size_t pageind = arena_mapelm_to_pageind(mapelm); 683aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 684aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 685aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans LG_PAGE)); 686aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_large(arena, run, size, zero); 687aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (run); 688aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 689aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 690aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (NULL); 691aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 692aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 693aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic arena_run_t * 694aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_alloc_large(arena_t *arena, size_t size, bool zero) 695aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 696aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *chunk; 697aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_t *run; 698aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 699aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(size <= arena_maxclass); 700aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert((size & PAGE_MASK) == 0); 701aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 702aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* Search the arena's chunks for the lowest best fit. */ 703aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run = arena_run_alloc_large_helper(arena, size, zero); 704aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (run != NULL) 705aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (run); 706aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 707aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 708aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * No usable runs. Create a new chunk from which to allocate the run. 709aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 710aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = arena_chunk_alloc(arena); 711aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (chunk != NULL) { 712aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); 713aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_large(arena, run, size, zero); 714aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (run); 715aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 716aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 717aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 718aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * arena_chunk_alloc() failed, but another thread may have made 719aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * sufficient memory available while this one dropped arena->lock in 720aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * arena_chunk_alloc(), so search one more time. 721aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 722aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (arena_run_alloc_large_helper(arena, size, zero)); 723aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 724aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 725aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic arena_run_t * 726aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind) 727e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 728e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 729f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer arena_chunk_map_t *mapelm; 730f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer arena_chunk_map_t *key; 731e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 732f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); 733f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); 734e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 735e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 736f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer size_t pageind = arena_mapelm_to_pageind(mapelm); 737e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 738e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 739ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 740aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_small(arena, run, size, binind); 741e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 742e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 743e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7445b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans return (NULL); 7455b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans} 7465b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans 7475b0c99649fa71674daadf4dd53b1ab05428483fbJason Evansstatic arena_run_t * 748aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_alloc_small(arena_t *arena, size_t size, size_t binind) 7495b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans{ 7505b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans arena_chunk_t *chunk; 7515b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans arena_run_t *run; 7525b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans 7535b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans assert(size <= arena_maxclass); 7545b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans assert((size & PAGE_MASK) == 0); 755aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(binind != BININD_INVALID); 7565b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans 7575b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans /* Search the arena's chunks for the lowest best fit. */ 758aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run = arena_run_alloc_small_helper(arena, size, binind); 7595b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans if (run != NULL) 7605b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans return (run); 7615b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans 762e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 763e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * No usable runs. Create a new chunk from which to allocate the run. 764e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 765e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena_chunk_alloc(arena); 766e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (chunk != NULL) { 767ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); 768aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_small(arena, run, size, binind); 769e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 770e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 771e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 772e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 773e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_chunk_alloc() failed, but another thread may have made 774e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * sufficient memory available while this one dropped arena->lock in 775e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_chunk_alloc(), so search one more time. 776e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 777aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (arena_run_alloc_small_helper(arena, size, binind)); 778e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 779e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 78005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansstatic inline void 78105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansarena_maybe_purge(arena_t *arena) 78205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans{ 783e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npurgeable, threshold; 784e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 785e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Don't purge if the option is disabled. */ 786e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (opt_lg_dirty_mult < 0) 787e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return; 788e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Don't purge if all dirty pages are already being purged. */ 789e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena->ndirty <= arena->npurgatory) 790e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return; 791e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgeable = arena->ndirty - arena->npurgatory; 792e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans threshold = (arena->nactive >> opt_lg_dirty_mult); 793e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 794e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Don't purge unless the number of purgeable pages exceeds the 795e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * threshold. 796e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 797e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (npurgeable <= threshold) 798e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return; 79905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 800e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_purge(arena, false); 80105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans} 80205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 803aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic arena_chunk_t * 804aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evanschunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) 80505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans{ 806aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t *ndirty = (size_t *)arg; 80705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 808aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(chunk->ndirty != 0); 809aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans *ndirty += chunk->ndirty; 810aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (NULL); 811aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 812aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 813aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic size_t 814aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_compute_npurgatory(arena_t *arena, bool all) 815aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 816aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t npurgatory, npurgeable; 81705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 81805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 819aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Compute the minimum number of pages that this thread should try to 820aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * purge. 82105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 822aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans npurgeable = arena->ndirty - arena->npurgatory; 82330fe12b866edbc2cf9aaef299063b392ea125aacJason Evans 824aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (all == false) { 825aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t threshold = (arena->nactive >> opt_lg_dirty_mult); 82605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 827aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans npurgatory = npurgeable - threshold; 828aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } else 829aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans npurgatory = npurgeable; 830e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 831aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (npurgatory); 832aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 833aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 834aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic void 835aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_stash_dirty(arena_t *arena, arena_chunk_t *chunk, bool all, 836aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_mapelms_t *mapelms) 837aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 838aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t pageind, npages; 839e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 840e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 841e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Temporarily allocate free dirty runs within chunk. If all is false, 842e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * only operate on dirty runs that are fragments; otherwise operate on 843e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * all dirty runs. 844e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 845e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { 846aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 847203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (arena_mapbits_allocated_get(chunk, pageind) == 0) { 848e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t run_size = 849e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_mapbits_unallocated_size_get(chunk, pageind); 85005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 851e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages = run_size >> LG_PAGE; 852e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind + npages <= chunk_npages); 85330fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_dirty_get(chunk, pageind) == 85430fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_mapbits_dirty_get(chunk, pageind+npages-1)); 855c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 856e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena_mapbits_dirty_get(chunk, pageind) != 0 && 857e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans (all || arena_avail_adjac(chunk, pageind, 858e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages))) { 859e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t) 860e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk + (uintptr_t)(pageind << LG_PAGE)); 861e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 862aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_large(arena, run, run_size, 863aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans false); 86419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Append to list for later processing. */ 86519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans ql_elm_new(mapelm, u.ql_link); 866aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ql_tail_insert(mapelms, mapelm, u.ql_link); 86705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 86805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } else { 869e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Skip run. */ 870e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena_mapbits_large_get(chunk, pageind) != 0) { 871e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages = arena_mapbits_large_size_get(chunk, 872203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind) >> LG_PAGE; 873e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } else { 8748b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey size_t binind; 8758b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey arena_bin_info_t *bin_info; 87605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t) 877ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans chunk + (uintptr_t)(pageind << LG_PAGE)); 878e69bee01de62b56d3e585042d341743239568043Jason Evans 879203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_small_runind_get(chunk, 880203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind) == 0); 8818b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey binind = arena_bin_index(arena, run->bin); 8828b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey bin_info = &arena_bin_info[binind]; 883e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages = bin_info->run_size >> LG_PAGE; 88405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 88505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 88605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 887e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind == chunk_npages); 888e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->ndirty == 0 || all == false); 889e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->nruns_adjac == 0); 890aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 891aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 892aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic size_t 893aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_purge_stashed(arena_t *arena, arena_chunk_t *chunk, 894aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_mapelms_t *mapelms) 895aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 896aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t npurged, pageind, npages, nmadvise; 897aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_map_t *mapelm; 89805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 89905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans malloc_mutex_unlock(&arena->lock); 9007372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 9017372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans nmadvise = 0; 902e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurged = 0; 903aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ql_foreach(mapelm, mapelms, u.ql_link) { 9047de92767c20cb72c94609b9c78985526fb84a679Jason Evans bool unzeroed; 9057de92767c20cb72c94609b9c78985526fb84a679Jason Evans size_t flag_unzeroed, i; 90605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 907f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer pageind = arena_mapelm_to_pageind(mapelm); 908e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages = arena_mapbits_large_size_get(chunk, pageind) >> 909e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans LG_PAGE; 910e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind + npages <= chunk_npages); 9117de92767c20cb72c94609b9c78985526fb84a679Jason Evans unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind << 9127de92767c20cb72c94609b9c78985526fb84a679Jason Evans LG_PAGE)), (npages << LG_PAGE)); 9137de92767c20cb72c94609b9c78985526fb84a679Jason Evans flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; 9147de92767c20cb72c94609b9c78985526fb84a679Jason Evans /* 9157de92767c20cb72c94609b9c78985526fb84a679Jason Evans * Set the unzeroed flag for all pages, now that pages_purge() 9167de92767c20cb72c94609b9c78985526fb84a679Jason Evans * has returned whether the pages were zeroed as a side effect 9177de92767c20cb72c94609b9c78985526fb84a679Jason Evans * of purging. This chunk map modification is safe even though 9187de92767c20cb72c94609b9c78985526fb84a679Jason Evans * the arena mutex isn't currently owned by this thread, 9197de92767c20cb72c94609b9c78985526fb84a679Jason Evans * because the run is marked as allocated, thus protecting it 9207de92767c20cb72c94609b9c78985526fb84a679Jason Evans * from being modified by any other thread. As long as these 9217de92767c20cb72c94609b9c78985526fb84a679Jason Evans * writes don't perturb the first and last elements' 9227de92767c20cb72c94609b9c78985526fb84a679Jason Evans * CHUNK_MAP_ALLOCATED bits, behavior is well defined. 9237de92767c20cb72c94609b9c78985526fb84a679Jason Evans */ 9247de92767c20cb72c94609b9c78985526fb84a679Jason Evans for (i = 0; i < npages; i++) { 9257de92767c20cb72c94609b9c78985526fb84a679Jason Evans arena_mapbits_unzeroed_set(chunk, pageind+i, 9267de92767c20cb72c94609b9c78985526fb84a679Jason Evans flag_unzeroed); 9277de92767c20cb72c94609b9c78985526fb84a679Jason Evans } 928e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurged += npages; 9297372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 9307372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans nmadvise++; 93105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 93205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans malloc_mutex_lock(&arena->lock); 9337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 9347372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmadvise += nmadvise; 93505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 936aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (npurged); 937aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 938aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 939aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic void 940aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_unstash_purged(arena_t *arena, arena_chunk_t *chunk, 941aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_mapelms_t *mapelms) 942aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 943aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_map_t *mapelm; 944aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t pageind; 945aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 94605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Deallocate runs. */ 947aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans for (mapelm = ql_first(mapelms); mapelm != NULL; 948aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans mapelm = ql_first(mapelms)) { 949e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_t *run; 95005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 951f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer pageind = arena_mapelm_to_pageind(mapelm); 952e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << 953e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans LG_PAGE)); 954aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ql_remove(mapelms, mapelm, u.ql_link); 955e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_dalloc(arena, run, false, true); 95605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 957e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 958e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 959aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic inline size_t 960aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) 961e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 962aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t npurged; 963aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_mapelms_t mapelms; 964e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 965aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ql_new(&mapelms); 966aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 967aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 968aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * If chunk is the spare, temporarily re-allocate it, 1) so that its 969aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * run is reinserted into runs_avail, and 2) so that it cannot be 970aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * completely discarded by another thread while arena->lock is dropped 971aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * by this thread. Note that the arena_run_dalloc() call will 972aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * implicitly deallocate the chunk, so no explicit action is required 973aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * in this function to deallocate the chunk. 974aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * 975aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Note that once a chunk contains dirty pages, it cannot again contain 976aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * a single run unless 1) it is a dirty run, or 2) this function purges 977aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * dirty pages and causes the transition to a single clean run. Thus 978aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * (chunk == arena->spare) is possible, but it is not possible for 979aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * this function to be called on the spare unless it contains a dirty 980aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * run. 981aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 982aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (chunk == arena->spare) { 983aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); 984aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); 985aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 986aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_alloc(arena); 987aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 988aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 989aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_stats) 990aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena->stats.purged += chunk->ndirty; 991aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 992aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 993aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Operate on all dirty runs if there is no clean/dirty run 994aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * fragmentation. 995aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 996aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (chunk->nruns_adjac == 0) 997aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans all = true; 998aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 999aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_stash_dirty(arena, chunk, all, &mapelms); 1000aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans npurged = arena_chunk_purge_stashed(arena, chunk, &mapelms); 1001aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_unstash_purged(arena, chunk, &mapelms); 1002aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1003aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (npurged); 100405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans} 100505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 1006e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 10076005f0710cf07d60659d91b20b7ff5592d310027Jason Evansarena_purge(arena_t *arena, bool all) 1008e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1009e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 101005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t npurgatory; 10117372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) { 10127372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t ndirty = 0; 1013e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1014e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_iter(&arena->chunks_dirty, NULL, 1015e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunks_dirty_iter_cb, (void *)&ndirty); 10167372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans assert(ndirty == arena->ndirty); 10172caa4715ed4f787f263239ff97dd824636289286Jason Evans } 1018af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans assert(arena->ndirty > arena->npurgatory || all); 1019af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - 1020f9a8edbb50f8cbcaf8ed62b36e8d7191ed223d2aJason Evans arena->npurgatory) || all); 1021e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 10227372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 10237372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.npurge++; 1024e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1025e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1026aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Add the minimum number of pages this thread should try to purge to 1027aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * arena->npurgatory. This will keep multiple threads from racing to 1028aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * reduce ndirty below the threshold. 1029e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1030aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans npurgatory = arena_compute_npurgatory(arena, all); 1031799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory += npurgatory; 1032799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans 103305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans while (npurgatory > 0) { 1034e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npurgeable, npurged, nunpurged; 1035e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 103605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Get next chunk with dirty pages. */ 1037e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk = arena_chunk_dirty_first(&arena->chunks_dirty); 103805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans if (chunk == NULL) { 103905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 104005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * This thread was unable to purge as many pages as 104105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * originally intended, due to races with other threads 1042799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * that either did some of the purging work, or re-used 1043799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * dirty pages. 104405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 1045799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory -= npurgatory; 1046799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans return; 1047e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1048e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgeable = chunk->ndirty; 1049e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(npurgeable != 0); 105005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 1051e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (npurgeable > npurgatory && chunk->nruns_adjac == 0) { 1052799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans /* 1053e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * This thread will purge all the dirty pages in chunk, 1054e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * so set npurgatory to reflect this thread's intent to 1055e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * purge the pages. This tends to reduce the chances 1056e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * of the following scenario: 1057799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 1058799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 1) This thread sets arena->npurgatory such that 1059799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * (arena->ndirty - arena->npurgatory) is at the 1060799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * threshold. 1061799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 2) This thread drops arena->lock. 1062799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 3) Another thread causes one or more pages to be 1063799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * dirtied, and immediately determines that it must 1064799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * purge dirty pages. 1065799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 1066799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * If this scenario *does* play out, that's okay, 1067799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * because all of the purging work being done really 1068799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * needs to happen. 1069799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans */ 1070e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->npurgatory += npurgeable - npurgatory; 1071e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgatory = npurgeable; 1072799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans } 1073799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans 1074e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 1075e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Keep track of how many pages are purgeable, versus how many 1076e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * actually get purged, and adjust counters accordingly. 1077e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 1078e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->npurgatory -= npurgeable; 1079e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgatory -= npurgeable; 1080e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurged = arena_chunk_purge(arena, chunk, all); 1081e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans nunpurged = npurgeable - npurged; 1082e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->npurgatory += nunpurged; 1083e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgatory += nunpurged; 1084e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1085e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1086e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 10876005f0710cf07d60659d91b20b7ff5592d310027Jason Evansvoid 10886005f0710cf07d60659d91b20b7ff5592d310027Jason Evansarena_purge_all(arena_t *arena) 10896005f0710cf07d60659d91b20b7ff5592d310027Jason Evans{ 10906005f0710cf07d60659d91b20b7ff5592d310027Jason Evans 10916005f0710cf07d60659d91b20b7ff5592d310027Jason Evans malloc_mutex_lock(&arena->lock); 10926005f0710cf07d60659d91b20b7ff5592d310027Jason Evans arena_purge(arena, true); 10936005f0710cf07d60659d91b20b7ff5592d310027Jason Evans malloc_mutex_unlock(&arena->lock); 10946005f0710cf07d60659d91b20b7ff5592d310027Jason Evans} 10956005f0710cf07d60659d91b20b7ff5592d310027Jason Evans 1096e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1097aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, 1098aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty) 1099e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1100aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t size = *p_size; 1101aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t run_ind = *p_run_ind; 1102aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t run_pages = *p_run_pages; 1103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce forward. */ 1105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run_ind + run_pages < chunk_npages && 1106203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && 1107203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) { 1108203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, 1109203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+run_pages); 1110ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t nrun_pages = nrun_size >> LG_PAGE; 1111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove successor from runs_avail; the coalesced run is 1114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 1115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1116203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_unallocated_size_get(chunk, 1117203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+run_pages+nrun_pages-1) == nrun_size); 1118203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, 1119203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+run_pages+nrun_pages-1) == flag_dirty); 1120e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages, 1121e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false, true); 1122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += nrun_size; 112412ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_pages += nrun_pages; 1125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1126203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1127203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1128203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size); 1129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1131e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce backward. */ 1132aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, 1133aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == 1134aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty) { 1135203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t prun_size = arena_mapbits_unallocated_size_get(chunk, 1136203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind-1); 1137ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t prun_pages = prun_size >> LG_PAGE; 1138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 113912ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_ind -= prun_pages; 1140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1141e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1142e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove predecessor from runs_avail; the coalesced run is 1143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 1144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1145203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1146203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans prun_size); 1147203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); 1148e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_remove(arena, chunk, run_ind, prun_pages, true, 1149e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false); 1150e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += prun_size; 115212ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_pages += prun_pages; 1153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1154203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1155203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1156203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size); 1157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1158e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1159aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans *p_size = size; 1160aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans *p_run_ind = run_ind; 1161aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans *p_run_pages = run_pages; 1162aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 1163aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1164aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic void 1165aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) 1166aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 1167aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *chunk; 1168aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t size, run_ind, run_pages, flag_dirty; 1169aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1170aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1171aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 1172aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(run_ind >= map_bias); 1173aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(run_ind < chunk_npages); 1174aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (arena_mapbits_large_get(chunk, run_ind) != 0) { 1175aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size = arena_mapbits_large_size_get(chunk, run_ind); 1176aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(size == PAGE || 1177aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_large_size_get(chunk, 1178aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+(size>>LG_PAGE)-1) == 0); 1179aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } else { 1180aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t binind = arena_bin_index(arena, run->bin); 1181aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 1182aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size = bin_info->run_size; 1183aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 1184aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_pages = (size >> LG_PAGE); 1185aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_cactive_update(arena, 0, run_pages); 1186aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena->nactive -= run_pages; 1187aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1188aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 1189aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * The run is dirty if the caller claims to have dirtied it, as well as 1190aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * if it was already dirty before being allocated and the caller 1191aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * doesn't claim to have cleaned it. 1192aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 1193aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_dirty_get(chunk, run_ind) == 1194aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1195aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) 1196aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans dirty = true; 1197aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 1198aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1199aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* Mark pages as unallocated in the chunk map. */ 1200aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (dirty) { 1201aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, run_ind, size, 1202aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans CHUNK_MAP_DIRTY); 1203aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1204aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans CHUNK_MAP_DIRTY); 1205aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } else { 1206aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, run_ind, size, 1207aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unzeroed_get(chunk, run_ind)); 1208aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1209aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); 1210aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 1211aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1212aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, 1213aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty); 1214aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1215e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert into runs_avail, now that coalescing is complete. */ 1216203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1217203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); 1218203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, run_ind) == 1219203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1220e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_insert(arena, chunk, run_ind, run_pages, true, true); 12218d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans 1222203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans /* Deallocate chunk if it is now completely unused. */ 1223203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (size == arena_maxclass) { 1224203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(run_ind == map_bias); 1225203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(run_pages == (arena_maxclass >> LG_PAGE)); 1226e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_dealloc(arena, chunk); 1227203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 1228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 12294fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 12308d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * It is okay to do dirty page processing here even if the chunk was 12314fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * deallocated above, since in that case it is the spare. Waiting 12324fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * until after possible chunk deallocation to do dirty processing 12334fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * allows for an old spare to be fully deallocated, thus decreasing the 12344fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * chances of spuriously crossing the dirty page purging threshold. 12354fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 12368d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (dirty) 123705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_maybe_purge(arena); 1238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1239e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1240e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1241e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize) 1243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1244ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1245ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t head_npages = (oldsize - newsize) >> LG_PAGE; 1246203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 1249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1251e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 1252940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * leading run as separately allocated. Set the last element of each 1253940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * run first, in case of single-page runs. 1254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1255203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1256d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1257d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty); 1258940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 12597372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) { 1260ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans UNUSED size_t tail_npages = newsize >> LG_PAGE; 1261203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, 1262203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+head_npages+tail_npages-1) == 0); 1263203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, 1264203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+head_npages+tail_npages-1) == flag_dirty); 1265940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 1266d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind+head_npages, newsize, 1267d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans flag_dirty); 1268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1269e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_dalloc(arena, run, false, false); 1270e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1272e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1273e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize, bool dirty) 1275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1276ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1277ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t head_npages = newsize >> LG_PAGE; 1278203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1279e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 1281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 1284940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * trailing run as separately allocated. Set the last element of each 1285940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * run first, in case of single-page runs. 1286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1287203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1288d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1289d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty); 1290203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1291203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (config_debug) { 1292203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; 1293203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, 1294203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+head_npages+tail_npages-1) == 0); 1295203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, 1296203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+head_npages+tail_npages-1) == flag_dirty); 1297203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 1298203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, 1299d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans flag_dirty); 1300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 1302e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans dirty, false); 1303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1305e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 1306e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_first(arena_bin_t *bin) 1307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1308e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); 1309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 1310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 13128b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey arena_run_t *run; 1313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 1315f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer pageind = arena_mapelm_to_pageind(mapelm); 1316203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1317203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_small_runind_get(chunk, pageind)) << 1318ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 1319e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1320e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans } 1321e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1322e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (NULL); 1323e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1324e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1325e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void 1326e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 1327e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1328e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); 1329ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1330203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1331e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1332e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); 1333e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1334e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_tree_insert(&bin->runs, mapelm); 1335e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1336e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1337e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void 1338e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) 1339e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1340e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1341ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1342203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1343e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1344e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); 1345e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1346e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_tree_remove(&bin->runs, mapelm); 1347e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1348e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1349e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t * 1350e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_nonfull_run_tryget(arena_bin_t *bin) 1351e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1352e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_t *run = arena_bin_runs_first(bin); 1353e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) { 1354e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_remove(bin, run); 13557372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 13567372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.reruns++; 1357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1358e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1359e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1360e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1361e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t * 1362e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 1363e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1364e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_t *run; 1365e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans size_t binind; 1366e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_info_t *bin_info; 1367e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1368e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans /* Look for a usable run. */ 1369e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans run = arena_bin_nonfull_run_tryget(bin); 1370e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) 1371e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* No existing runs have any space available. */ 1373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 137449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(arena, bin); 137549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 137649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 1377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Allocate a new run. */ 1378e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&bin->lock); 1379e69bee01de62b56d3e585042d341743239568043Jason Evans /******************************/ 138086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1381aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run = arena_run_alloc_small(arena, bin_info->run_size, binind); 1382e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run != NULL) { 138384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 138484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 138584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 1386e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* Initialize run internals. */ 1387e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run->bin = bin; 138884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans run->nextind = 0; 138949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans run->nfree = bin_info->nregs; 139084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_init(bitmap, &bin_info->bitmap_info); 1391e69bee01de62b56d3e585042d341743239568043Jason Evans } 1392e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_unlock(&arena->lock); 1393e69bee01de62b56d3e585042d341743239568043Jason Evans /********************************/ 1394e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 1395e69bee01de62b56d3e585042d341743239568043Jason Evans if (run != NULL) { 13967372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 13977372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nruns++; 13987372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.curruns++; 13997372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1400e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 1401e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1402e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1403e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 1404aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * arena_run_alloc_small() failed, but another thread may have made 1405940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * sufficient memory available while this one dropped bin->lock above, 1406e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * so search one more time. 1407e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 1408e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans run = arena_bin_nonfull_run_tryget(bin); 1409e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) 1410e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 1411e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1412e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (NULL); 1413e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1414e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 14151e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 1416e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void * 1417e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1418e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1419e00572b384c81bd2aba57fac32f7077a34388915Jason Evans void *ret; 142049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 142149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1422e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_t *run; 1423e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 142449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(arena, bin); 142549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 1426e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->runcur = NULL; 1427e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = arena_bin_nonfull_run_get(arena, bin); 1428e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (bin->runcur != NULL && bin->runcur->nfree > 0) { 1429e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 1430e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * Another thread updated runcur while this one ran without the 1431e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * bin lock in arena_bin_nonfull_run_get(). 1432e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 1433e00572b384c81bd2aba57fac32f7077a34388915Jason Evans assert(bin->runcur->nfree > 0); 143449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_run_reg_alloc(bin->runcur, bin_info); 1435e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run != NULL) { 1436940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_chunk_t *chunk; 1437940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1438940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* 1439aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * arena_run_alloc_small() may have allocated run, or 1440aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * it may have pulled run from the bin's run tree. 1441aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Therefore it is unsafe to make any assumptions about 1442aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * how run has previously been used, and 1443aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * arena_bin_lower_run() must be called, as if a region 1444aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * were just deallocated from the run. 1445940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans */ 1446940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 144749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (run->nfree == bin_info->nregs) 14488de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 14498de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans else 14508de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_bin_lower_run(arena, chunk, run, bin); 1451e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1452e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (ret); 1453e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1454e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1455e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run == NULL) 1456e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1457e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1458e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->runcur = run; 1459e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1460e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->runcur->nfree > 0); 1461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 146249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (arena_run_reg_alloc(bin->runcur, bin_info)); 1463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 146586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evansvoid 14667372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansarena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, 14677372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans uint64_t prof_accumbytes) 1468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, nfill; 1470e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1471e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1472e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr; 1473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1474e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tbin->ncached == 0); 1475e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 147688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans if (config_prof && arena_prof_accum(arena, prof_accumbytes)) 147788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans prof_idump(); 1478e69bee01de62b56d3e585042d341743239568043Jason Evans bin = &arena->bins[binind]; 1479e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 14801dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 14811dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans tbin->lg_fill_div); i < nfill; i++) { 1482e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 148349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1484e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1485e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ptr = arena_bin_malloc_hard(arena, bin); 14863fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (ptr == NULL) 1487e476f8a161d445211fd6e54fe370275196e66bcbJason Evans break; 1488122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (config_fill && opt_junk) { 1489122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_alloc_junk_small(ptr, &arena_bin_info[binind], 1490122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans true); 1491122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 14929c43c13a35220c10d97a886616899189daceb359Jason Evans /* Insert such that low regions get used first. */ 14939c43c13a35220c10d97a886616899189daceb359Jason Evans tbin->avail[nfill - 1 - i] = ptr; 1494e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 14957372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 14967372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated += i * arena_bin_info[binind].reg_size; 14977372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nmalloc += i; 14987372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 14997372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nfills++; 15007372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tbin->tstats.nrequests = 0; 15017372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 150286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1503e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->ncached = i; 1504e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1506122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid 1507122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansarena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) 1508122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans{ 1509122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 1510122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (zero) { 1511122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t redzone_size = bin_info->redzone_size; 1512122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, 1513122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans redzone_size); 1514122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, 1515122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans redzone_size); 1516122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 1517122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, 1518122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval); 1519122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1520122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans} 1521122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 15220d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#ifdef JEMALLOC_JET 15230d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#undef arena_redzone_corruption 15240d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption_impl) 15250d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#endif 15260d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansstatic void 15270d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansarena_redzone_corruption(void *ptr, size_t usize, bool after, 15280d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans size_t offset, uint8_t byte) 15290d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans{ 15300d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans 15310d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans malloc_printf("<jemalloc>: Corrupt redzone %zu byte%s %s %p " 15320d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s", 15330d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans after ? "after" : "before", ptr, usize, byte); 15340d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans} 15350d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#ifdef JEMALLOC_JET 15360d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#undef arena_redzone_corruption 15376b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) 15386b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansarena_redzone_corruption_t *arena_redzone_corruption = 15396b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans JEMALLOC_N(arena_redzone_corruption_impl); 15400d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#endif 15410d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans 15420d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansstatic void 15430d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansarena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) 1544122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans{ 1545122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t size = bin_info->reg_size; 1546122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t redzone_size = bin_info->redzone_size; 1547122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t i; 1548122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bool error = false; 1549122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 1550122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans for (i = 1; i <= redzone_size; i++) { 15510d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); 15520d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans if (*byte != 0xa5) { 1553122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans error = true; 15540d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans arena_redzone_corruption(ptr, size, false, i, *byte); 15550d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans if (reset) 15560d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans *byte = 0xa5; 1557122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1558122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1559122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans for (i = 0; i < redzone_size; i++) { 15600d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); 15610d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans if (*byte != 0xa5) { 1562122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans error = true; 15630d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans arena_redzone_corruption(ptr, size, true, i, *byte); 15640d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans if (reset) 15650d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans *byte = 0xa5; 1566122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1567122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1568122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (opt_abort && error) 1569122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans abort(); 15700d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans} 15710d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans 15726b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 15736b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#undef arena_dalloc_junk_small 15746b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small_impl) 15756b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 15760d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansvoid 15770d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansarena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) 15780d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans{ 15790d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans size_t redzone_size = bin_info->redzone_size; 1580122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 15810d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans arena_redzones_validate(ptr, bin_info, false); 1582122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, 1583122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval); 1584122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans} 15856b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 15866b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#undef arena_dalloc_junk_small 15876b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) 15886b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansarena_dalloc_junk_small_t *arena_dalloc_junk_small = 15896b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans JEMALLOC_N(arena_dalloc_junk_small_impl); 15906b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 1591122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 15920d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansvoid 15930d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansarena_quarantine_junk_small(void *ptr, size_t usize) 15940d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans{ 15950d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans size_t binind; 15960d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans arena_bin_info_t *bin_info; 15970d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans cassert(config_fill); 15980d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans assert(opt_junk); 15990d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans assert(opt_quarantine); 16000d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans assert(usize <= SMALL_MAXCLASS); 16010d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans 16020d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans binind = SMALL_SIZE2BIN(usize); 16030d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans bin_info = &arena_bin_info[binind]; 16040d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans arena_redzones_validate(ptr, bin_info, true); 16050d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans} 16060d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans 1607e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1608e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_small(arena_t *arena, size_t size, bool zero) 1609e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1610e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1611e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1612e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1613e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 1614e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 161541ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans binind = SMALL_SIZE2BIN(size); 1616b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 1617e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 161849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size = arena_bin_info[binind].reg_size; 1619e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 162086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 1621e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 162249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1623e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1624e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_hard(arena, bin); 1625e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1626e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 162786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1628e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1629e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1630e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 16317372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 16327372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated += size; 16337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nmalloc++; 16347372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nrequests++; 16357372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 163686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 163788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans if (config_prof && isthreaded == false && arena_prof_accum(arena, size)) 163888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans prof_idump(); 1639e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1640e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 16417372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill) { 1642122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (opt_junk) { 1643122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_alloc_junk_small(ret, 1644122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans &arena_bin_info[binind], false); 1645122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else if (opt_zero) 16467372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0, size); 16477372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1648dda90f59e2b67903668a2799970f64df163e9ccfJason Evans VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1649122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 1650122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (config_fill && opt_junk) { 1651122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_alloc_junk_small(ret, &arena_bin_info[binind], 1652122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans true); 1653122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1654122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1655e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1656122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1657e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1658e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1659e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1660e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1661e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1662e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_large(arena_t *arena, size_t size, bool zero) 1663e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1664e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 166588c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans UNUSED bool idump; 1666e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1667e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1668e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = PAGE_CEILING(size); 1669e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1670aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ret = (void *)arena_run_alloc_large(arena, size, zero); 1671e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1672e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1673e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1674e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 16757372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 16767372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 16777372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 16787372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 1679ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1680ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1681ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1682e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 16837372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) 168488c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans idump = arena_prof_accum_locked(arena, size); 1685e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 168688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans if (config_prof && idump) 168788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans prof_idump(); 1688e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1689e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 16907372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill) { 16917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (opt_junk) 16927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0xa5, size); 16937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans else if (opt_zero) 16947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0, size); 16957372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1696e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1697e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1698e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1699e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1700e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1701e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Only handles large allocations that require more than page alignment. */ 1702e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 17035ff709c264e52651de25b788692c62ff1f6f389cJason Evansarena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) 1704e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1705e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 17065ff709c264e52651de25b788692c62ff1f6f389cJason Evans size_t alloc_size, leadsize, trailsize; 17075ff709c264e52651de25b788692c62ff1f6f389cJason Evans arena_run_t *run; 1708e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1709e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1710e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 171193443689a4367cc6fe3de1c9e918adc13d8f9100Jason Evans 171293443689a4367cc6fe3de1c9e918adc13d8f9100Jason Evans alignment = PAGE_CEILING(alignment); 17135ff709c264e52651de25b788692c62ff1f6f389cJason Evans alloc_size = size + alignment - PAGE; 1714e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1715e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1716aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run = arena_run_alloc_large(arena, alloc_size, false); 17175ff709c264e52651de25b788692c62ff1f6f389cJason Evans if (run == NULL) { 1718e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1719e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1720e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 17215ff709c264e52651de25b788692c62ff1f6f389cJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1722e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17235ff709c264e52651de25b788692c62ff1f6f389cJason Evans leadsize = ALIGNMENT_CEILING((uintptr_t)run, alignment) - 17245ff709c264e52651de25b788692c62ff1f6f389cJason Evans (uintptr_t)run; 17255ff709c264e52651de25b788692c62ff1f6f389cJason Evans assert(alloc_size >= leadsize + size); 17265ff709c264e52651de25b788692c62ff1f6f389cJason Evans trailsize = alloc_size - leadsize - size; 17275ff709c264e52651de25b788692c62ff1f6f389cJason Evans ret = (void *)((uintptr_t)run + leadsize); 17285ff709c264e52651de25b788692c62ff1f6f389cJason Evans if (leadsize != 0) { 17295ff709c264e52651de25b788692c62ff1f6f389cJason Evans arena_run_trim_head(arena, chunk, run, alloc_size, alloc_size - 17305ff709c264e52651de25b788692c62ff1f6f389cJason Evans leadsize); 17315ff709c264e52651de25b788692c62ff1f6f389cJason Evans } 17325ff709c264e52651de25b788692c62ff1f6f389cJason Evans if (trailsize != 0) { 17335ff709c264e52651de25b788692c62ff1f6f389cJason Evans arena_run_trim_tail(arena, chunk, ret, size + trailsize, size, 17345ff709c264e52651de25b788692c62ff1f6f389cJason Evans false); 1735e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1736aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_init_large(arena, (arena_run_t *)ret, size, zero); 1737e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17387372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 17397372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 17407372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 17417372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 1742ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1743ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1744ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1745e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1746e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1747e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17487372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && zero == false) { 17498e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (opt_junk) 17508e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset(ret, 0xa5, size); 17518e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans else if (opt_zero) 17528e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset(ret, 0, size); 17538e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 1754e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1755e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1756e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17570b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansvoid 17580b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansarena_prof_promoted(const void *ptr, size_t size) 17590b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans{ 17600b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans arena_chunk_t *chunk; 17610b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t pageind, binind; 17620b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 176378f7352259768f670f8e1f9b000388dd32b62493Jason Evans cassert(config_prof); 17640b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(ptr != NULL); 17650b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1766122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(isalloc(ptr, false) == PAGE); 1767122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(isalloc(ptr, true) == PAGE); 1768b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(size <= SMALL_MAXCLASS); 17690b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 17700b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1771ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 177241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans binind = SMALL_SIZE2BIN(size); 1773b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 1774203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_binind_set(chunk, pageind, binind); 17750b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 1776122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(isalloc(ptr, false) == PAGE); 1777122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(isalloc(ptr, true) == size); 17780b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans} 17796109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1780e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1781088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansarena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 1782e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin) 1783e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1784e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 178519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Dissociate run from bin. */ 1786e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run == bin->runcur) 1787e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 178849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans else { 178949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, bin); 179049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 179149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 179249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (bin_info->nregs != 1) { 179349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 179449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * This block's conditional is necessary because if the 179549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * run only contains one region, then it never gets 179649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * inserted into the non-full runs tree. 179749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 1798e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_remove(bin, run); 179949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 1800e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1801088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans} 1802088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans 1803088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansstatic void 1804088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansarena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1805088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_bin_t *bin) 1806088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans{ 180749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 180849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1809088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans size_t npages, run_ind, past; 1810088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans 1811088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans assert(run != bin->runcur); 1812203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_run_tree_search(&bin->runs, 1813203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapp_get(chunk, ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) 1814203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans == NULL); 181586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 181649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(chunk->arena, run->bin); 181749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 181849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 1819e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&bin->lock); 1820e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /******************************/ 1821ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans npages = bin_info->run_size >> LG_PAGE; 1822ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 182384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans past = (size_t)(PAGE_CEILING((uintptr_t)run + 182484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * 1825122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval - bin_info->redzone_size) - 1826122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (uintptr_t)chunk) >> LG_PAGE); 182786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 182819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 182919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 183019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * If the run was originally clean, and some pages were never touched, 183119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * trim the clean pages before deallocating the dirty portion of the 183219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run. 183319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 183430fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_dirty_get(chunk, run_ind) == 183530fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_mapbits_dirty_get(chunk, run_ind+npages-1)); 1836203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind < 1837203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans npages) { 183830fe12b866edbc2cf9aaef299063b392ea125aacJason Evans /* Trim clean pages. Convert to large run beforehand. */ 183930fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(npages > 0); 1840d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, run_ind, bin_info->run_size, 0); 1841d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); 1842ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), 1843ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans ((past - run_ind) << LG_PAGE), false); 1844940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* npages = past - run_ind; */ 18451e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans } 1846e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_dalloc(arena, run, true, false); 184786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 1848e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /****************************/ 1849e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&bin->lock); 18507372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 18517372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.curruns--; 1852e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1853e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1854940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansstatic void 1855940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansarena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1856940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_bin_t *bin) 1857e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1858e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18598de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* 1860e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * Make sure that if bin->runcur is non-NULL, it refers to the lowest 1861e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * non-full run. It is okay to NULL runcur out rather than proactively 1862e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * keeping it pointing at the lowest non-full run. 18638de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans */ 1864e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if ((uintptr_t)run < (uintptr_t)bin->runcur) { 18658de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* Switch runcur. */ 1866e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (bin->runcur->nfree > 0) 1867e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_insert(bin, bin->runcur); 18688de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans bin->runcur = run; 1869e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (config_stats) 1870e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans bin->stats.reruns++; 1871e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans } else 1872e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_insert(bin, run); 1873940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans} 1874940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1875940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansvoid 1876203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1877940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_chunk_map_t *mapelm) 1878940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans{ 1879940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t pageind; 1880940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_t *run; 1881940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_bin_t *bin; 18828b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey arena_bin_info_t *bin_info; 18838b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey size_t size, binind; 1884940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1885ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1886940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1887203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1888940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans bin = run->bin; 1889f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer binind = arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)); 18908b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey bin_info = &arena_bin_info[binind]; 18917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill || config_stats) 18927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size = bin_info->reg_size; 1893940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 18947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk) 1895122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_dalloc_junk_small(ptr, bin_info); 1896940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1897940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_reg_dalloc(run, ptr); 189849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (run->nfree == bin_info->nregs) { 1899088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_dissociate_bin_run(chunk, run, bin); 19008de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 1901088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans } else if (run->nfree == 1 && run != bin->runcur) 19028de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_bin_lower_run(arena, chunk, run, bin); 1903e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 19057372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated -= size; 19067372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.ndalloc++; 19077372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1908e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1909e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1910e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1911203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1912203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind, arena_chunk_map_t *mapelm) 1913203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 1914203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_run_t *run; 1915203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_bin_t *bin; 1916203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1917203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1918203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1919203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans bin = run->bin; 1920203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans malloc_mutex_lock(&bin->lock); 1921203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_bin_locked(arena, chunk, ptr, mapelm); 1922203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans malloc_mutex_unlock(&bin->lock); 1923203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 1924203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1925203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid 1926203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1927203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind) 1928203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 1929203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_chunk_map_t *mapelm; 1930203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1931203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (config_debug) { 193280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans /* arena_ptr_small_binind_get() does extra sanity checking. */ 193380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 193480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind)) != BININD_INVALID); 1935203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 1936203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapelm = arena_mapp_get(chunk, pageind); 1937203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); 1938203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 1939e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19406b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 19416b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#undef arena_dalloc_junk_large 19426b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl) 19436b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 19446b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansstatic void 19456b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansarena_dalloc_junk_large(void *ptr, size_t usize) 19466b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans{ 19476b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans 19486b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans if (config_fill && opt_junk) 19496b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans memset(ptr, 0x5a, usize); 19506b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans} 19516b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 19526b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#undef arena_dalloc_junk_large 19536b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) 19546b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansarena_dalloc_junk_large_t *arena_dalloc_junk_large = 19556b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans JEMALLOC_N(arena_dalloc_junk_large_impl); 19566b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 19576b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans 1958e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1959203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1960e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 196113668262d17fb5950e2441bc9d56a15db9c93877Jason Evans 19627372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill || config_stats) { 1963ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 19646b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans size_t usize = arena_mapbits_large_size_get(chunk, pageind); 1965e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19666b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans arena_dalloc_junk_large(ptr, usize); 19677372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 19687372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 19696b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans arena->stats.allocated_large -= usize; 19706b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans arena->stats.lstats[(usize >> LG_PAGE) - 1].ndalloc++; 19716b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans arena->stats.lstats[(usize >> LG_PAGE) - 1].curruns--; 19727372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1973e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1974e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1975e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_dalloc(arena, (arena_run_t *)ptr, true, false); 1976e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1977e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1978203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid 1979203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1980203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 1981203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1982203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans malloc_mutex_lock(&arena->lock); 1983203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_large_locked(arena, chunk, ptr); 1984203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans malloc_mutex_unlock(&arena->lock); 1985203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 1986203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1987e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1988e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 19898e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t oldsize, size_t size) 1990e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1991e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1992e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size < oldsize); 1993e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1994e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1995e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Shrink the run, and make trailing pages available for other 1996e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocations. 1997e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1998e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1999e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 2000e476f8a161d445211fd6e54fe370275196e66bcbJason Evans true); 20017372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 20027372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 20037372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= oldsize; 2004ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 2005ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 20067372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 20077372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 20087372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 20097372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 2010ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 2011ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 2012ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 2013e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2014e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 2015e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2016e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2017e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 2018e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 20198e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t oldsize, size_t size, size_t extra, bool zero) 2020e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2021ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2022ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t npages = oldsize >> LG_PAGE; 20238e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t followsize; 2024e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2025203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(oldsize == arena_mapbits_large_size_get(chunk, pageind)); 2026e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2027e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to extend the run. */ 20288e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans assert(size + extra > oldsize); 2029e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 20307393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if (pageind + npages < chunk_npages && 2031203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && 2032203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (followsize = arena_mapbits_unallocated_size_get(chunk, 2033203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+npages)) >= size - oldsize) { 2034e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2035e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The next run is available and sufficiently large. Split the 2036e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * following run, then merge the first part with the existing 2037e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocation. 2038e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2039940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t flag_dirty; 20408e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t splitsize = (oldsize + followsize <= size + extra) 20418e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ? followsize : size + extra - oldsize; 2042aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_large(arena, (arena_run_t *)((uintptr_t)chunk + 2043aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ((pageind+npages) << LG_PAGE)), splitsize, zero); 2044e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2045088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans size = oldsize + splitsize; 2046ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans npages = size >> LG_PAGE; 2047e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2048940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* 2049940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * Mark the extended run as dirty if either portion of the run 2050940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * was dirty before allocation. This is rather pedantic, 2051940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * because there's not actually any sequence of events that 2052940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * could cause the resulting run to be passed to 2053940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * arena_run_dalloc() with the dirty argument set to false 2054940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * (which is when dirty flag consistency would really matter). 2055940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans */ 2056203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | 2057203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_dirty_get(chunk, pageind+npages-1); 2058203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_set(chunk, pageind, size, flag_dirty); 2059203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty); 2060990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans 20617372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 20627372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 20637372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= oldsize; 2064203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 2065203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 20667372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 20677372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 20687372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 20697372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 2070ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 2071203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 2072ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 2073940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 2074e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 2075e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2076e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2077e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 2078e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2079e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2080e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2081e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 20826b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 20836b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#undef arena_ralloc_junk_large 20846b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large_impl) 20856b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 20866b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansstatic void 20876b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansarena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) 20886b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans{ 20896b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans 20906b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans if (config_fill && opt_junk) { 20916b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans memset((void *)((uintptr_t)ptr + usize), 0x5a, 20926b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans old_usize - usize); 20936b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans } 20946b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans} 20956b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 20966b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#undef arena_ralloc_junk_large 20976b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) 20986b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansarena_ralloc_junk_large_t *arena_ralloc_junk_large = 20996b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans JEMALLOC_N(arena_ralloc_junk_large_impl); 21006b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 21016b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans 2102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 2103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Try to resize a large allocation, in order to avoid copying. This will 2104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * always fail if growing an object, and the following run is already in use. 2105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2106e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 21078e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, 21088e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 2109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t psize; 2111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 21128e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize = PAGE_CEILING(size + extra); 2113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize == oldsize) { 2114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Same size class. */ 2115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 2117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 2118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 2119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 2122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize < oldsize) { 2124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Fill before shrinking in order avoid a race. */ 21256b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans arena_ralloc_junk_large(ptr, oldsize, psize); 21268e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, 21278e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize); 2128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 2130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 21318e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans oldsize, PAGE_CEILING(size), 21328e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize - PAGE_CEILING(size), zero); 21336b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans if (config_fill && ret == false && zero == false) { 21346b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans if (opt_junk) { 21356b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans memset((void *)((uintptr_t)ptr + 21366b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans oldsize), 0xa5, isalloc(ptr, 21376b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans config_prof) - oldsize); 21386b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans } else if (opt_zero) { 21396b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans memset((void *)((uintptr_t)ptr + 21406b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans oldsize), 0, isalloc(ptr, 21416b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans config_prof) - oldsize); 21426b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans } 2143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 2145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2149b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evansbool 21508e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, 21518e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 2152e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 21548e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* 21558e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * Avoid moving the allocation if the size class can be left the same. 21568e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans */ 2157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= arena_maxclass) { 2158b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (oldsize <= SMALL_MAXCLASS) { 215949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size 216049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans == oldsize); 2161b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if ((size + extra <= SMALL_MAXCLASS && 216241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans SMALL_SIZE2BIN(size + extra) == 216341ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && 21646e62984ef6ca4312cf0a2e49ea2cc38feb94175bJason Evans size + extra >= oldsize)) 2165b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evans return (false); 2166e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 2167e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 2168b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (size + extra > SMALL_MAXCLASS) { 21698e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (arena_ralloc_large(ptr, oldsize, size, 21708e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans extra, zero) == false) 2171b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evans return (false); 2172e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2175e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 21768e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Reallocation would require a move. */ 2177b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evans return (true); 21788e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans} 21798e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 21808e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansvoid * 2181609ae595f0358157b19311b0f9f9591db7cee705Jason Evansarena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, 2182609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, 2183609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bool try_tcache_dalloc) 21848e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans{ 21858e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ret; 21868e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t copysize; 21878e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 21888e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Try to avoid moving the allocation. */ 2189b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evans if (arena_ralloc_no_move(ptr, oldsize, size, extra, zero) == false) 2190b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evans return (ptr); 21918e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 2192e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 21938e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * size and oldsize are different enough that we need to move the 21948e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * object. In that case, fall back to allocating new space and 21958e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * copying. 2196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 219738d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (alignment != 0) { 21985ff709c264e52651de25b788692c62ff1f6f389cJason Evans size_t usize = sa2u(size + extra, alignment); 219938d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (usize == 0) 220038d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans return (NULL); 2201d82a5e6a34f20698ab9368bb2b4953b81d175552Jason Evans ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); 220238d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans } else 2203609ae595f0358157b19311b0f9f9591db7cee705Jason Evans ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc); 2204e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 22058e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == NULL) { 22068e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (extra == 0) 22078e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 22088e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Try again, this time without extra. */ 220938d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (alignment != 0) { 22105ff709c264e52651de25b788692c62ff1f6f389cJason Evans size_t usize = sa2u(size, alignment); 221138d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (usize == 0) 221238d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans return (NULL); 2213d82a5e6a34f20698ab9368bb2b4953b81d175552Jason Evans ret = ipalloct(usize, alignment, zero, try_tcache_alloc, 2214609ae595f0358157b19311b0f9f9591db7cee705Jason Evans arena); 221538d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans } else 2216609ae595f0358157b19311b0f9f9591db7cee705Jason Evans ret = arena_malloc(arena, size, zero, try_tcache_alloc); 22178e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 22188e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == NULL) 22198e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 22208e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 22218e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 22228e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ 22238e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 22248e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* 22258e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * Copy at most size bytes (not size+extra), since the caller has no 22268e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * expectation that the extra bytes will be reliably preserved. 22278e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans */ 2228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans copysize = (size < oldsize) ? size : oldsize; 2229f54166e7ef5313c3b5c773cbb0ca2af95f5a15aeJason Evans VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); 2230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(ret, ptr, copysize); 2231d82a5e6a34f20698ab9368bb2b4953b81d175552Jason Evans iqalloct(ptr, try_tcache_dalloc); 2232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 2233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2235609ae595f0358157b19311b0f9f9591db7cee705Jason Evansdss_prec_t 2236609ae595f0358157b19311b0f9f9591db7cee705Jason Evansarena_dss_prec_get(arena_t *arena) 2237609ae595f0358157b19311b0f9f9591db7cee705Jason Evans{ 2238609ae595f0358157b19311b0f9f9591db7cee705Jason Evans dss_prec_t ret; 2239609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2240609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&arena->lock); 2241609ae595f0358157b19311b0f9f9591db7cee705Jason Evans ret = arena->dss_prec; 2242609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&arena->lock); 2243609ae595f0358157b19311b0f9f9591db7cee705Jason Evans return (ret); 2244609ae595f0358157b19311b0f9f9591db7cee705Jason Evans} 2245609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2246609ae595f0358157b19311b0f9f9591db7cee705Jason Evansvoid 2247609ae595f0358157b19311b0f9f9591db7cee705Jason Evansarena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) 2248609ae595f0358157b19311b0f9f9591db7cee705Jason Evans{ 2249609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2250609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&arena->lock); 2251609ae595f0358157b19311b0f9f9591db7cee705Jason Evans arena->dss_prec = dss_prec; 2252609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&arena->lock); 2253609ae595f0358157b19311b0f9f9591db7cee705Jason Evans} 2254609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2255609ae595f0358157b19311b0f9f9591db7cee705Jason Evansvoid 2256609ae595f0358157b19311b0f9f9591db7cee705Jason Evansarena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, 2257609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 2258609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_large_stats_t *lstats) 2259609ae595f0358157b19311b0f9f9591db7cee705Jason Evans{ 2260609ae595f0358157b19311b0f9f9591db7cee705Jason Evans unsigned i; 2261609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2262609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&arena->lock); 2263609ae595f0358157b19311b0f9f9591db7cee705Jason Evans *dss = dss_prec_names[arena->dss_prec]; 2264609ae595f0358157b19311b0f9f9591db7cee705Jason Evans *nactive += arena->nactive; 2265609ae595f0358157b19311b0f9f9591db7cee705Jason Evans *ndirty += arena->ndirty; 2266609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2267609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->mapped += arena->stats.mapped; 2268609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->npurge += arena->stats.npurge; 2269609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->nmadvise += arena->stats.nmadvise; 2270609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->purged += arena->stats.purged; 2271609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->allocated_large += arena->stats.allocated_large; 2272609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->nmalloc_large += arena->stats.nmalloc_large; 2273609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->ndalloc_large += arena->stats.ndalloc_large; 2274609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->nrequests_large += arena->stats.nrequests_large; 2275609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2276609ae595f0358157b19311b0f9f9591db7cee705Jason Evans for (i = 0; i < nlclasses; i++) { 2277609ae595f0358157b19311b0f9f9591db7cee705Jason Evans lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 2278609ae595f0358157b19311b0f9f9591db7cee705Jason Evans lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 2279609ae595f0358157b19311b0f9f9591db7cee705Jason Evans lstats[i].nrequests += arena->stats.lstats[i].nrequests; 2280609ae595f0358157b19311b0f9f9591db7cee705Jason Evans lstats[i].curruns += arena->stats.lstats[i].curruns; 2281609ae595f0358157b19311b0f9f9591db7cee705Jason Evans } 2282609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&arena->lock); 2283609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2284609ae595f0358157b19311b0f9f9591db7cee705Jason Evans for (i = 0; i < NBINS; i++) { 2285609ae595f0358157b19311b0f9f9591db7cee705Jason Evans arena_bin_t *bin = &arena->bins[i]; 2286609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2287609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&bin->lock); 2288609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].allocated += bin->stats.allocated; 2289609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nmalloc += bin->stats.nmalloc; 2290609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].ndalloc += bin->stats.ndalloc; 2291609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nrequests += bin->stats.nrequests; 2292609ae595f0358157b19311b0f9f9591db7cee705Jason Evans if (config_tcache) { 2293609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nfills += bin->stats.nfills; 2294609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nflushes += bin->stats.nflushes; 2295609ae595f0358157b19311b0f9f9591db7cee705Jason Evans } 2296609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nruns += bin->stats.nruns; 2297609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].reruns += bin->stats.reruns; 2298609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].curruns += bin->stats.curruns; 2299609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&bin->lock); 2300609ae595f0358157b19311b0f9f9591db7cee705Jason Evans } 2301609ae595f0358157b19311b0f9f9591db7cee705Jason Evans} 2302609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2303e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 2304e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_new(arena_t *arena, unsigned ind) 2305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 2307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 2308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 23096109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena->ind = ind; 2310597632be188d2bcc135dad2145cc46ef44897aadJason Evans arena->nthreads = 0; 23116109fe07a14b7a619365977d9523db9f8b333792Jason Evans 2312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (malloc_mutex_init(&arena->lock)) 2313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 23157372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 23167372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(&arena->stats, 0, sizeof(arena_stats_t)); 23177372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats = 23187372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans (malloc_large_stats_t *)base_alloc(nlclasses * 23197372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans sizeof(malloc_large_stats_t)); 23207372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (arena->stats.lstats == NULL) 23217372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans return (true); 23227372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(arena->stats.lstats, 0, nlclasses * 23237372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans sizeof(malloc_large_stats_t)); 23247372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_tcache) 23257372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ql_new(&arena->tcache_ql); 23267372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 2327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 23287372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) 23297372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->prof_accumbytes = 0; 2330d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 2331609ae595f0358157b19311b0f9f9591db7cee705Jason Evans arena->dss_prec = chunk_dss_prec_get(); 2332609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize chunks. */ 2334e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_new(&arena->chunks_dirty); 2335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 2336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive = 0; 2338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty = 0; 2339799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory = 0; 2340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2341e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_new(&arena->runs_avail); 2342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize bins. */ 2344b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans for (i = 0; i < NBINS; i++) { 2345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 234686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 234786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 2348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 2349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 23507372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 23517372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2355e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2356e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 235749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 235849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Calculate bin_info->run_size such that it meets the following constraints: 235949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 236049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) bin_info->run_size >= min_run_size 236149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) bin_info->run_size <= arena_maxclass 236249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 236347e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans * *) bin_info->nregs <= RUN_MAXREGS 236449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 236584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also 236684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * calculated here, since these settings are all interdependent. 236749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 236849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstatic size_t 236949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansbin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) 237049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 2371122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t pad_size; 237249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t try_run_size, good_run_size; 237349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_nregs, good_nregs; 237449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_hdr_size, good_hdr_size; 237584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans uint32_t try_bitmap_offset, good_bitmap_offset; 237649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_ctx0_offset, good_ctx0_offset; 2377122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans uint32_t try_redzone0_offset, good_redzone0_offset; 237849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2379ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans assert(min_run_size >= PAGE); 238049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(min_run_size <= arena_maxclass); 238149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 238249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 2383122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Determine redzone size based on minimum alignment and minimum 2384122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * redzone size. Add padding to the end of the run if it is needed to 2385122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * align the regions. The padding allows each redzone to be half the 2386122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * minimum alignment; without the padding, each redzone would have to 2387122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * be twice as large in order to maintain alignment. 2388122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans */ 2389122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (config_fill && opt_redzone) { 2390122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t align_min = ZU(1) << (ffs(bin_info->reg_size) - 1); 2391122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (align_min <= REDZONE_MINSIZE) { 2392122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->redzone_size = REDZONE_MINSIZE; 2393122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans pad_size = 0; 2394122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 2395122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->redzone_size = align_min >> 1; 2396122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans pad_size = bin_info->redzone_size; 2397122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 2398122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 2399122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->redzone_size = 0; 2400122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans pad_size = 0; 2401122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 2402122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval = bin_info->reg_size + 2403122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (bin_info->redzone_size << 1); 2404122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 2405122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* 240649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Calculate known-valid settings before entering the run_size 240749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * expansion loop, so that the first part of the loop always copies 240849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * valid settings. 240949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 241049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * The do..while loop iteratively reduces the number of regions until 241149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * the run header and the regions no longer overlap. A closed formula 241249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * would be quite messy, since there is an interdependency between the 241349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * header's mask length and the number of regions. 241449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 241549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_run_size = min_run_size; 2416122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / 2417122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval) 241849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 241947e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans if (try_nregs > RUN_MAXREGS) { 242047e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans try_nregs = RUN_MAXREGS 242147e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 242247e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans } 242349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 242449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs--; 242549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = sizeof(arena_run_t); 242684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Pad to a long boundary. */ 242784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size = LONG_CEILING(try_hdr_size); 242884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_bitmap_offset = try_hdr_size; 242984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Add space for bitmap. */ 243084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size += bitmap_size(try_nregs); 24317372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof && opt_prof && prof_promote == false) { 243249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Pad to a quantum boundary. */ 243349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 243449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = try_hdr_size; 243549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Add space for one (prof_ctx_t *) per region. */ 243649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size += try_nregs * sizeof(prof_ctx_t *); 243749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } else 243849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = 0; 2439122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans try_redzone0_offset = try_run_size - (try_nregs * 2440122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval) - pad_size; 2441122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } while (try_hdr_size > try_redzone0_offset); 244249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 244349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* run_size expansion loop. */ 244449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 244549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 244649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Copy valid settings before trying more aggressive settings. 244749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 244849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_run_size = try_run_size; 244949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_nregs = try_nregs; 245049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_hdr_size = try_hdr_size; 245184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans good_bitmap_offset = try_bitmap_offset; 245249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_ctx0_offset = try_ctx0_offset; 2453122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans good_redzone0_offset = try_redzone0_offset; 245449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 245549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Try more aggressive settings. */ 2456ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans try_run_size += PAGE; 2457122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) / 2458122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval) 245949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 246047e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans if (try_nregs > RUN_MAXREGS) { 246147e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans try_nregs = RUN_MAXREGS 246247e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 246347e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans } 246449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 246549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs--; 246649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = sizeof(arena_run_t); 246784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Pad to a long boundary. */ 246884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size = LONG_CEILING(try_hdr_size); 246984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_bitmap_offset = try_hdr_size; 247084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Add space for bitmap. */ 247184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size += bitmap_size(try_nregs); 24727372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof && opt_prof && prof_promote == false) { 247349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Pad to a quantum boundary. */ 247449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 247549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = try_hdr_size; 247649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 247749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Add space for one (prof_ctx_t *) per region. 247849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 247949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size += try_nregs * 248049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans sizeof(prof_ctx_t *); 248149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 2482122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans try_redzone0_offset = try_run_size - (try_nregs * 2483122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval) - pad_size; 2484122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } while (try_hdr_size > try_redzone0_offset); 248549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } while (try_run_size <= arena_maxclass 2486122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > 2487122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans RUN_MAX_OVRHD_RELAX 2488122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size 248947e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans && try_nregs < RUN_MAXREGS); 249049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2491122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(good_hdr_size <= good_redzone0_offset); 249249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 249349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Copy final settings. */ 249449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->run_size = good_run_size; 249549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->nregs = good_nregs; 249684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bin_info->bitmap_offset = good_bitmap_offset; 249749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->ctx0_offset = good_ctx0_offset; 2498122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; 2499122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 2500122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs 2501122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * bin_info->reg_interval) + pad_size == bin_info->run_size); 250249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 250349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (good_run_size); 250449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 250549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2506b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansstatic void 250749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansbin_info_init(void) 250849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 250949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 2510ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t prev_run_size = PAGE; 2511b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans 2512b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define SIZE_CLASS(bin, delta, size) \ 2513b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info = &arena_bin_info[bin]; \ 2514b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info->reg_size = size; \ 2515b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ 2516b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 2517b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans SIZE_CLASSES 2518b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef SIZE_CLASS 251949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 252049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2521b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansvoid 2522a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evansarena_boot(void) 2523e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2524a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans size_t header_size; 25257393f44ff025ca67716fc53b68003fd65122fd97Jason Evans unsigned i; 2526e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2527e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2528e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Compute the header size such that it is large enough to contain the 25297393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * page map. The page map is biased to omit entries for the header 25307393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * itself, so some iteration is necessary to compute the map bias. 25317393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 25327393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 1) Compute safe header_size and map_bias values that include enough 25337393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * space for an unbiased page map. 25347393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 2) Refine map_bias based on (1) to omit the header pages in the page 25357393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * map. The resulting map_bias may be one too small. 25367393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 3) Refine map_bias based on (2). The result will be >= the result 25377393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * from (2), and will always be correct. 2538e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 25397393f44ff025ca67716fc53b68003fd65122fd97Jason Evans map_bias = 0; 25407393f44ff025ca67716fc53b68003fd65122fd97Jason Evans for (i = 0; i < 3; i++) { 2541ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans header_size = offsetof(arena_chunk_t, map) + 2542ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); 2543ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) 2544ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans != 0); 25457393f44ff025ca67716fc53b68003fd65122fd97Jason Evans } 25467393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(map_bias > 0); 25477393f44ff025ca67716fc53b68003fd65122fd97Jason Evans 2548ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena_maxclass = chunksize - (map_bias << LG_PAGE); 2549a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans 2550b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info_init(); 2551e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 25524e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 25534e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 25544e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansarena_prefork(arena_t *arena) 25554e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 25564e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans unsigned i; 25574e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 25584e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_prefork(&arena->lock); 25594e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans for (i = 0; i < NBINS; i++) 25604e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_prefork(&arena->bins[i].lock); 25614e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 25624e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 25634e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 25644e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansarena_postfork_parent(arena_t *arena) 25654e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 25664e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans unsigned i; 25674e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 25684e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans for (i = 0; i < NBINS; i++) 25694e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_parent(&arena->bins[i].lock); 25704e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_parent(&arena->lock); 25714e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 25724e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 25734e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 25744e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansarena_postfork_child(arena_t *arena) 25754e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 25764e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans unsigned i; 25774e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 25784e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans for (i = 0; i < NBINS; i++) 25794e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_child(&arena->bins[i].lock); 25804e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_child(&arena->lock); 25814e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 2582