1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_ARENA_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evansssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 8b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansarena_bin_info_t arena_bin_info[NBINS]; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 10da99e31105eb709ef4ec8a120b115c32a6b9723aMike HommeyJEMALLOC_ALIGNED(CACHELINE) 113541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evansconst uint32_t small_bin2size_tab[NBINS] = { 128d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define B2S_bin_yes(size) \ 13021136ce4db79f50031a1fd5dd751891888fbc7bBen Maurer size, 148d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define B2S_bin_no(size) 158d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 168d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans B2S_bin_##bin((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)) 17021136ce4db79f50031a1fd5dd751891888fbc7bBen Maurer SIZE_CLASSES 188d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef B2S_bin_yes 198d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef B2S_bin_no 208d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef SC 21021136ce4db79f50031a1fd5dd751891888fbc7bBen Maurer}; 22021136ce4db79f50031a1fd5dd751891888fbc7bBen Maurer 23021136ce4db79f50031a1fd5dd751891888fbc7bBen MaurerJEMALLOC_ALIGNED(CACHELINE) 243541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evansconst uint8_t small_size2bin_tab[] = { 258d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define S2B_3(i) i, 268d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define S2B_4(i) S2B_3(i) S2B_3(i) 278d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define S2B_5(i) S2B_4(i) S2B_4(i) 288d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define S2B_6(i) S2B_5(i) S2B_5(i) 298d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define S2B_7(i) S2B_6(i) S2B_6(i) 308d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define S2B_8(i) S2B_7(i) S2B_7(i) 318d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define S2B_9(i) S2B_8(i) S2B_8(i) 328d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define S2B_no(i) 338d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 348d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans S2B_##lg_delta_lookup(index) 35b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans SIZE_CLASSES 368d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef S2B_3 378d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef S2B_4 388d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef S2B_5 398d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef S2B_6 408d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef S2B_7 41e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_8 428d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef S2B_9 438d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef S2B_no 448d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef SC 45b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans}; 46e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 48aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans/* 49aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Function prototypes for static functions that are referenced prior to 50aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * definition. 51aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 52aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 536005f0710cf07d60659d91b20b7ff5592d310027Jason Evansstatic void arena_purge(arena_t *arena, bool all); 54e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, 55e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans bool cleaned); 56e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 57e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, arena_bin_t *bin); 58940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansstatic void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 59940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_t *run, arena_bin_t *bin); 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 61e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 62e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 63f9ff60346d7c25ad653ea062e496a5d0864233b2Ben MaurerJEMALLOC_INLINE_C size_t 64f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurerarena_mapelm_to_pageind(arena_chunk_map_t *mapelm) 65f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer{ 66f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer uintptr_t map_offset = 67f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer CHUNK_ADDR2OFFSET(mapelm) - offsetof(arena_chunk_t, map); 68f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer 69f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer return ((map_offset / sizeof(arena_chunk_map_t)) + map_bias); 70f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer} 71f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer 72f9ff60346d7c25ad653ea062e496a5d0864233b2Ben MaurerJEMALLOC_INLINE_C size_t 73f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurerarena_mapelm_to_bits(arena_chunk_map_t *mapelm) 74f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer{ 75f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer 76f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer return (mapelm->bits); 77f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer} 78f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm = (uintptr_t)a; 83e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t b_mapelm = (uintptr_t)b; 84e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 85e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(a != NULL); 86e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(b != NULL); 87e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 89e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 90e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 91f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansrb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, 937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans u.rb_link, arena_run_comp) 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 95e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 98e476f8a161d445211fd6e54fe370275196e66bcbJason Evans int ret; 99f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer size_t a_size; 100f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer size_t b_size = arena_mapelm_to_bits(b) & ~PAGE_MASK; 101f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer uintptr_t a_mapelm = (uintptr_t)a; 102f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer uintptr_t b_mapelm = (uintptr_t)b; 103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 104f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer if (a_mapelm & CHUNK_MAP_KEY) 105f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer a_size = a_mapelm & ~PAGE_MASK; 106f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer else 107f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer a_size = arena_mapelm_to_bits(a) & ~PAGE_MASK; 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 109f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer ret = (a_size > b_size) - (a_size < b_size); 110f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer if (ret == 0 && (!(a_mapelm & CHUNK_MAP_KEY))) 111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 116f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 1177372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansrb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, 1187372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans u.rb_link, arena_avail_comp) 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 120e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline int 121e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_chunk_dirty_comp(arena_chunk_t *a, arena_chunk_t *b) 122e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 123e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 124e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(a != NULL); 125e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(b != NULL); 126e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 127e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 128abf6739317742ca4677bf885178984a8757ee14aJason Evans * Short-circuit for self comparison. The following comparison code 129abf6739317742ca4677bf885178984a8757ee14aJason Evans * would come to the same result, but at the cost of executing the slow 130abf6739317742ca4677bf885178984a8757ee14aJason Evans * path. 131abf6739317742ca4677bf885178984a8757ee14aJason Evans */ 132abf6739317742ca4677bf885178984a8757ee14aJason Evans if (a == b) 133abf6739317742ca4677bf885178984a8757ee14aJason Evans return (0); 134abf6739317742ca4677bf885178984a8757ee14aJason Evans 135abf6739317742ca4677bf885178984a8757ee14aJason Evans /* 136e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Order such that chunks with higher fragmentation are "less than" 137abf6739317742ca4677bf885178984a8757ee14aJason Evans * those with lower fragmentation -- purging order is from "least" to 138abf6739317742ca4677bf885178984a8757ee14aJason Evans * "greatest". Fragmentation is measured as: 139e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 140e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * mean current avail run size 141e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * -------------------------------- 142e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * mean defragmented avail run size 143e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 144e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * navail 145e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * ----------- 146e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * nruns_avail nruns_avail-nruns_adjac 147e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * = ========================= = ----------------------- 148e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * navail nruns_avail 149e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * ----------------------- 150e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * nruns_avail-nruns_adjac 151e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 152e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * The following code multiplies away the denominator prior to 153e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * comparison, in order to avoid division. 154e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 155e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 156abf6739317742ca4677bf885178984a8757ee14aJason Evans { 157abf6739317742ca4677bf885178984a8757ee14aJason Evans size_t a_val = (a->nruns_avail - a->nruns_adjac) * 158abf6739317742ca4677bf885178984a8757ee14aJason Evans b->nruns_avail; 159abf6739317742ca4677bf885178984a8757ee14aJason Evans size_t b_val = (b->nruns_avail - b->nruns_adjac) * 160abf6739317742ca4677bf885178984a8757ee14aJason Evans a->nruns_avail; 161abf6739317742ca4677bf885178984a8757ee14aJason Evans 162abf6739317742ca4677bf885178984a8757ee14aJason Evans if (a_val < b_val) 163abf6739317742ca4677bf885178984a8757ee14aJason Evans return (1); 164abf6739317742ca4677bf885178984a8757ee14aJason Evans if (a_val > b_val) 165abf6739317742ca4677bf885178984a8757ee14aJason Evans return (-1); 166abf6739317742ca4677bf885178984a8757ee14aJason Evans } 167abf6739317742ca4677bf885178984a8757ee14aJason Evans /* 168abf6739317742ca4677bf885178984a8757ee14aJason Evans * Break ties by chunk address. For fragmented chunks, report lower 169abf6739317742ca4677bf885178984a8757ee14aJason Evans * addresses as "lower", so that fragmentation reduction happens first 170abf6739317742ca4677bf885178984a8757ee14aJason Evans * at lower addresses. However, use the opposite ordering for 171abf6739317742ca4677bf885178984a8757ee14aJason Evans * unfragmented chunks, in order to increase the chances of 172abf6739317742ca4677bf885178984a8757ee14aJason Evans * re-allocating dirty runs. 173abf6739317742ca4677bf885178984a8757ee14aJason Evans */ 174e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans { 175e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans uintptr_t a_chunk = (uintptr_t)a; 176e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans uintptr_t b_chunk = (uintptr_t)b; 177abf6739317742ca4677bf885178984a8757ee14aJason Evans int ret = ((a_chunk > b_chunk) - (a_chunk < b_chunk)); 178abf6739317742ca4677bf885178984a8757ee14aJason Evans if (a->nruns_adjac == 0) { 179abf6739317742ca4677bf885178984a8757ee14aJason Evans assert(b->nruns_adjac == 0); 180abf6739317742ca4677bf885178984a8757ee14aJason Evans ret = -ret; 181abf6739317742ca4677bf885178984a8757ee14aJason Evans } 182abf6739317742ca4677bf885178984a8757ee14aJason Evans return (ret); 183e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 184e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 185e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 186e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans/* Generate red-black tree functions. */ 187e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansrb_gen(static UNUSED, arena_chunk_dirty_, arena_chunk_tree_t, arena_chunk_t, 188e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans dirty_link, arena_chunk_dirty_comp) 189e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 190e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline bool 191e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind) 192e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 193e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans bool ret; 194e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 195e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (pageind-1 < map_bias) 196e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans ret = false; 197e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans else { 198e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0); 199e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(ret == false || arena_mapbits_dirty_get(chunk, 200e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans pageind-1) != arena_mapbits_dirty_get(chunk, pageind)); 201e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 202e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (ret); 203e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 204e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 205e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline bool 206e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages) 207e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 208e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans bool ret; 209e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 210e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (pageind+npages == chunk_npages) 211e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans ret = false; 212e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans else { 213e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(pageind+npages < chunk_npages); 214e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0); 215e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(ret == false || arena_mapbits_dirty_get(chunk, pageind) 216e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans != arena_mapbits_dirty_get(chunk, pageind+npages)); 217e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 218e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (ret); 219e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 220e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 221e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic inline bool 222e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages) 223e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 224e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 225e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return (arena_avail_adjac_pred(chunk, pageind) || 226e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_adjac_succ(chunk, pageind, npages)); 227e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 228e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 229e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic void 230e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 231e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 232e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 233e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 234e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 235e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans LG_PAGE)); 236e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 237e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 238e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 239e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * removed and reinserted even if the run to be inserted is clean. 240e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 241e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->ndirty != 0) 242e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 243e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 244e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 245e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac++; 246e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 247e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac++; 248e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_avail++; 249e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->nruns_avail > chunk->nruns_adjac); 250e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 251e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 252e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->ndirty += npages; 253e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->ndirty += npages; 254e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 255e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->ndirty != 0) 256e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 257e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 258e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, 259e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans pageind)); 260e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 261e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 262e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansstatic void 263e3d13060c8a04f08764b16b003169eb205fa09ebJason Evansarena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, 264e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) 265e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 266e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 267e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> 268e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans LG_PAGE)); 269e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 270e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 271e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be 272e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * removed and reinserted even if the run to be removed is clean. 273e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 274e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->ndirty != 0) 275e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); 276e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 277e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) 278e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac--; 279e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) 280e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_adjac--; 281e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->nruns_avail--; 282e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->nruns_avail > chunk->nruns_adjac || (chunk->nruns_avail 283e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans == 0 && chunk->nruns_adjac == 0)); 284e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 285e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena_mapbits_dirty_get(chunk, pageind) != 0) { 286e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->ndirty -= npages; 287e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk->ndirty -= npages; 288e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } 289e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (chunk->ndirty != 0) 290e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); 291e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 292e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, 293e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans pageind)); 294e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 295e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 296e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void * 29749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 30084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned regind; 30184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 30284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 3041e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(run->nfree > 0); 30584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); 306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 30784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); 30884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + 309122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (uintptr_t)(bin_info->reg_interval * regind)); 3101e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree--; 31184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans if (regind == run->nextind) 31284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans run->nextind++; 31384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(regind < run->nextind); 3141e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans return (ret); 3156109fe07a14b7a619365977d9523db9f8b333792Jason Evans} 3166109fe07a14b7a619365977d9523db9f8b333792Jason Evans 3176109fe07a14b7a619365977d9523db9f8b333792Jason Evansstatic inline void 3181e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evansarena_run_reg_dalloc(arena_run_t *run, void *ptr) 3196109fe07a14b7a619365977d9523db9f8b333792Jason Evans{ 32049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 321203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 322203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits = arena_mapbits_get(chunk, pageind); 32380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t binind = arena_ptr_small_binind_get(ptr, mapbits); 32449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 32584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned regind = arena_run_regind(run, bin_info, ptr); 32684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 32784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 32884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 32949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(run->nfree < bin_info->nregs); 3301e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Freeing an interior pointer can cause assertion failure. */ 3311e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 332122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (uintptr_t)bin_info->reg0_offset)) % 333122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (uintptr_t)bin_info->reg_interval == 0); 33421fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans assert((uintptr_t)ptr >= (uintptr_t)run + 33549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset); 33684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Freeing an unallocated pointer can cause assertion failure. */ 33784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); 338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 33984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_unset(bitmap, &bin_info->bitmap_info, regind); 3401e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree++; 341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 34321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evansstatic inline void 34438067483c542adfe092644d1ecc103c6bc74add0Jason Evansarena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) 34538067483c542adfe092644d1ecc103c6bc74add0Jason Evans{ 34638067483c542adfe092644d1ecc103c6bc74add0Jason Evans 347bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 348bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans (run_ind << LG_PAGE)), (npages << LG_PAGE)); 34938067483c542adfe092644d1ecc103c6bc74add0Jason Evans memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, 35038067483c542adfe092644d1ecc103c6bc74add0Jason Evans (npages << LG_PAGE)); 35138067483c542adfe092644d1ecc103c6bc74add0Jason Evans} 35238067483c542adfe092644d1ecc103c6bc74add0Jason Evans 35338067483c542adfe092644d1ecc103c6bc74add0Jason Evansstatic inline void 354dda90f59e2b67903668a2799970f64df163e9ccfJason Evansarena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) 355dda90f59e2b67903668a2799970f64df163e9ccfJason Evans{ 356dda90f59e2b67903668a2799970f64df163e9ccfJason Evans 357bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind 358bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans << LG_PAGE)), PAGE); 359dda90f59e2b67903668a2799970f64df163e9ccfJason Evans} 360dda90f59e2b67903668a2799970f64df163e9ccfJason Evans 361dda90f59e2b67903668a2799970f64df163e9ccfJason Evansstatic inline void 36238067483c542adfe092644d1ecc103c6bc74add0Jason Evansarena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 36321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans{ 364d4bab21756279db540866998099522dbd39c05f7Jason Evans size_t i; 365ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); 366d4bab21756279db540866998099522dbd39c05f7Jason Evans 367dda90f59e2b67903668a2799970f64df163e9ccfJason Evans arena_run_page_mark_zeroed(chunk, run_ind); 368ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans for (i = 0; i < PAGE / sizeof(size_t); i++) 36921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans assert(p[i] == 0); 37021fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans} 37121fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans 372e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 373aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_cactive_update(arena_t *arena, size_t add_pages, size_t sub_pages) 374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 376aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_stats) { 377aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ssize_t cactive_diff = CHUNK_CEILING((arena->nactive + 378aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans add_pages) << LG_PAGE) - CHUNK_CEILING((arena->nactive - 379aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans sub_pages) << LG_PAGE); 380aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (cactive_diff != 0) 381aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans stats_cactive_add(cactive_diff); 382aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 383aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 384aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 385aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic void 386aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, 387aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t flag_dirty, size_t need_pages) 388aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 389aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t total_pages, rem_pages; 390aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 391aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> 392aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans LG_PAGE; 393aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == 394aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty); 395aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(need_pages <= total_pages); 396aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans rem_pages = total_pages - need_pages; 397aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 398aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); 399aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_cactive_update(arena, need_pages, 0); 400aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena->nactive += need_pages; 401aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 402aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* Keep track of trailing unused pages for later use. */ 403aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (rem_pages > 0) { 404aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (flag_dirty != 0) { 405aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, 406aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+need_pages, (rem_pages << LG_PAGE), 407aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty); 408aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, 409aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+total_pages-1, (rem_pages << LG_PAGE), 410aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty); 411aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } else { 412aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, run_ind+need_pages, 413aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans (rem_pages << LG_PAGE), 414aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unzeroed_get(chunk, 415aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+need_pages)); 416aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, 417aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+total_pages-1, (rem_pages << LG_PAGE), 418aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unzeroed_get(chunk, 419aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+total_pages-1)); 420aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 421aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages, 422aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans false, true); 423aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 424aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 425aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 426aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic void 427aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, 428aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans bool remove, bool zero) 429aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 430aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *chunk; 431aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t flag_dirty, run_ind, need_pages, i; 432203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 433e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 434ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 435203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 436ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans need_pages = (size >> LG_PAGE); 437e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages > 0); 438e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 439c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans if (remove) { 440aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_remove(arena, chunk, run_ind, flag_dirty, 441aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans need_pages); 442aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 443c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans 444aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (zero) { 445aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (flag_dirty == 0) { 446c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans /* 447aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * The run is clean, so some pages may be zeroed (i.e. 448aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * never before touched). 449c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans */ 450aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans for (i = 0; i < need_pages; i++) { 451aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (arena_mapbits_unzeroed_get(chunk, run_ind+i) 452aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans != 0) 453aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_zero(chunk, run_ind+i, 1); 454aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans else if (config_debug) { 455aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_page_validate_zeroed(chunk, 456aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+i); 457aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } else { 458aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_page_mark_zeroed(chunk, 459aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+i); 46019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 462dda90f59e2b67903668a2799970f64df163e9ccfJason Evans } else { 463aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* The run is dirty, so all pages must be zeroed. */ 464aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_zero(chunk, run_ind, need_pages); 465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 46619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 467bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 468dda90f59e2b67903668a2799970f64df163e9ccfJason Evans (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 470aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 471aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 472aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Set the last element first, in case the run only contains one page 473aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * (i.e. both statements set the same element). 474aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 475aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty); 476aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); 477e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 478e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 479c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evansstatic void 480aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 481c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans{ 482c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans 483aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_large_helper(arena, run, size, true, zero); 484c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans} 485c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans 486c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evansstatic void 487aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) 488c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans{ 489c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans 490aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_large_helper(arena, run, size, false, zero); 491aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 492aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 493aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic void 494aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, 495aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t binind) 496aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 497aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *chunk; 498aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t flag_dirty, run_ind, need_pages, i; 499aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 500aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(binind != BININD_INVALID); 501aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 502aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 503aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 504aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); 505aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans need_pages = (size >> LG_PAGE); 506aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(need_pages > 0); 507aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 508aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_remove(arena, chunk, run_ind, flag_dirty, need_pages); 509aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 510aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 511aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Propagate the dirty and unzeroed flags to the allocated small run, 512aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * so that arena_dalloc_bin_run() has the ability to conditionally trim 513aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * clean pages. 514aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 515aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); 516aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 517aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * The first page will always be dirtied during small run 518aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * initialization, so a validation failure here would not actually 519aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * cause an observable failure. 520aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 521aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, 522aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind) == 0) 523aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_page_validate_zeroed(chunk, run_ind); 524aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans for (i = 1; i < need_pages - 1; i++) { 525aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); 526aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_debug && flag_dirty == 0 && 527aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) 528aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_page_validate_zeroed(chunk, run_ind+i); 529aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 530aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_small_set(chunk, run_ind+need_pages-1, need_pages-1, 531aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans binind, flag_dirty); 532aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, 533aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+need_pages-1) == 0) 534aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_page_validate_zeroed(chunk, run_ind+need_pages-1); 535bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + 536aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); 537c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans} 538c368f8c8a243248feb7771f4d32691e7b2aa6f1aJason Evans 539e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t * 540aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_init_spare(arena_t *arena) 541e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 542e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 543e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 544aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena->spare != NULL); 54519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 546aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = arena->spare; 547aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena->spare = NULL; 54841631d00618d7262125e501c91d31b4d70e605faJason Evans 549aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 550aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 551aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 552aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_maxclass); 553aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 554aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_maxclass); 555aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_dirty_get(chunk, map_bias) == 556aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_dirty_get(chunk, chunk_npages-1)); 557e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 558aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (chunk); 559aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 560e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 561aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic arena_chunk_t * 56212141150fdbda57651a53ae2fe0edaea4891d814Jason Evansarena_chunk_alloc_internal(arena_t *arena, size_t size, size_t alignment, 56312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans bool *zero) 56412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans{ 56512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena_chunk_t *chunk; 56612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_alloc_t *chunk_alloc; 56712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc_t *chunk_dalloc; 56812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 56912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_alloc = arena->chunk_alloc; 57012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc = arena->chunk_dalloc; 57112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans malloc_mutex_unlock(&arena->lock); 57212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk = (arena_chunk_t *)chunk_alloc_arena(chunk_alloc, chunk_dalloc, 57312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->ind, size, alignment, zero); 57412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans malloc_mutex_lock(&arena->lock); 57512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (config_stats && chunk != NULL) 57612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->stats.mapped += chunksize; 57712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 57812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans return (chunk); 57912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans} 58012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 58112141150fdbda57651a53ae2fe0edaea4891d814Jason Evansvoid * 58212141150fdbda57651a53ae2fe0edaea4891d814Jason Evansarena_chunk_alloc_huge(arena_t *arena, size_t size, size_t alignment, 58312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans bool *zero) 58412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans{ 58512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans void *ret; 58612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_alloc_t *chunk_alloc; 58712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc_t *chunk_dalloc; 58812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 58912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans malloc_mutex_lock(&arena->lock); 59012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_alloc = arena->chunk_alloc; 59112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc = arena->chunk_dalloc; 59212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (config_stats) { 59312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans /* Optimistically update stats prior to unlocking. */ 59412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->stats.mapped += size; 59512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->stats.allocated_huge += size; 59612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->stats.nmalloc_huge++; 59712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->stats.nrequests_huge++; 59812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans } 59912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->nactive += (size >> LG_PAGE); 60012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans malloc_mutex_unlock(&arena->lock); 60112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 60212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans ret = chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, 60312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans size, alignment, zero); 60412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (config_stats) { 60512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (ret != NULL) 60612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans stats_cactive_add(size); 60712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans else { 60812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans /* Revert optimistic stats updates. */ 60912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans malloc_mutex_lock(&arena->lock); 61012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->stats.mapped -= size; 61112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->stats.allocated_huge -= size; 61212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->stats.nmalloc_huge--; 61312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans malloc_mutex_unlock(&arena->lock); 61412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans } 61512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans } 61612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 61712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans return (ret); 61812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans} 61912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 62012141150fdbda57651a53ae2fe0edaea4891d814Jason Evansstatic arena_chunk_t * 621aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_init_hard(arena_t *arena) 622aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 623aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *chunk; 624aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans bool zero; 625aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t unzeroed, i; 626e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 627aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena->spare == NULL); 628e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 629aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans zero = false; 63012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk = arena_chunk_alloc_internal(arena, chunksize, chunksize, &zero); 631aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (chunk == NULL) 632aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (NULL); 633aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 634aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk->arena = arena; 635aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 636aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 637aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Claim that no pages are in use, since the header is merely overhead. 638aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 639aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk->ndirty = 0; 640aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 641aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk->nruns_avail = 0; 642aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk->nruns_adjac = 0; 643aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 644aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 645aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Initialize the map to contain one maximal free untouched run. Mark 646aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * the pages as zeroed iff chunk_alloc() returned a zeroed chunk. 647aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 648aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; 649aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, 650aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans unzeroed); 651aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 652aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * There is no need to initialize the internal page map entries unless 653aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * the chunk is not zeroed. 654aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 655aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (zero == false) { 656bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( 657bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans (void *)arena_mapp_get(chunk, map_bias+1), 658bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans (size_t)((uintptr_t) arena_mapp_get(chunk, chunk_npages-1) - 659bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans (uintptr_t)arena_mapp_get(chunk, map_bias+1))); 660aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans for (i = map_bias+1; i < chunk_npages-1; i++) 661aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unzeroed_set(chunk, i, unzeroed); 662aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } else { 663bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)arena_mapp_get(chunk, 664aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, 665aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, 666aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans map_bias+1))); 667aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_debug) { 668aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans for (i = map_bias+1; i < chunk_npages-1; i++) { 669aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_unzeroed_get(chunk, i) == 670aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans unzeroed); 671203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 672940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 67319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 674aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxclass, 675aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans unzeroed); 676aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 677aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (chunk); 678aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 679aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 680aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic arena_chunk_t * 681aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_alloc(arena_t *arena) 682aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 683aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *chunk; 684aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 685aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (arena->spare != NULL) 686aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = arena_chunk_init_spare(arena); 68720a8c78bfe3310e0f0f72b596d4e10ca7336063bChris Pride else { 688aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = arena_chunk_init_hard(arena); 68920a8c78bfe3310e0f0f72b596d4e10ca7336063bChris Pride if (chunk == NULL) 69020a8c78bfe3310e0f0f72b596d4e10ca7336063bChris Pride return (NULL); 69120a8c78bfe3310e0f0f72b596d4e10ca7336063bChris Pride } 692e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 693e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Insert the run into the runs_avail tree. */ 694e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias, 695e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false, false); 696e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 697e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (chunk); 698e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 699e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 700e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 70112141150fdbda57651a53ae2fe0edaea4891d814Jason Evansarena_chunk_dalloc_internal(arena_t *arena, arena_chunk_t *chunk) 70212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans{ 70312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc_t *chunk_dalloc; 70412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 70512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc = arena->chunk_dalloc; 70612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans malloc_mutex_unlock(&arena->lock); 70712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc((void *)chunk, chunksize, arena->ind); 70812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans malloc_mutex_lock(&arena->lock); 70912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (config_stats) 71012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->stats.mapped -= chunksize; 71112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans} 71212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 71312141150fdbda57651a53ae2fe0edaea4891d814Jason Evansvoid 71412141150fdbda57651a53ae2fe0edaea4891d814Jason Evansarena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t size) 71512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans{ 71612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc_t *chunk_dalloc; 71712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 71812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans malloc_mutex_lock(&arena->lock); 71912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc = arena->chunk_dalloc; 72012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans if (config_stats) { 72112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->stats.mapped -= size; 72212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->stats.allocated_huge -= size; 72312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->stats.ndalloc_huge++; 72412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans stats_cactive_sub(size); 72512141150fdbda57651a53ae2fe0edaea4891d814Jason Evans } 72612141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->nactive -= (size >> LG_PAGE); 72712141150fdbda57651a53ae2fe0edaea4891d814Jason Evans malloc_mutex_unlock(&arena->lock); 72812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans chunk_dalloc(chunk, size, arena->ind); 72912141150fdbda57651a53ae2fe0edaea4891d814Jason Evans} 73012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans 73112141150fdbda57651a53ae2fe0edaea4891d814Jason Evansstatic void 73212141150fdbda57651a53ae2fe0edaea4891d814Jason Evansarena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) 733e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 73430fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); 73530fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); 73630fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == 73730fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_maxclass); 73830fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == 73930fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_maxclass); 74030fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_dirty_get(chunk, map_bias) == 74130fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_mapbits_dirty_get(chunk, chunk_npages-1)); 74230fe12b866edbc2cf9aaef299063b392ea125aacJason Evans 7438d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans /* 744e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Remove run from the runs_avail tree, so that the arena does not use 745e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * it. 7468d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans */ 747e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias, 748e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false, false); 7498d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans 7508d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (arena->spare != NULL) { 751e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_chunk_t *spare = arena->spare; 752e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 7538d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans arena->spare = chunk; 75412141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena_chunk_dalloc_internal(arena, spare); 7558d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans } else 7568d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans arena->spare = chunk; 757e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 758e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 759e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 760aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) 761aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 762aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_t *run; 763f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer arena_chunk_map_t *mapelm; 764f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer arena_chunk_map_t *key; 765aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 766f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); 767f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); 768aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (mapelm != NULL) { 769aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 770f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer size_t pageind = arena_mapelm_to_pageind(mapelm); 771aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 772aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 773aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans LG_PAGE)); 774aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_large(arena, run, size, zero); 775aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (run); 776aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 777aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 778aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (NULL); 779aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 780aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 781aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic arena_run_t * 782aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_alloc_large(arena_t *arena, size_t size, bool zero) 783aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 784aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *chunk; 785aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_t *run; 786aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 787aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(size <= arena_maxclass); 788aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert((size & PAGE_MASK) == 0); 789aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 790aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* Search the arena's chunks for the lowest best fit. */ 791aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run = arena_run_alloc_large_helper(arena, size, zero); 792aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (run != NULL) 793aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (run); 794aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 795aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 796aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * No usable runs. Create a new chunk from which to allocate the run. 797aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 798aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = arena_chunk_alloc(arena); 799aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (chunk != NULL) { 800aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); 801aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_large(arena, run, size, zero); 802aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (run); 803aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 804aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 805aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 806aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * arena_chunk_alloc() failed, but another thread may have made 807aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * sufficient memory available while this one dropped arena->lock in 808aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * arena_chunk_alloc(), so search one more time. 809aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 810aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (arena_run_alloc_large_helper(arena, size, zero)); 811aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 812aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 813aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic arena_run_t * 814aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind) 815e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 816e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 817f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer arena_chunk_map_t *mapelm; 818f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer arena_chunk_map_t *key; 819e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 820f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); 821f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); 822e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 823e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 824f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer size_t pageind = arena_mapelm_to_pageind(mapelm); 825e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 826e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 827ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 828aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_small(arena, run, size, binind); 829e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 830e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 831e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 8325b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans return (NULL); 8335b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans} 8345b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans 8355b0c99649fa71674daadf4dd53b1ab05428483fbJason Evansstatic arena_run_t * 836aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_alloc_small(arena_t *arena, size_t size, size_t binind) 8375b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans{ 8385b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans arena_chunk_t *chunk; 8395b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans arena_run_t *run; 8405b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans 8415b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans assert(size <= arena_maxclass); 8425b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans assert((size & PAGE_MASK) == 0); 843aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(binind != BININD_INVALID); 8445b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans 8455b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans /* Search the arena's chunks for the lowest best fit. */ 846aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run = arena_run_alloc_small_helper(arena, size, binind); 8475b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans if (run != NULL) 8485b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans return (run); 8495b0c99649fa71674daadf4dd53b1ab05428483fbJason Evans 850e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 851e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * No usable runs. Create a new chunk from which to allocate the run. 852e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 853e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena_chunk_alloc(arena); 854e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (chunk != NULL) { 855ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); 856aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_small(arena, run, size, binind); 857e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 858e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 859e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 860e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 861e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_chunk_alloc() failed, but another thread may have made 862e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * sufficient memory available while this one dropped arena->lock in 863e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_chunk_alloc(), so search one more time. 864e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 865aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (arena_run_alloc_small_helper(arena, size, binind)); 866e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 867e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 86805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansstatic inline void 86905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansarena_maybe_purge(arena_t *arena) 87005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans{ 871fb795867f0b3aa28bbdf177e1026f3e3408e0338Christopher Ferris#if !defined(ANDROID_ALWAYS_PURGE) 872e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npurgeable, threshold; 873e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 874e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Don't purge if the option is disabled. */ 875e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (opt_lg_dirty_mult < 0) 876e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return; 877fb795867f0b3aa28bbdf177e1026f3e3408e0338Christopher Ferris#endif 878e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Don't purge if all dirty pages are already being purged. */ 879e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena->ndirty <= arena->npurgatory) 880e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return; 881fb795867f0b3aa28bbdf177e1026f3e3408e0338Christopher Ferris#if !defined(ANDROID_ALWAYS_PURGE) 882e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgeable = arena->ndirty - arena->npurgatory; 883e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans threshold = (arena->nactive >> opt_lg_dirty_mult); 884e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 885e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Don't purge unless the number of purgeable pages exceeds the 886e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * threshold. 887e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 888e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (npurgeable <= threshold) 889e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans return; 890fb795867f0b3aa28bbdf177e1026f3e3408e0338Christopher Ferris#endif 89105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 892e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_purge(arena, false); 89305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans} 89405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 895aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic arena_chunk_t * 896aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evanschunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) 89705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans{ 898aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t *ndirty = (size_t *)arg; 89905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 900aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(chunk->ndirty != 0); 901aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans *ndirty += chunk->ndirty; 902aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (NULL); 903aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 904aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 905aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic size_t 906aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_compute_npurgatory(arena_t *arena, bool all) 907aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 908aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t npurgatory, npurgeable; 90905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 91005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 911aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Compute the minimum number of pages that this thread should try to 912aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * purge. 91305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 914aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans npurgeable = arena->ndirty - arena->npurgatory; 91530fe12b866edbc2cf9aaef299063b392ea125aacJason Evans 916aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (all == false) { 917aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t threshold = (arena->nactive >> opt_lg_dirty_mult); 91805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 919aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans npurgatory = npurgeable - threshold; 920aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } else 921aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans npurgatory = npurgeable; 922e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 923aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (npurgatory); 924aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 925aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 926aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic void 927aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_stash_dirty(arena_t *arena, arena_chunk_t *chunk, bool all, 928aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_mapelms_t *mapelms) 929aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 930aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t pageind, npages; 931e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 932e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 933e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Temporarily allocate free dirty runs within chunk. If all is false, 934e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * only operate on dirty runs that are fragments; otherwise operate on 935e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * all dirty runs. 936e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 937e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { 938aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 939203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (arena_mapbits_allocated_get(chunk, pageind) == 0) { 940e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t run_size = 941e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_mapbits_unallocated_size_get(chunk, pageind); 94205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 943e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages = run_size >> LG_PAGE; 944e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind + npages <= chunk_npages); 94530fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_dirty_get(chunk, pageind) == 94630fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_mapbits_dirty_get(chunk, pageind+npages-1)); 947c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 948e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena_mapbits_dirty_get(chunk, pageind) != 0 && 949e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans (all || arena_avail_adjac(chunk, pageind, 950e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages))) { 951e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t) 952e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk + (uintptr_t)(pageind << LG_PAGE)); 953e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 954aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_large(arena, run, run_size, 955aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans false); 95619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Append to list for later processing. */ 95719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans ql_elm_new(mapelm, u.ql_link); 958aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ql_tail_insert(mapelms, mapelm, u.ql_link); 95905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 96005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } else { 961e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Skip run. */ 962e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (arena_mapbits_large_get(chunk, pageind) != 0) { 963e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages = arena_mapbits_large_size_get(chunk, 964203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind) >> LG_PAGE; 965e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans } else { 9668b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey size_t binind; 9678b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey arena_bin_info_t *bin_info; 96805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t) 969ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans chunk + (uintptr_t)(pageind << LG_PAGE)); 970e69bee01de62b56d3e585042d341743239568043Jason Evans 971203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_small_runind_get(chunk, 972203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind) == 0); 9738b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey binind = arena_bin_index(arena, run->bin); 9748b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey bin_info = &arena_bin_info[binind]; 975e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages = bin_info->run_size >> LG_PAGE; 97605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 97705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 97805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 979e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind == chunk_npages); 980e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->ndirty == 0 || all == false); 981e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(chunk->nruns_adjac == 0); 982aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 983aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 984aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic size_t 985aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_purge_stashed(arena_t *arena, arena_chunk_t *chunk, 986aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_mapelms_t *mapelms) 987aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 988aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t npurged, pageind, npages, nmadvise; 989aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_map_t *mapelm; 99005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 99105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans malloc_mutex_unlock(&arena->lock); 9927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 9937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans nmadvise = 0; 994e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurged = 0; 995aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ql_foreach(mapelm, mapelms, u.ql_link) { 9967de92767c20cb72c94609b9c78985526fb84a679Jason Evans bool unzeroed; 9977de92767c20cb72c94609b9c78985526fb84a679Jason Evans size_t flag_unzeroed, i; 99805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 999f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer pageind = arena_mapelm_to_pageind(mapelm); 1000e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npages = arena_mapbits_large_size_get(chunk, pageind) >> 1001e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans LG_PAGE; 1002e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind + npages <= chunk_npages); 10037de92767c20cb72c94609b9c78985526fb84a679Jason Evans unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind << 10047de92767c20cb72c94609b9c78985526fb84a679Jason Evans LG_PAGE)), (npages << LG_PAGE)); 10057de92767c20cb72c94609b9c78985526fb84a679Jason Evans flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; 10067de92767c20cb72c94609b9c78985526fb84a679Jason Evans /* 10077de92767c20cb72c94609b9c78985526fb84a679Jason Evans * Set the unzeroed flag for all pages, now that pages_purge() 10087de92767c20cb72c94609b9c78985526fb84a679Jason Evans * has returned whether the pages were zeroed as a side effect 10097de92767c20cb72c94609b9c78985526fb84a679Jason Evans * of purging. This chunk map modification is safe even though 10107de92767c20cb72c94609b9c78985526fb84a679Jason Evans * the arena mutex isn't currently owned by this thread, 10117de92767c20cb72c94609b9c78985526fb84a679Jason Evans * because the run is marked as allocated, thus protecting it 10127de92767c20cb72c94609b9c78985526fb84a679Jason Evans * from being modified by any other thread. As long as these 10137de92767c20cb72c94609b9c78985526fb84a679Jason Evans * writes don't perturb the first and last elements' 10147de92767c20cb72c94609b9c78985526fb84a679Jason Evans * CHUNK_MAP_ALLOCATED bits, behavior is well defined. 10157de92767c20cb72c94609b9c78985526fb84a679Jason Evans */ 10167de92767c20cb72c94609b9c78985526fb84a679Jason Evans for (i = 0; i < npages; i++) { 10177de92767c20cb72c94609b9c78985526fb84a679Jason Evans arena_mapbits_unzeroed_set(chunk, pageind+i, 10187de92767c20cb72c94609b9c78985526fb84a679Jason Evans flag_unzeroed); 10197de92767c20cb72c94609b9c78985526fb84a679Jason Evans } 1020e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurged += npages; 10217372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 10227372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans nmadvise++; 102305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 102405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans malloc_mutex_lock(&arena->lock); 10257372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 10267372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmadvise += nmadvise; 102705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 1028aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (npurged); 1029aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 1030aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1031aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic void 1032aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_unstash_purged(arena_t *arena, arena_chunk_t *chunk, 1033aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_mapelms_t *mapelms) 1034aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 1035aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_map_t *mapelm; 1036aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t pageind; 1037aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 103805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Deallocate runs. */ 1039aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans for (mapelm = ql_first(mapelms); mapelm != NULL; 1040aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans mapelm = ql_first(mapelms)) { 1041e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_t *run; 104205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 1043f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer pageind = arena_mapelm_to_pageind(mapelm); 1044e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << 1045e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans LG_PAGE)); 1046aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ql_remove(mapelms, mapelm, u.ql_link); 1047e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_dalloc(arena, run, false, true); 104805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 1049e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans} 1050e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 1051aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic inline size_t 1052aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) 1053e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans{ 1054aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t npurged; 1055aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_mapelms_t mapelms; 1056e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 1057aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ql_new(&mapelms); 1058aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1059aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 1060aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * If chunk is the spare, temporarily re-allocate it, 1) so that its 1061aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * run is reinserted into runs_avail, and 2) so that it cannot be 1062aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * completely discarded by another thread while arena->lock is dropped 1063aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * by this thread. Note that the arena_run_dalloc() call will 1064aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * implicitly deallocate the chunk, so no explicit action is required 1065aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * in this function to deallocate the chunk. 1066aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * 1067aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Note that once a chunk contains dirty pages, it cannot again contain 1068aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * a single run unless 1) it is a dirty run, or 2) this function purges 1069aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * dirty pages and causes the transition to a single clean run. Thus 1070aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * (chunk == arena->spare) is possible, but it is not possible for 1071aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * this function to be called on the spare unless it contains a dirty 1072aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * run. 1073aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 1074aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (chunk == arena->spare) { 1075aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); 1076aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); 1077aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1078aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_alloc(arena); 1079aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 1080aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1081aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (config_stats) 1082aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena->stats.purged += chunk->ndirty; 1083aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1084aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 1085aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Operate on all dirty runs if there is no clean/dirty run 1086aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * fragmentation. 1087aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 1088aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (chunk->nruns_adjac == 0) 1089aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans all = true; 1090aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1091aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_stash_dirty(arena, chunk, all, &mapelms); 1092aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans npurged = arena_chunk_purge_stashed(arena, chunk, &mapelms); 1093aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_unstash_purged(arena, chunk, &mapelms); 1094aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1095aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans return (npurged); 109605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans} 109705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 1098e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 10996005f0710cf07d60659d91b20b7ff5592d310027Jason Evansarena_purge(arena_t *arena, bool all) 1100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 110205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t npurgatory; 11037372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) { 11047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t ndirty = 0; 1105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1106e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_iter(&arena->chunks_dirty, NULL, 1107e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunks_dirty_iter_cb, (void *)&ndirty); 11087372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans assert(ndirty == arena->ndirty); 11092caa4715ed4f787f263239ff97dd824636289286Jason Evans } 1110af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans assert(arena->ndirty > arena->npurgatory || all); 1111af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - 1112f9a8edbb50f8cbcaf8ed62b36e8d7191ed223d2aJason Evans arena->npurgatory) || all); 1113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 11147372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 11157372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.npurge++; 1116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1118aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Add the minimum number of pages this thread should try to purge to 1119aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * arena->npurgatory. This will keep multiple threads from racing to 1120aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * reduce ndirty below the threshold. 1121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1122aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans npurgatory = arena_compute_npurgatory(arena, all); 1123799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory += npurgatory; 1124799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans 112505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans while (npurgatory > 0) { 1126e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t npurgeable, npurged, nunpurged; 1127e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 112805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Get next chunk with dirty pages. */ 1129e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans chunk = arena_chunk_dirty_first(&arena->chunks_dirty); 113005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans if (chunk == NULL) { 113105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 113205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * This thread was unable to purge as many pages as 113305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * originally intended, due to races with other threads 1134799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * that either did some of the purging work, or re-used 1135799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * dirty pages. 113605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 1137799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory -= npurgatory; 1138799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans return; 1139e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1140e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgeable = chunk->ndirty; 1141e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans assert(npurgeable != 0); 114205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 1143e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans if (npurgeable > npurgatory && chunk->nruns_adjac == 0) { 1144799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans /* 1145e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * This thread will purge all the dirty pages in chunk, 1146e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * so set npurgatory to reflect this thread's intent to 1147e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * purge the pages. This tends to reduce the chances 1148e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * of the following scenario: 1149799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 1150799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 1) This thread sets arena->npurgatory such that 1151799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * (arena->ndirty - arena->npurgatory) is at the 1152799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * threshold. 1153799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 2) This thread drops arena->lock. 1154799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 3) Another thread causes one or more pages to be 1155799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * dirtied, and immediately determines that it must 1156799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * purge dirty pages. 1157799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 1158799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * If this scenario *does* play out, that's okay, 1159799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * because all of the purging work being done really 1160799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * needs to happen. 1161799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans */ 1162e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->npurgatory += npurgeable - npurgatory; 1163e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgatory = npurgeable; 1164799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans } 1165799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans 1166e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 1167e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Keep track of how many pages are purgeable, versus how many 1168e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * actually get purged, and adjust counters accordingly. 1169e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans */ 1170e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->npurgatory -= npurgeable; 1171e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgatory -= npurgeable; 1172e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurged = arena_chunk_purge(arena, chunk, all); 1173e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans nunpurged = npurgeable - npurged; 1174e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena->npurgatory += nunpurged; 1175e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans npurgatory += nunpurged; 1176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1177e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1178e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 11796005f0710cf07d60659d91b20b7ff5592d310027Jason Evansvoid 11806005f0710cf07d60659d91b20b7ff5592d310027Jason Evansarena_purge_all(arena_t *arena) 11816005f0710cf07d60659d91b20b7ff5592d310027Jason Evans{ 11826005f0710cf07d60659d91b20b7ff5592d310027Jason Evans 11836005f0710cf07d60659d91b20b7ff5592d310027Jason Evans malloc_mutex_lock(&arena->lock); 11846005f0710cf07d60659d91b20b7ff5592d310027Jason Evans arena_purge(arena, true); 11856005f0710cf07d60659d91b20b7ff5592d310027Jason Evans malloc_mutex_unlock(&arena->lock); 11866005f0710cf07d60659d91b20b7ff5592d310027Jason Evans} 11876005f0710cf07d60659d91b20b7ff5592d310027Jason Evans 1188e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1189aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, 1190aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty) 1191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1192aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t size = *p_size; 1193aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t run_ind = *p_run_ind; 1194aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t run_pages = *p_run_pages; 1195e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce forward. */ 1197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run_ind + run_pages < chunk_npages && 1198203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && 1199203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) { 1200203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, 1201203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+run_pages); 1202ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t nrun_pages = nrun_size >> LG_PAGE; 1203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1204e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove successor from runs_avail; the coalesced run is 1206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 1207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1208203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_unallocated_size_get(chunk, 1209203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+run_pages+nrun_pages-1) == nrun_size); 1210203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, 1211203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind+run_pages+nrun_pages-1) == flag_dirty); 1212e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages, 1213e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false, true); 1214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1215e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += nrun_size; 121612ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_pages += nrun_pages; 1217e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1218203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1219203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1220203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size); 1221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce backward. */ 1224aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, 1225aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == 1226aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty) { 1227203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t prun_size = arena_mapbits_unallocated_size_get(chunk, 1228203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run_ind-1); 1229ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t prun_pages = prun_size >> LG_PAGE; 1230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 123112ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_ind -= prun_pages; 1232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove predecessor from runs_avail; the coalesced run is 1235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 1236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1237203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1238203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans prun_size); 1239203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); 1240e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_remove(arena, chunk, run_ind, prun_pages, true, 1241e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans false); 1242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += prun_size; 124412ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_pages += prun_pages; 1245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1246203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_set(chunk, run_ind, size); 1247203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, 1248203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size); 1249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1251aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans *p_size = size; 1252aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans *p_run_ind = run_ind; 1253aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans *p_run_pages = run_pages; 1254aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans} 1255aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1256aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansstatic void 1257aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evansarena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) 1258aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans{ 1259aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_chunk_t *chunk; 1260aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t size, run_ind, run_pages, flag_dirty; 1261aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1262aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1263aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 1264aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(run_ind >= map_bias); 1265aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(run_ind < chunk_npages); 1266aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (arena_mapbits_large_get(chunk, run_ind) != 0) { 1267aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size = arena_mapbits_large_size_get(chunk, run_ind); 1268aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(size == PAGE || 1269aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_large_size_get(chunk, 1270aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_ind+(size>>LG_PAGE)-1) == 0); 1271aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } else { 1272aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size_t binind = arena_bin_index(arena, run->bin); 1273aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 1274aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans size = bin_info->run_size; 1275aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 1276aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run_pages = (size >> LG_PAGE); 1277aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_cactive_update(arena, 0, run_pages); 1278aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena->nactive -= run_pages; 1279aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1280aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* 1281aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * The run is dirty if the caller claims to have dirtied it, as well as 1282aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * if it was already dirty before being allocated and the caller 1283aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * doesn't claim to have cleaned it. 1284aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans */ 1285aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans assert(arena_mapbits_dirty_get(chunk, run_ind) == 1286aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1287aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) 1288aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans dirty = true; 1289aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 1290aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1291aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans /* Mark pages as unallocated in the chunk map. */ 1292aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans if (dirty) { 1293aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, run_ind, size, 1294aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans CHUNK_MAP_DIRTY); 1295aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1296aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans CHUNK_MAP_DIRTY); 1297aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } else { 1298aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, run_ind, size, 1299aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unzeroed_get(chunk, run_ind)); 1300aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, 1301aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); 1302aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans } 1303aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1304aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, 1305aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans flag_dirty); 1306aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans 1307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert into runs_avail, now that coalescing is complete. */ 1308203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == 1309203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); 1310203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, run_ind) == 1311203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); 1312e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_insert(arena, chunk, run_ind, run_pages, true, true); 13138d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans 1314203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans /* Deallocate chunk if it is now completely unused. */ 1315203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (size == arena_maxclass) { 1316203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(run_ind == map_bias); 1317203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(run_pages == (arena_maxclass >> LG_PAGE)); 131812141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena_chunk_dalloc(arena, chunk); 1319203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 1320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 13214fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 13228d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * It is okay to do dirty page processing here even if the chunk was 13234fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * deallocated above, since in that case it is the spare. Waiting 13244fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * until after possible chunk deallocation to do dirty processing 13254fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * allows for an old spare to be fully deallocated, thus decreasing the 13264fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * chances of spuriously crossing the dirty page purging threshold. 13274fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 13288d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (dirty) 132905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_maybe_purge(arena); 1330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1332e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1333e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize) 1335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1336ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1337ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t head_npages = (oldsize - newsize) >> LG_PAGE; 1338203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 1341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 1344940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * leading run as separately allocated. Set the last element of each 1345940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * run first, in case of single-page runs. 1346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1347203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1348d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1349d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty); 1350940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 13517372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) { 1352ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans UNUSED size_t tail_npages = newsize >> LG_PAGE; 1353203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, 1354203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+head_npages+tail_npages-1) == 0); 1355203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, 1356203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+head_npages+tail_npages-1) == flag_dirty); 1357940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 1358d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind+head_npages, newsize, 1359d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans flag_dirty); 1360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1361e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_dalloc(arena, run, false, false); 1362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1364e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1365e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize, bool dirty) 1367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1368ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1369ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t head_npages = newsize >> LG_PAGE; 1370203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); 1371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 1373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 1376940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * trailing run as separately allocated. Set the last element of each 1377940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * run first, in case of single-page runs. 1378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1379203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); 1380d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); 1381d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty); 1382203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1383203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (config_debug) { 1384203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; 1385203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, 1386203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+head_npages+tail_npages-1) == 0); 1387203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_dirty_get(chunk, 1388203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+head_npages+tail_npages-1) == flag_dirty); 1389203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 1390203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, 1391d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans flag_dirty); 1392e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1393e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 1394e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans dirty, false); 1395e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1397e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 1398e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_first(arena_bin_t *bin) 1399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1400e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); 1401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 1402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1403e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 14048b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey arena_run_t *run; 1405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1406e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 1407f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer pageind = arena_mapelm_to_pageind(mapelm); 1408203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1409203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_small_runind_get(chunk, pageind)) << 1410ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 1411e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1412e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans } 1413e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1414e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (NULL); 1415e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1416e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1417e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void 1418e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 1419e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1420e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); 1421ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1422203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1423e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1424e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); 1425e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1426e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_tree_insert(&bin->runs, mapelm); 1427e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1428e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1429e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void 1430e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) 1431e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1432e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1433ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1434203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); 1435e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1436e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); 1437e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1438e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_tree_remove(&bin->runs, mapelm); 1439e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1440e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1441e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t * 1442e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_nonfull_run_tryget(arena_bin_t *bin) 1443e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1444e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_t *run = arena_bin_runs_first(bin); 1445e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) { 1446e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_remove(bin, run); 14477372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 14487372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.reruns++; 1449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1450e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1451e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1452e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1453e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t * 1454e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 1455e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1456e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_t *run; 1457e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans size_t binind; 1458e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_info_t *bin_info; 1459e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1460e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans /* Look for a usable run. */ 1461e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans run = arena_bin_nonfull_run_tryget(bin); 1462e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) 1463e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* No existing runs have any space available. */ 1465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 146649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(arena, bin); 146749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 146849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 1469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Allocate a new run. */ 1470e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&bin->lock); 1471e69bee01de62b56d3e585042d341743239568043Jason Evans /******************************/ 147286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1473aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run = arena_run_alloc_small(arena, bin_info->run_size, binind); 1474e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run != NULL) { 147584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 147684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 147784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 1478e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* Initialize run internals. */ 1479e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run->bin = bin; 148084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans run->nextind = 0; 148149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans run->nfree = bin_info->nregs; 148284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_init(bitmap, &bin_info->bitmap_info); 1483e69bee01de62b56d3e585042d341743239568043Jason Evans } 1484e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_unlock(&arena->lock); 1485e69bee01de62b56d3e585042d341743239568043Jason Evans /********************************/ 1486e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 1487e69bee01de62b56d3e585042d341743239568043Jason Evans if (run != NULL) { 14887372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 14897372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nruns++; 14907372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.curruns++; 14917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1492e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 1493e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1494e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1495e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 1496aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * arena_run_alloc_small() failed, but another thread may have made 1497940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * sufficient memory available while this one dropped bin->lock above, 1498e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * so search one more time. 1499e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 1500e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans run = arena_bin_nonfull_run_tryget(bin); 1501e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) 1502e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 1503e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1504e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (NULL); 1505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1506e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 15071e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 1508e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void * 1509e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1510e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1511e00572b384c81bd2aba57fac32f7077a34388915Jason Evans void *ret; 151249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 151349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1514e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_t *run; 1515e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 151649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(arena, bin); 151749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 1518e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->runcur = NULL; 1519e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = arena_bin_nonfull_run_get(arena, bin); 1520e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (bin->runcur != NULL && bin->runcur->nfree > 0) { 1521e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 1522e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * Another thread updated runcur while this one ran without the 1523e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * bin lock in arena_bin_nonfull_run_get(). 1524e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 1525e00572b384c81bd2aba57fac32f7077a34388915Jason Evans assert(bin->runcur->nfree > 0); 152649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_run_reg_alloc(bin->runcur, bin_info); 1527e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run != NULL) { 1528940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_chunk_t *chunk; 1529940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1530940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* 1531aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * arena_run_alloc_small() may have allocated run, or 1532aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * it may have pulled run from the bin's run tree. 1533aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * Therefore it is unsafe to make any assumptions about 1534aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * how run has previously been used, and 1535aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * arena_bin_lower_run() must be called, as if a region 1536aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans * were just deallocated from the run. 1537940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans */ 1538940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 153949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (run->nfree == bin_info->nregs) 15408de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 15418de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans else 15428de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_bin_lower_run(arena, chunk, run, bin); 1543e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1544e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (ret); 1545e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1546e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1547e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run == NULL) 1548e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1549e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1550e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->runcur = run; 1551e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1552e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->runcur->nfree > 0); 1553e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 155449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (arena_run_reg_alloc(bin->runcur, bin_info)); 1555e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1556e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 155786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evansvoid 15587372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansarena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, 15597372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans uint64_t prof_accumbytes) 1560e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1561e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, nfill; 1562e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1563e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1564e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr; 1565e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1566e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tbin->ncached == 0); 1567e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 156888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans if (config_prof && arena_prof_accum(arena, prof_accumbytes)) 156988c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans prof_idump(); 1570e69bee01de62b56d3e585042d341743239568043Jason Evans bin = &arena->bins[binind]; 1571e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 15721dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 15731dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans tbin->lg_fill_div); i < nfill; i++) { 1574e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 157549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1576e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1577e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ptr = arena_bin_malloc_hard(arena, bin); 15783fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (ptr == NULL) 1579e476f8a161d445211fd6e54fe370275196e66bcbJason Evans break; 1580122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (config_fill && opt_junk) { 1581122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_alloc_junk_small(ptr, &arena_bin_info[binind], 1582122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans true); 1583122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 15849c43c13a35220c10d97a886616899189daceb359Jason Evans /* Insert such that low regions get used first. */ 15859c43c13a35220c10d97a886616899189daceb359Jason Evans tbin->avail[nfill - 1 - i] = ptr; 1586e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 15877372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 15887372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated += i * arena_bin_info[binind].reg_size; 15897372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nmalloc += i; 15907372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 15917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nfills++; 15927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tbin->tstats.nrequests = 0; 15937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 159486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1595e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->ncached = i; 1596e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1597e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1598122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid 1599122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansarena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) 1600122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans{ 1601122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 1602122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (zero) { 1603122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t redzone_size = bin_info->redzone_size; 1604122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, 1605122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans redzone_size); 1606122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, 1607122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans redzone_size); 1608122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 1609122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, 1610122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval); 1611122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1612122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans} 1613122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 16140d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#ifdef JEMALLOC_JET 16150d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#undef arena_redzone_corruption 16160d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption_impl) 16170d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#endif 16180d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansstatic void 16190d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansarena_redzone_corruption(void *ptr, size_t usize, bool after, 16200d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans size_t offset, uint8_t byte) 16210d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans{ 16220d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans 16230d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans malloc_printf("<jemalloc>: Corrupt redzone %zu byte%s %s %p " 16240d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s", 16250d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans after ? "after" : "before", ptr, usize, byte); 16260d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans} 16270d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#ifdef JEMALLOC_JET 16280d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#undef arena_redzone_corruption 16296b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) 16306b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansarena_redzone_corruption_t *arena_redzone_corruption = 16316b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans JEMALLOC_N(arena_redzone_corruption_impl); 16320d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#endif 16330d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans 16340d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansstatic void 16350d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansarena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) 1636122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans{ 1637122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t size = bin_info->reg_size; 1638122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t redzone_size = bin_info->redzone_size; 1639122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t i; 1640122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bool error = false; 1641122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 1642122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans for (i = 1; i <= redzone_size; i++) { 16430d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); 16440d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans if (*byte != 0xa5) { 1645122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans error = true; 16460d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans arena_redzone_corruption(ptr, size, false, i, *byte); 16470d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans if (reset) 16480d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans *byte = 0xa5; 1649122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1650122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1651122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans for (i = 0; i < redzone_size; i++) { 16520d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); 16530d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans if (*byte != 0xa5) { 1654122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans error = true; 16550d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans arena_redzone_corruption(ptr, size, true, i, *byte); 16560d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans if (reset) 16570d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans *byte = 0xa5; 1658122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1659122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1660122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (opt_abort && error) 1661122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans abort(); 16620d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans} 16630d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans 16646b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 16656b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#undef arena_dalloc_junk_small 16666b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small_impl) 16676b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 16680d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansvoid 16690d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansarena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) 16700d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans{ 16710d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans size_t redzone_size = bin_info->redzone_size; 1672122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 16730d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans arena_redzones_validate(ptr, bin_info, false); 1674122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, 1675122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval); 1676122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans} 16776b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 16786b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#undef arena_dalloc_junk_small 16796b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) 16806b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansarena_dalloc_junk_small_t *arena_dalloc_junk_small = 16816b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans JEMALLOC_N(arena_dalloc_junk_small_impl); 16826b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 1683122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 16840d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansvoid 16850d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansarena_quarantine_junk_small(void *ptr, size_t usize) 16860d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans{ 16870d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans size_t binind; 16880d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans arena_bin_info_t *bin_info; 16890d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans cassert(config_fill); 16900d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans assert(opt_junk); 16910d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans assert(opt_quarantine); 16920d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans assert(usize <= SMALL_MAXCLASS); 16930d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans 16943541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans binind = small_size2bin(usize); 16950d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans bin_info = &arena_bin_info[binind]; 16960d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans arena_redzones_validate(ptr, bin_info, true); 16970d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans} 16980d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans 1699e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1700e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_small(arena_t *arena, size_t size, bool zero) 1701e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1702e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1703e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1704e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1705e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 1706e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17073541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans binind = small_size2bin(size); 1708b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 1709e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 17103541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans size = small_bin2size(binind); 1711e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 171286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 1713e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 171449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1715e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1716e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_hard(arena, bin); 1717e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1718e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 171986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1720e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1721e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1722e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17237372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 17247372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated += size; 17257372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nmalloc++; 17267372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nrequests++; 17277372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 172886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 172988c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans if (config_prof && isthreaded == false && arena_prof_accum(arena, size)) 173088c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans prof_idump(); 1731e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1732e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 17337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill) { 1734122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (opt_junk) { 1735122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_alloc_junk_small(ret, 1736122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans &arena_bin_info[binind], false); 1737122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else if (opt_zero) 17387372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0, size); 17397372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1740bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1741122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 1742122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (config_fill && opt_junk) { 1743122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_alloc_junk_small(ret, &arena_bin_info[binind], 1744122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans true); 1745122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1746bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); 1747e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1748122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 1749e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1750e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1751e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1752e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1753e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1754e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_large(arena_t *arena, size_t size, bool zero) 1755e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1756e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 175788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans UNUSED bool idump; 1758e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1759e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1760e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = PAGE_CEILING(size); 1761e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1762aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ret = (void *)arena_run_alloc_large(arena, size, zero); 1763e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1764e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1765e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1766e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 17677372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 17687372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 17697372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 17707372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 1771ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1772ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1773ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1774e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 17757372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) 177688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans idump = arena_prof_accum_locked(arena, size); 1777e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 177888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans if (config_prof && idump) 177988c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans prof_idump(); 1780e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1781e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 17827372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill) { 17837372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (opt_junk) 17847372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0xa5, size); 17857372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans else if (opt_zero) 17867372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0, size); 17877372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1788e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1789e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1790e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1791e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1792e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1793e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Only handles large allocations that require more than page alignment. */ 1794e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 17955ff709c264e52651de25b788692c62ff1f6f389cJason Evansarena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) 1796e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1797e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 17985ff709c264e52651de25b788692c62ff1f6f389cJason Evans size_t alloc_size, leadsize, trailsize; 17995ff709c264e52651de25b788692c62ff1f6f389cJason Evans arena_run_t *run; 1800e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1801e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1802e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 180393443689a4367cc6fe3de1c9e918adc13d8f9100Jason Evans 180493443689a4367cc6fe3de1c9e918adc13d8f9100Jason Evans alignment = PAGE_CEILING(alignment); 18055ff709c264e52651de25b788692c62ff1f6f389cJason Evans alloc_size = size + alignment - PAGE; 1806e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1807e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1808aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans run = arena_run_alloc_large(arena, alloc_size, false); 18095ff709c264e52651de25b788692c62ff1f6f389cJason Evans if (run == NULL) { 1810e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1811e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1812e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 18135ff709c264e52651de25b788692c62ff1f6f389cJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1814e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18155ff709c264e52651de25b788692c62ff1f6f389cJason Evans leadsize = ALIGNMENT_CEILING((uintptr_t)run, alignment) - 18165ff709c264e52651de25b788692c62ff1f6f389cJason Evans (uintptr_t)run; 18175ff709c264e52651de25b788692c62ff1f6f389cJason Evans assert(alloc_size >= leadsize + size); 18185ff709c264e52651de25b788692c62ff1f6f389cJason Evans trailsize = alloc_size - leadsize - size; 18195ff709c264e52651de25b788692c62ff1f6f389cJason Evans ret = (void *)((uintptr_t)run + leadsize); 18205ff709c264e52651de25b788692c62ff1f6f389cJason Evans if (leadsize != 0) { 18215ff709c264e52651de25b788692c62ff1f6f389cJason Evans arena_run_trim_head(arena, chunk, run, alloc_size, alloc_size - 18225ff709c264e52651de25b788692c62ff1f6f389cJason Evans leadsize); 18235ff709c264e52651de25b788692c62ff1f6f389cJason Evans } 18245ff709c264e52651de25b788692c62ff1f6f389cJason Evans if (trailsize != 0) { 18255ff709c264e52651de25b788692c62ff1f6f389cJason Evans arena_run_trim_tail(arena, chunk, ret, size + trailsize, size, 18265ff709c264e52651de25b788692c62ff1f6f389cJason Evans false); 1827e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1828aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_init_large(arena, (arena_run_t *)ret, size, zero); 1829e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18307372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 18317372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 18327372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 18337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 1834ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1835ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1836ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1837e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1838e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1839e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18407372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && zero == false) { 18418e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (opt_junk) 18428e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset(ret, 0xa5, size); 18438e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans else if (opt_zero) 18448e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset(ret, 0, size); 18458e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 1846e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1847e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1848e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18490b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansvoid 18500b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansarena_prof_promoted(const void *ptr, size_t size) 18510b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans{ 18520b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans arena_chunk_t *chunk; 18530b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t pageind, binind; 18540b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 185578f7352259768f670f8e1f9b000388dd32b62493Jason Evans cassert(config_prof); 18560b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(ptr != NULL); 18570b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1858122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(isalloc(ptr, false) == PAGE); 1859122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(isalloc(ptr, true) == PAGE); 1860b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(size <= SMALL_MAXCLASS); 18610b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 18620b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1863ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 18643541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans binind = small_size2bin(size); 1865b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 1866203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_binind_set(chunk, pageind, binind); 18670b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 1868122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(isalloc(ptr, false) == PAGE); 1869122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(isalloc(ptr, true) == size); 18700b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans} 18716109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1872e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1873088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansarena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 1874e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin) 1875e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1876e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 187719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Dissociate run from bin. */ 1878e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run == bin->runcur) 1879e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 188049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans else { 188149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, bin); 188249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 188349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 188449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (bin_info->nregs != 1) { 188549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 188649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * This block's conditional is necessary because if the 188749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * run only contains one region, then it never gets 188849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * inserted into the non-full runs tree. 188949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 1890e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_remove(bin, run); 189149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 1892e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1893088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans} 1894088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans 1895088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansstatic void 1896088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansarena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1897088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_bin_t *bin) 1898088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans{ 189949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 190049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1901088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans size_t npages, run_ind, past; 1902088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans 1903088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans assert(run != bin->runcur); 1904203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_run_tree_search(&bin->runs, 1905203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapp_get(chunk, ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) 1906203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans == NULL); 190786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 190849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(chunk->arena, run->bin); 190949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 191049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 1911e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&bin->lock); 1912e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /******************************/ 1913ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans npages = bin_info->run_size >> LG_PAGE; 1914ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 191584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans past = (size_t)(PAGE_CEILING((uintptr_t)run + 191684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * 1917122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval - bin_info->redzone_size) - 1918122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (uintptr_t)chunk) >> LG_PAGE); 191986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 192019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 192119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 192219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * If the run was originally clean, and some pages were never touched, 192319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * trim the clean pages before deallocating the dirty portion of the 192419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run. 192519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 192630fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(arena_mapbits_dirty_get(chunk, run_ind) == 192730fe12b866edbc2cf9aaef299063b392ea125aacJason Evans arena_mapbits_dirty_get(chunk, run_ind+npages-1)); 1928203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind < 1929203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans npages) { 193030fe12b866edbc2cf9aaef299063b392ea125aacJason Evans /* Trim clean pages. Convert to large run beforehand. */ 193130fe12b866edbc2cf9aaef299063b392ea125aacJason Evans assert(npages > 0); 1932d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, run_ind, bin_info->run_size, 0); 1933d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); 1934ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), 1935ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans ((past - run_ind) << LG_PAGE), false); 1936940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* npages = past - run_ind; */ 19371e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans } 1938e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_dalloc(arena, run, true, false); 193986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 1940e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /****************************/ 1941e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&bin->lock); 19427372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 19437372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.curruns--; 1944e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1945e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1946940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansstatic void 1947940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansarena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1948940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_bin_t *bin) 1949e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1950e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19518de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* 1952e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * Make sure that if bin->runcur is non-NULL, it refers to the lowest 1953e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * non-full run. It is okay to NULL runcur out rather than proactively 1954e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * keeping it pointing at the lowest non-full run. 19558de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans */ 1956e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if ((uintptr_t)run < (uintptr_t)bin->runcur) { 19578de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* Switch runcur. */ 1958e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (bin->runcur->nfree > 0) 1959e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_insert(bin, bin->runcur); 19608de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans bin->runcur = run; 1961e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (config_stats) 1962e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans bin->stats.reruns++; 1963e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans } else 1964e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_insert(bin, run); 1965940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans} 1966940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1967940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansvoid 1968203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1969940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_chunk_map_t *mapelm) 1970940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans{ 1971940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t pageind; 1972940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_t *run; 1973940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_bin_t *bin; 19748b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey arena_bin_info_t *bin_info; 19758b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey size_t size, binind; 1976940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1977ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1978940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1979203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 1980940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans bin = run->bin; 1981f9ff60346d7c25ad653ea062e496a5d0864233b2Ben Maurer binind = arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)); 19828b49971d0ce0819af78aa2a278c26ecb298ee134Mike Hommey bin_info = &arena_bin_info[binind]; 19837372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill || config_stats) 19847372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size = bin_info->reg_size; 1985940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 19867372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk) 1987122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans arena_dalloc_junk_small(ptr, bin_info); 1988940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1989940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_reg_dalloc(run, ptr); 199049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (run->nfree == bin_info->nregs) { 1991088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_dissociate_bin_run(chunk, run, bin); 19928de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 1993088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans } else if (run->nfree == 1 && run != bin->runcur) 19948de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_bin_lower_run(arena, chunk, run, bin); 1995e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19967372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 19977372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated -= size; 19987372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.ndalloc++; 19997372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 2000e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2001e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2002e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 2003203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2004203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind, arena_chunk_map_t *mapelm) 2005203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 2006203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_run_t *run; 2007203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_bin_t *bin; 2008203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 2009203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 2010203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); 2011203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans bin = run->bin; 2012203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans malloc_mutex_lock(&bin->lock); 2013203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_bin_locked(arena, chunk, ptr, mapelm); 2014203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans malloc_mutex_unlock(&bin->lock); 2015203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 2016203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 2017203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid 2018203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 2019203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind) 2020203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 2021203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_chunk_map_t *mapelm; 2022203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 2023203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (config_debug) { 202480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans /* arena_ptr_small_binind_get() does extra sanity checking. */ 202580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 202680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind)) != BININD_INVALID); 2027203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 2028203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapelm = arena_mapp_get(chunk, pageind); 2029203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); 2030203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 2031e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 20326b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 20336b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#undef arena_dalloc_junk_large 20346b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl) 20356b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 20366b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansstatic void 20376b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansarena_dalloc_junk_large(void *ptr, size_t usize) 20386b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans{ 20396b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans 20406b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans if (config_fill && opt_junk) 20416b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans memset(ptr, 0x5a, usize); 20426b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans} 20436b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 20446b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#undef arena_dalloc_junk_large 20456b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) 20466b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansarena_dalloc_junk_large_t *arena_dalloc_junk_large = 20476b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans JEMALLOC_N(arena_dalloc_junk_large_impl); 20486b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 20496b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans 2050e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 2051203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) 2052e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 205313668262d17fb5950e2441bc9d56a15db9c93877Jason Evans 20547372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill || config_stats) { 2055ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 20566b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans size_t usize = arena_mapbits_large_size_get(chunk, pageind); 2057e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 20586b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans arena_dalloc_junk_large(ptr, usize); 20597372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 20607372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 20616b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans arena->stats.allocated_large -= usize; 20626b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans arena->stats.lstats[(usize >> LG_PAGE) - 1].ndalloc++; 20636b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans arena->stats.lstats[(usize >> LG_PAGE) - 1].curruns--; 20647372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 2065e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2066e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2067e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_run_dalloc(arena, (arena_run_t *)ptr, true, false); 2068e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2069e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2070203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid 2071203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 2072203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 2073203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 2074203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans malloc_mutex_lock(&arena->lock); 2075203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_large_locked(arena, chunk, ptr); 2076203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans malloc_mutex_unlock(&arena->lock); 2077203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 2078203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 2079e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 2080e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 20818e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t oldsize, size_t size) 2082e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2083e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2084e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size < oldsize); 2085e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2086e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2087e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Shrink the run, and make trailing pages available for other 2088e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocations. 2089e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2090e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 2091e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 2092e476f8a161d445211fd6e54fe370275196e66bcbJason Evans true); 20937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 20947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 20957372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= oldsize; 2096ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 2097ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 20987372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 20997372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 21007372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 21017372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 2102ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 2103ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 2104ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 2105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 2107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2109e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 2110e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 21118e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t oldsize, size_t size, size_t extra, bool zero) 2112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2113ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 2114ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t npages = oldsize >> LG_PAGE; 21158e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t followsize; 2116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2117203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(oldsize == arena_mapbits_large_size_get(chunk, pageind)); 2118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to extend the run. */ 21208e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans assert(size + extra > oldsize); 2121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 21227393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if (pageind + npages < chunk_npages && 2123203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && 2124203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (followsize = arena_mapbits_unallocated_size_get(chunk, 2125203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans pageind+npages)) >= size - oldsize) { 2126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The next run is available and sufficiently large. Split the 2128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * following run, then merge the first part with the existing 2129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocation. 2130e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2131940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t flag_dirty; 21328e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t splitsize = (oldsize + followsize <= size + extra) 21338e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ? followsize : size + extra - oldsize; 2134aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans arena_run_split_large(arena, (arena_run_t *)((uintptr_t)chunk + 2135aa5113b1fdafd1129c22512837c6c3d66c295fc8Jason Evans ((pageind+npages) << LG_PAGE)), splitsize, zero); 2136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2137088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans size = oldsize + splitsize; 2138ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans npages = size >> LG_PAGE; 2139e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2140940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* 2141940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * Mark the extended run as dirty if either portion of the run 2142940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * was dirty before allocation. This is rather pedantic, 2143940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * because there's not actually any sequence of events that 2144940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * could cause the resulting run to be passed to 2145940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * arena_run_dalloc() with the dirty argument set to false 2146940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * (which is when dirty flag consistency would really matter). 2147940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans */ 2148203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | 2149203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_dirty_get(chunk, pageind+npages-1); 2150203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_set(chunk, pageind, size, flag_dirty); 2151203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty); 2152990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans 21537372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 21547372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 21557372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= oldsize; 2156203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 2157203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 21587372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 21597372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 21607372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 21617372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 2162ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 2163203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 2164ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 2165940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 2166e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 2167e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2168e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2169e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 2170e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2171e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2172e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 21746b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 21756b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#undef arena_ralloc_junk_large 21766b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large_impl) 21776b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 21786b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansstatic void 21796b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansarena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) 21806b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans{ 21816b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans 21826b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans if (config_fill && opt_junk) { 21836b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans memset((void *)((uintptr_t)ptr + usize), 0x5a, 21846b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans old_usize - usize); 21856b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans } 21866b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans} 21876b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 21886b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#undef arena_ralloc_junk_large 21896b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) 21906b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansarena_ralloc_junk_large_t *arena_ralloc_junk_large = 21916b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans JEMALLOC_N(arena_ralloc_junk_large_impl); 21926b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 21936b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans 2194e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 2195e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Try to resize a large allocation, in order to avoid copying. This will 2196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * always fail if growing an object, and the following run is already in use. 2197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2198e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 21998e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, 22008e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 2201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t psize; 2203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 22048e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize = PAGE_CEILING(size + extra); 2205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize == oldsize) { 2206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Same size class. */ 2207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2208e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 2209e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 2210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 2211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2212e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 2213e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 2214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2215e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize < oldsize) { 2216e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Fill before shrinking in order avoid a race. */ 22176b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans arena_ralloc_junk_large(ptr, oldsize, psize); 22188e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, 22198e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize); 2220e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 2222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 22238e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans oldsize, PAGE_CEILING(size), 22248e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize - PAGE_CEILING(size), zero); 22256b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans if (config_fill && ret == false && zero == false) { 22266b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans if (opt_junk) { 22276b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans memset((void *)((uintptr_t)ptr + 22286b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans oldsize), 0xa5, isalloc(ptr, 22296b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans config_prof) - oldsize); 22306b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans } else if (opt_zero) { 22316b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans memset((void *)((uintptr_t)ptr + 22326b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans oldsize), 0, isalloc(ptr, 22336b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans config_prof) - oldsize); 22346b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans } 2235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 2237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2239e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2240e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2241b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evansbool 22428e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, 22438e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 2244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 22468e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* 22478e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * Avoid moving the allocation if the size class can be left the same. 22488e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans */ 2249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= arena_maxclass) { 2250b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (oldsize <= SMALL_MAXCLASS) { 22513541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans assert(arena_bin_info[small_size2bin(oldsize)].reg_size 225249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans == oldsize); 2253b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if ((size + extra <= SMALL_MAXCLASS && 22543541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans small_size2bin(size + extra) == 22553541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans small_size2bin(oldsize)) || (size <= oldsize && 22566e62984ef6ca4312cf0a2e49ea2cc38feb94175bJason Evans size + extra >= oldsize)) 2257b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evans return (false); 2258e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 2259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 2260b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (size + extra > SMALL_MAXCLASS) { 22618e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (arena_ralloc_large(ptr, oldsize, size, 22628e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans extra, zero) == false) 2263b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evans return (false); 2264e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2265e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2266e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2267e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 22688e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Reallocation would require a move. */ 2269b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evans return (true); 22708e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans} 22718e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 22728e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansvoid * 2273609ae595f0358157b19311b0f9f9591db7cee705Jason Evansarena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, 2274609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, 2275609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bool try_tcache_dalloc) 22768e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans{ 22778e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ret; 22788e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t copysize; 22798e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 22808e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Try to avoid moving the allocation. */ 2281b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evans if (arena_ralloc_no_move(ptr, oldsize, size, extra, zero) == false) 2282b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evans return (ptr); 22838e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 2284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 22858e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * size and oldsize are different enough that we need to move the 22868e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * object. In that case, fall back to allocating new space and 22878e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * copying. 2288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 228938d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (alignment != 0) { 22905ff709c264e52651de25b788692c62ff1f6f389cJason Evans size_t usize = sa2u(size + extra, alignment); 229138d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (usize == 0) 229238d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans return (NULL); 2293d82a5e6a34f20698ab9368bb2b4953b81d175552Jason Evans ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); 229438d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans } else 2295609ae595f0358157b19311b0f9f9591db7cee705Jason Evans ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc); 2296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 22978e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == NULL) { 22988e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (extra == 0) 22998e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 23008e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Try again, this time without extra. */ 230138d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (alignment != 0) { 23025ff709c264e52651de25b788692c62ff1f6f389cJason Evans size_t usize = sa2u(size, alignment); 230338d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (usize == 0) 230438d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans return (NULL); 2305d82a5e6a34f20698ab9368bb2b4953b81d175552Jason Evans ret = ipalloct(usize, alignment, zero, try_tcache_alloc, 2306609ae595f0358157b19311b0f9f9591db7cee705Jason Evans arena); 230738d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans } else 2308609ae595f0358157b19311b0f9f9591db7cee705Jason Evans ret = arena_malloc(arena, size, zero, try_tcache_alloc); 23098e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 23108e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == NULL) 23118e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 23128e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 23138e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 23148e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ 23158e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 23168e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* 23178e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * Copy at most size bytes (not size+extra), since the caller has no 23188e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * expectation that the extra bytes will be reliably preserved. 23198e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans */ 2320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans copysize = (size < oldsize) ? size : oldsize; 2321bd87b01999416ec7418ff8bdb504d9b6c009ff68Jason Evans JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); 2322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(ret, ptr, copysize); 2323d82a5e6a34f20698ab9368bb2b4953b81d175552Jason Evans iqalloct(ptr, try_tcache_dalloc); 2324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 2325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2327609ae595f0358157b19311b0f9f9591db7cee705Jason Evansdss_prec_t 2328609ae595f0358157b19311b0f9f9591db7cee705Jason Evansarena_dss_prec_get(arena_t *arena) 2329609ae595f0358157b19311b0f9f9591db7cee705Jason Evans{ 2330609ae595f0358157b19311b0f9f9591db7cee705Jason Evans dss_prec_t ret; 2331609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2332609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&arena->lock); 2333609ae595f0358157b19311b0f9f9591db7cee705Jason Evans ret = arena->dss_prec; 2334609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&arena->lock); 2335609ae595f0358157b19311b0f9f9591db7cee705Jason Evans return (ret); 2336609ae595f0358157b19311b0f9f9591db7cee705Jason Evans} 2337609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 23384d434adb146375ad17f0d5e994ed5728d2942e3fJason Evansbool 2339609ae595f0358157b19311b0f9f9591db7cee705Jason Evansarena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) 2340609ae595f0358157b19311b0f9f9591db7cee705Jason Evans{ 2341609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 23424d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans if (have_dss == false) 23434d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans return (dss_prec != dss_prec_disabled); 2344609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&arena->lock); 2345609ae595f0358157b19311b0f9f9591db7cee705Jason Evans arena->dss_prec = dss_prec; 2346609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&arena->lock); 23474d434adb146375ad17f0d5e994ed5728d2942e3fJason Evans return (false); 2348609ae595f0358157b19311b0f9f9591db7cee705Jason Evans} 2349609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2350609ae595f0358157b19311b0f9f9591db7cee705Jason Evansvoid 2351609ae595f0358157b19311b0f9f9591db7cee705Jason Evansarena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, 2352609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 2353609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_large_stats_t *lstats) 2354609ae595f0358157b19311b0f9f9591db7cee705Jason Evans{ 2355609ae595f0358157b19311b0f9f9591db7cee705Jason Evans unsigned i; 2356609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2357609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&arena->lock); 2358609ae595f0358157b19311b0f9f9591db7cee705Jason Evans *dss = dss_prec_names[arena->dss_prec]; 2359609ae595f0358157b19311b0f9f9591db7cee705Jason Evans *nactive += arena->nactive; 2360609ae595f0358157b19311b0f9f9591db7cee705Jason Evans *ndirty += arena->ndirty; 2361609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2362609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->mapped += arena->stats.mapped; 2363609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->npurge += arena->stats.npurge; 2364609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->nmadvise += arena->stats.nmadvise; 2365609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->purged += arena->stats.purged; 2366609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->allocated_large += arena->stats.allocated_large; 2367609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->nmalloc_large += arena->stats.nmalloc_large; 2368609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->ndalloc_large += arena->stats.ndalloc_large; 2369609ae595f0358157b19311b0f9f9591db7cee705Jason Evans astats->nrequests_large += arena->stats.nrequests_large; 237012141150fdbda57651a53ae2fe0edaea4891d814Jason Evans astats->allocated_huge += arena->stats.allocated_huge; 237112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans astats->nmalloc_huge += arena->stats.nmalloc_huge; 237212141150fdbda57651a53ae2fe0edaea4891d814Jason Evans astats->ndalloc_huge += arena->stats.ndalloc_huge; 237312141150fdbda57651a53ae2fe0edaea4891d814Jason Evans astats->nrequests_huge += arena->stats.nrequests_huge; 2374609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2375609ae595f0358157b19311b0f9f9591db7cee705Jason Evans for (i = 0; i < nlclasses; i++) { 2376609ae595f0358157b19311b0f9f9591db7cee705Jason Evans lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 2377609ae595f0358157b19311b0f9f9591db7cee705Jason Evans lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 2378609ae595f0358157b19311b0f9f9591db7cee705Jason Evans lstats[i].nrequests += arena->stats.lstats[i].nrequests; 2379609ae595f0358157b19311b0f9f9591db7cee705Jason Evans lstats[i].curruns += arena->stats.lstats[i].curruns; 2380609ae595f0358157b19311b0f9f9591db7cee705Jason Evans } 2381609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&arena->lock); 2382609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2383609ae595f0358157b19311b0f9f9591db7cee705Jason Evans for (i = 0; i < NBINS; i++) { 2384609ae595f0358157b19311b0f9f9591db7cee705Jason Evans arena_bin_t *bin = &arena->bins[i]; 2385609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2386609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_lock(&bin->lock); 2387609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].allocated += bin->stats.allocated; 2388609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nmalloc += bin->stats.nmalloc; 2389609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].ndalloc += bin->stats.ndalloc; 2390609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nrequests += bin->stats.nrequests; 2391609ae595f0358157b19311b0f9f9591db7cee705Jason Evans if (config_tcache) { 2392609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nfills += bin->stats.nfills; 2393609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nflushes += bin->stats.nflushes; 2394609ae595f0358157b19311b0f9f9591db7cee705Jason Evans } 2395609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].nruns += bin->stats.nruns; 2396609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].reruns += bin->stats.reruns; 2397609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bstats[i].curruns += bin->stats.curruns; 2398609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_mutex_unlock(&bin->lock); 2399609ae595f0358157b19311b0f9f9591db7cee705Jason Evans } 2400609ae595f0358157b19311b0f9f9591db7cee705Jason Evans} 2401609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2402e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 2403e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_new(arena_t *arena, unsigned ind) 2404e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 2406e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 2407e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 24086109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena->ind = ind; 2409597632be188d2bcc135dad2145cc46ef44897aadJason Evans arena->nthreads = 0; 241059113bcc94b9fc7549611afb99ca99cad1a7f196aravind arena->chunk_alloc = chunk_alloc_default; 241112141150fdbda57651a53ae2fe0edaea4891d814Jason Evans arena->chunk_dalloc = chunk_dalloc_default; 24126109fe07a14b7a619365977d9523db9f8b333792Jason Evans 2413e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (malloc_mutex_init(&arena->lock)) 2414e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 2415e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 24167372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 24177372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(&arena->stats, 0, sizeof(arena_stats_t)); 24187372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats = 24197372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans (malloc_large_stats_t *)base_alloc(nlclasses * 24207372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans sizeof(malloc_large_stats_t)); 24217372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (arena->stats.lstats == NULL) 24227372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans return (true); 24237372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(arena->stats.lstats, 0, nlclasses * 24247372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans sizeof(malloc_large_stats_t)); 24257372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_tcache) 24267372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ql_new(&arena->tcache_ql); 24277372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 2428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 24297372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) 24307372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->prof_accumbytes = 0; 2431d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 2432609ae595f0358157b19311b0f9f9591db7cee705Jason Evans arena->dss_prec = chunk_dss_prec_get(); 2433609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 2434e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize chunks. */ 2435e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_dirty_new(&arena->chunks_dirty); 2436e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 2437e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2438e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive = 0; 2439e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty = 0; 2440799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory = 0; 2441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2442e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_new(&arena->runs_avail); 2443e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize bins. */ 2445b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans for (i = 0; i < NBINS; i++) { 2446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 244786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 244886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 2449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 2450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 24517372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 24527372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2453e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2454e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2455e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2456e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2457e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 245849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 245949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Calculate bin_info->run_size such that it meets the following constraints: 246049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 246149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) bin_info->run_size >= min_run_size 246249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) bin_info->run_size <= arena_maxclass 246349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 246447e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans * *) bin_info->nregs <= RUN_MAXREGS 246549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 246684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also 246784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * calculated here, since these settings are all interdependent. 246849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 246949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstatic size_t 247049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansbin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) 247149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 2472122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t pad_size; 247349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t try_run_size, good_run_size; 247449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_nregs, good_nregs; 247549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_hdr_size, good_hdr_size; 247684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans uint32_t try_bitmap_offset, good_bitmap_offset; 2477122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans uint32_t try_redzone0_offset, good_redzone0_offset; 247849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2479ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans assert(min_run_size >= PAGE); 248049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(min_run_size <= arena_maxclass); 248149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 248249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 2483122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Determine redzone size based on minimum alignment and minimum 2484122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * redzone size. Add padding to the end of the run if it is needed to 2485122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * align the regions. The padding allows each redzone to be half the 2486122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * minimum alignment; without the padding, each redzone would have to 2487122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * be twice as large in order to maintain alignment. 2488122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans */ 2489122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (config_fill && opt_redzone) { 24902aa7fed9c983d8dcde7c0cacf1b024c966758b88Richard Diamond size_t align_min = ZU(1) << (jemalloc_ffs(bin_info->reg_size) - 1); 2491122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (align_min <= REDZONE_MINSIZE) { 2492122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->redzone_size = REDZONE_MINSIZE; 2493122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans pad_size = 0; 2494122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 2495122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->redzone_size = align_min >> 1; 2496122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans pad_size = bin_info->redzone_size; 2497122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 2498122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else { 2499122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->redzone_size = 0; 2500122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans pad_size = 0; 2501122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } 2502122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval = bin_info->reg_size + 2503122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans (bin_info->redzone_size << 1); 2504122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 2505122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* 250649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Calculate known-valid settings before entering the run_size 250749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * expansion loop, so that the first part of the loop always copies 250849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * valid settings. 250949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 251049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * The do..while loop iteratively reduces the number of regions until 251149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * the run header and the regions no longer overlap. A closed formula 251249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * would be quite messy, since there is an interdependency between the 251349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * header's mask length and the number of regions. 251449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 251549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_run_size = min_run_size; 2516122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / 2517122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval) 251849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 251947e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans if (try_nregs > RUN_MAXREGS) { 252047e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans try_nregs = RUN_MAXREGS 252147e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 252247e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans } 252349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 252449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs--; 252549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = sizeof(arena_run_t); 252684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Pad to a long boundary. */ 252784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size = LONG_CEILING(try_hdr_size); 252884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_bitmap_offset = try_hdr_size; 252984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Add space for bitmap. */ 253084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size += bitmap_size(try_nregs); 2531122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans try_redzone0_offset = try_run_size - (try_nregs * 2532122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval) - pad_size; 2533122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } while (try_hdr_size > try_redzone0_offset); 253449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 253549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* run_size expansion loop. */ 253649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 253749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 253849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Copy valid settings before trying more aggressive settings. 253949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 254049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_run_size = try_run_size; 254149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_nregs = try_nregs; 254249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_hdr_size = try_hdr_size; 254384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans good_bitmap_offset = try_bitmap_offset; 2544122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans good_redzone0_offset = try_redzone0_offset; 254549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 254649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Try more aggressive settings. */ 2547ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans try_run_size += PAGE; 2548122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) / 2549122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval) 255049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 255147e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans if (try_nregs > RUN_MAXREGS) { 255247e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans try_nregs = RUN_MAXREGS 255347e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 255447e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans } 255549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 255649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs--; 255749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = sizeof(arena_run_t); 255884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Pad to a long boundary. */ 255984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size = LONG_CEILING(try_hdr_size); 256084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_bitmap_offset = try_hdr_size; 256184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Add space for bitmap. */ 256284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size += bitmap_size(try_nregs); 2563122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans try_redzone0_offset = try_run_size - (try_nregs * 2564122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg_interval) - pad_size; 2565122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } while (try_hdr_size > try_redzone0_offset); 256649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } while (try_run_size <= arena_maxclass 2567122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > 2568122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans RUN_MAX_OVRHD_RELAX 2569122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size 257047e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans && try_nregs < RUN_MAXREGS); 257149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2572122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(good_hdr_size <= good_redzone0_offset); 257349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 257449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Copy final settings. */ 257549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->run_size = good_run_size; 257649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->nregs = good_nregs; 257784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bin_info->bitmap_offset = good_bitmap_offset; 2578122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; 2579122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 2580122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs 2581122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * bin_info->reg_interval) + pad_size == bin_info->run_size); 258249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 258349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (good_run_size); 258449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 258549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2586b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansstatic void 258749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansbin_info_init(void) 258849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 258949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 2590ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t prev_run_size = PAGE; 2591b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans 25928d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define BIN_INFO_INIT_bin_yes(index, size) \ 25938d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans bin_info = &arena_bin_info[index]; \ 2594b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info->reg_size = size; \ 2595b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ 2596b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 25978d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define BIN_INFO_INIT_bin_no(index, size) 25988d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ 25998d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans BIN_INFO_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)) 2600b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans SIZE_CLASSES 26018d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef BIN_INFO_INIT_bin_yes 26028d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef BIN_INFO_INIT_bin_no 26038d8f9aeeaa77514d5732db5bd0111232af21fcfdJason Evans#undef SC 260449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 260549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2606b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansvoid 2607a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evansarena_boot(void) 2608e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2609a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans size_t header_size; 26107393f44ff025ca67716fc53b68003fd65122fd97Jason Evans unsigned i; 2611e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2612e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2613e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Compute the header size such that it is large enough to contain the 26147393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * page map. The page map is biased to omit entries for the header 26157393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * itself, so some iteration is necessary to compute the map bias. 26167393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 26177393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 1) Compute safe header_size and map_bias values that include enough 26187393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * space for an unbiased page map. 26197393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 2) Refine map_bias based on (1) to omit the header pages in the page 26207393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * map. The resulting map_bias may be one too small. 26217393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 3) Refine map_bias based on (2). The result will be >= the result 26227393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * from (2), and will always be correct. 2623e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 26247393f44ff025ca67716fc53b68003fd65122fd97Jason Evans map_bias = 0; 26257393f44ff025ca67716fc53b68003fd65122fd97Jason Evans for (i = 0; i < 3; i++) { 2626ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans header_size = offsetof(arena_chunk_t, map) + 2627ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); 2628ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) 2629ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans != 0); 26307393f44ff025ca67716fc53b68003fd65122fd97Jason Evans } 26317393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(map_bias > 0); 26327393f44ff025ca67716fc53b68003fd65122fd97Jason Evans 2633ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena_maxclass = chunksize - (map_bias << LG_PAGE); 2634a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans 2635b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info_init(); 2636e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 26374e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 26384e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 26394e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansarena_prefork(arena_t *arena) 26404e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 26414e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans unsigned i; 26424e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 26434e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_prefork(&arena->lock); 26444e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans for (i = 0; i < NBINS; i++) 26454e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_prefork(&arena->bins[i].lock); 26464e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 26474e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 26484e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 26494e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansarena_postfork_parent(arena_t *arena) 26504e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 26514e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans unsigned i; 26524e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 26534e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans for (i = 0; i < NBINS; i++) 26544e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_parent(&arena->bins[i].lock); 26554e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_parent(&arena->lock); 26564e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 26574e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 26584e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 26594e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansarena_postfork_child(arena_t *arena) 26604e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 26614e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans unsigned i; 26624e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 26634e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans for (i = 0; i < NBINS; i++) 26644e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_child(&arena->bins[i].lock); 26654e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_child(&arena->lock); 26664e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 2667