arena.c revision ae4c7b4b4092906c641d69b4bf9fcb4a7d50790d
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_ARENA_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evansssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 8b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansarena_bin_info_t arena_bin_info[NBINS]; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 10b172610317babc7f365584ddd7fdaf4eb8d9d04cJason EvansJEMALLOC_ATTR(aligned(CACHELINE)) 11b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansconst uint8_t small_size2bin[] = { 1241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#define S2B_8(i) i, 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_16(i) S2B_8(i) S2B_8(i) 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_32(i) S2B_16(i) S2B_16(i) 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_64(i) S2B_32(i) S2B_32(i) 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_128(i) S2B_64(i) S2B_64(i) 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_256(i) S2B_128(i) S2B_128(i) 18b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_512(i) S2B_256(i) S2B_256(i) 19b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_1024(i) S2B_512(i) S2B_512(i) 20b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_2048(i) S2B_1024(i) S2B_1024(i) 21b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_4096(i) S2B_2048(i) S2B_2048(i) 22b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_8192(i) S2B_4096(i) S2B_4096(i) 23b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define SIZE_CLASS(bin, delta, size) \ 24b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans S2B_##delta(bin) 25b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans SIZE_CLASSES 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_8 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_16 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_32 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_64 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_128 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_256 32b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_512 33b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_1024 34b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_2048 35b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_4096 36b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_8192 37b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef SIZE_CLASS 38b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans}; 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 41e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Function prototypes for non-inline static functions. */ 42e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 43e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, 44e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool large, bool zero); 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t *arena_chunk_alloc(arena_t *arena); 46e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool zero); 496005f0710cf07d60659d91b20b7ff5592d310027Jason Evansstatic void arena_purge(arena_t *arena, bool all); 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty); 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize); 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); 55e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t *arena_bin_runs_first(arena_bin_t *bin); 56e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run); 57e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run); 58e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t *arena_bin_nonfull_run_tryget(arena_bin_t *bin); 59e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); 61088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansstatic void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 62088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_bin_t *bin); 63e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 64e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, arena_bin_t *bin); 65940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansstatic void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 66940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_t *run, arena_bin_t *bin); 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, 688e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ptr, size_t oldsize, size_t size); 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, 708e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); 718e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansstatic bool arena_ralloc_large(void *ptr, size_t oldsize, size_t size, 728e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t extra, bool zero); 7349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstatic size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, 7449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t min_run_size); 75b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansstatic void bin_info_init(void); 76e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm = (uintptr_t)a; 83e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t b_mapelm = (uintptr_t)b; 84e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 85e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(a != NULL); 86e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(b != NULL); 87e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 89e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 90e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 91f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansrb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, 937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans u.rb_link, arena_run_comp) 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 95e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 98e476f8a161d445211fd6e54fe370275196e66bcbJason Evans int ret; 99e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t a_size = a->bits & ~PAGE_MASK; 100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t b_size = b->bits & ~PAGE_MASK; 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1020b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert((a->bits & CHUNK_MAP_KEY) == CHUNK_MAP_KEY || (a->bits & 1030b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans CHUNK_MAP_DIRTY) == (b->bits & CHUNK_MAP_DIRTY)); 10419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_size > b_size) - (a_size < b_size); 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == 0) { 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm, b_mapelm; 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((a->bits & CHUNK_MAP_KEY) != CHUNK_MAP_KEY) 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = (uintptr_t)a; 111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Treat keys as though they are lower than anything 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * else. 115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = 0; 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans b_mapelm = (uintptr_t)b; 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 126f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 1277372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansrb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, 1287372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans u.rb_link, arena_avail_comp) 129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 130e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void * 13149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 13484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned regind; 13584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 13684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1381e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(run->nfree > 0); 13984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); 140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 14184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); 14284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + 14384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)(bin_info->reg_size * regind)); 1441e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree--; 14584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans if (regind == run->nextind) 14684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans run->nextind++; 14784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(regind < run->nextind); 1481e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans return (ret); 1496109fe07a14b7a619365977d9523db9f8b333792Jason Evans} 1506109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1516109fe07a14b7a619365977d9523db9f8b333792Jason Evansstatic inline void 1521e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evansarena_run_reg_dalloc(arena_run_t *run, void *ptr) 1536109fe07a14b7a619365977d9523db9f8b333792Jason Evans{ 15449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 15549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, run->bin); 15649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 15784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned regind = arena_run_regind(run, bin_info, ptr); 15884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 15984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 16084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 16149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(run->nfree < bin_info->nregs); 1621e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Freeing an interior pointer can cause assertion failure. */ 1631e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 16449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset)) % (uintptr_t)bin_info->reg_size 1651e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans == 0); 16621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans assert((uintptr_t)ptr >= (uintptr_t)run + 16749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset); 16884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Freeing an unallocated pointer can cause assertion failure. */ 16984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); 170e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_unset(bitmap, &bin_info->bitmap_info, regind); 1721e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree++; 173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17521fb95bba6ea922e0523f269c0d9a32640047a29Jason Evansstatic inline void 17621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evansarena_chunk_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 17721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans{ 178d4bab21756279db540866998099522dbd39c05f7Jason Evans size_t i; 179ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); 180d4bab21756279db540866998099522dbd39c05f7Jason Evans 181ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans for (i = 0; i < PAGE / sizeof(size_t); i++) 18221fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans assert(p[i] == 0); 18321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans} 18421fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans 185e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 186e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, 187e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool zero) 188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 190bdcadf41e961a3c6bbb37d8d24e4b68a27f2b952Jason Evans size_t run_ind, total_pages, need_pages, rem_pages, i; 19119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t flag_dirty; 19219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t *runs_avail; 193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 194e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 195ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 1967393f44ff025ca67716fc53b68003fd65122fd97Jason Evans flag_dirty = chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY; 19719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans runs_avail = (flag_dirty != 0) ? &arena->runs_avail_dirty : 19819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans &arena->runs_avail_clean; 1997393f44ff025ca67716fc53b68003fd65122fd97Jason Evans total_pages = (chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) >> 200ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE; 2017393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert((chunk->map[run_ind+total_pages-1-map_bias].bits & 2027393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_DIRTY) == flag_dirty); 203ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans need_pages = (size >> LG_PAGE); 204e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages > 0); 205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages <= total_pages); 206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans rem_pages = total_pages - need_pages; 207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2087393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_remove(runs_avail, &chunk->map[run_ind-map_bias]); 2097372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 2107372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans /* 2117372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * Update stats_cactive if nactive is crossing a chunk 2127372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * multiple. 2137372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans */ 2147372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t cactive_diff = CHUNK_CEILING((arena->nactive + 215ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans need_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive << 216ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE); 2177372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (cactive_diff != 0) 2187372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans stats_cactive_add(cactive_diff); 2197372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 220e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive += need_pages; 221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Keep track of trailing unused pages for later use. */ 223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (rem_pages > 0) { 22419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (flag_dirty != 0) { 2257393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+need_pages-map_bias].bits = 226ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (rem_pages << LG_PAGE) | CHUNK_MAP_DIRTY; 2277393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+total_pages-1-map_bias].bits = 228ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (rem_pages << LG_PAGE) | CHUNK_MAP_DIRTY; 22919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 2307393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+need_pages-map_bias].bits = 231ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (rem_pages << LG_PAGE) | 2327393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+need_pages-map_bias].bits & 2333377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans CHUNK_MAP_UNZEROED); 2347393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+total_pages-1-map_bias].bits = 235ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (rem_pages << LG_PAGE) | 2367393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+total_pages-1-map_bias].bits & 2373377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans CHUNK_MAP_UNZEROED); 23819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 23919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_insert(runs_avail, 2407393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[run_ind+need_pages-map_bias]); 241e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 24319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Update dirty page accounting. */ 24419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (flag_dirty != 0) { 24519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans chunk->ndirty -= need_pages; 24619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena->ndirty -= need_pages; 24719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 24819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 24919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 25019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Update the page map separately for large vs. small runs, since it is 25119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * possible to avoid iteration for large mallocs. 25219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 25319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (large) { 254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero) { 25519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (flag_dirty == 0) { 25619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 25719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * The run is clean, so some pages may be 25819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * zeroed (i.e. never before touched). 25919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 26019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans for (i = 0; i < need_pages; i++) { 2617393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[run_ind+i-map_bias].bits 2623377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans & CHUNK_MAP_UNZEROED) != 0) { 26319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans memset((void *)((uintptr_t) 26421fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans chunk + ((run_ind+i) << 265ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)), 0, PAGE); 2667372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } else if (config_debug) { 26721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed( 26821fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans chunk, run_ind+i); 269940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 27019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 27119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 27219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 27319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * The run is dirty, so all pages must be 27419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * zeroed. 27519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 27619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans memset((void *)((uintptr_t)chunk + (run_ind << 277ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)), 0, (need_pages << LG_PAGE)); 278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 279e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 28219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Set the last element first, in case the run only contains one 28319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * page (i.e. both statements set the same element). 284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2857393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+need_pages-1-map_bias].bits = 2867393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED | flag_dirty; 287e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans chunk->map[run_ind-map_bias].bits = size | flag_dirty | 288e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 28919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 29019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans assert(zero == false); 29119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 292940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * Propagate the dirty and unzeroed flags to the allocated 293940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * small run, so that arena_dalloc_bin_run() has the ability to 294940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * conditionally trim clean pages. 29519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 296397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans chunk->map[run_ind-map_bias].bits = 297397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) | 298397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans CHUNK_MAP_ALLOCATED | flag_dirty; 29921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans /* 30021fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans * The first page will always be dirtied during small run 30121fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans * initialization, so a validation failure here would not 30221fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans * actually cause an observable failure. 30321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans */ 3047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug && flag_dirty == 0 && 30521fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) 30621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans == 0) 30721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed(chunk, run_ind); 30819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans for (i = 1; i < need_pages - 1; i++) { 309ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans chunk->map[run_ind+i-map_bias].bits = (i << LG_PAGE) 310397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans | (chunk->map[run_ind+i-map_bias].bits & 311397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED; 3127372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug && flag_dirty == 0 && 31321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans (chunk->map[run_ind+i-map_bias].bits & 31421fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans CHUNK_MAP_UNZEROED) == 0) 31521fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed(chunk, run_ind+i); 31619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 317004ed142a66529ecf4a55e8f4fa42ff2e535f586Jason Evans chunk->map[run_ind+need_pages-1-map_bias].bits = ((need_pages 318ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans - 1) << LG_PAGE) | 319397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans (chunk->map[run_ind+need_pages-1-map_bias].bits & 320397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED | flag_dirty; 3217372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug && flag_dirty == 0 && 32221fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans (chunk->map[run_ind+need_pages-1-map_bias].bits & 32321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans CHUNK_MAP_UNZEROED) == 0) { 32421fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed(chunk, 32521fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans run_ind+need_pages-1); 32621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans } 327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 328e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 329e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 330e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t * 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_alloc(arena_t *arena) 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare != NULL) { 33719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t *runs_avail; 33819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena->spare; 340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 34119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 34219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Insert the run into the appropriate runs_avail_* tree. */ 3437393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) 34419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans runs_avail = &arena->runs_avail_clean; 34519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans else 34619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans runs_avail = &arena->runs_avail_dirty; 3478de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[0].bits & ~PAGE_MASK) == arena_maxclass); 3488de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[chunk_npages-1-map_bias].bits & ~PAGE_MASK) 3498de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans == arena_maxclass); 3508de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 3518de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (chunk->map[chunk_npages-1-map_bias].bits & 3528de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_DIRTY)); 3537393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_insert(runs_avail, &chunk->map[0]); 354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 35541631d00618d7262125e501c91d31b4d70e605faJason Evans bool zero; 3563377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans size_t unzeroed; 35741631d00618d7262125e501c91d31b4d70e605faJason Evans 35841631d00618d7262125e501c91d31b4d70e605faJason Evans zero = false; 359e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&arena->lock); 3602dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans chunk = (arena_chunk_t *)chunk_alloc(chunksize, false, &zero); 361e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&arena->lock); 362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk == NULL) 363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 3647372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 3657372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.mapped += chunksize; 366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->arena = arena; 3682caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_elm_new(chunk, link_dirty); 369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = false; 370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Claim that no pages are in use, since the header is merely 373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * overhead. 374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty = 0; 376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Initialize the map to contain one maximal free untouched run. 37941631d00618d7262125e501c91d31b4d70e605faJason Evans * Mark the pages as zeroed iff chunk_alloc() returned a zeroed 38041631d00618d7262125e501c91d31b4d70e605faJason Evans * chunk. 381e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 3823377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; 3833377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans chunk->map[0].bits = arena_maxclass | unzeroed; 3843377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans /* 3853377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * There is no need to initialize the internal page map entries 3863377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * unless the chunk is not zeroed. 3873377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans */ 3883377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans if (zero == false) { 3893377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans for (i = map_bias+1; i < chunk_npages-1; i++) 3903377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans chunk->map[i-map_bias].bits = unzeroed; 3917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } else if (config_debug) { 392940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans for (i = map_bias+1; i < chunk_npages-1; i++) 393940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert(chunk->map[i-map_bias].bits == unzeroed); 394940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 395004ed142a66529ecf4a55e8f4fa42ff2e535f586Jason Evans chunk->map[chunk_npages-1-map_bias].bits = arena_maxclass | 396004ed142a66529ecf4a55e8f4fa42ff2e535f586Jason Evans unzeroed; 397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 39819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Insert the run into the runs_avail_clean tree. */ 39919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_insert(&arena->runs_avail_clean, 4007393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[0]); 40119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 403e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (chunk); 404e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 406e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 407e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) 408e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 40919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t *runs_avail; 410e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4118d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans /* 4128d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * Remove run from the appropriate runs_avail_* tree, so that the arena 4138d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * does not use it. 4148d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans */ 4157393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) 4168d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans runs_avail = &arena->runs_avail_clean; 4178d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans else 4188d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans runs_avail = &arena->runs_avail_dirty; 4197393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_remove(runs_avail, &chunk->map[0]); 4208d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans 4218d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (arena->spare != NULL) { 422e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_chunk_t *spare = arena->spare; 423e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 4248d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans arena->spare = chunk; 425e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (spare->dirtied) { 426e00572b384c81bd2aba57fac32f7077a34388915Jason Evans ql_remove(&chunk->arena->chunks_dirty, spare, 4272caa4715ed4f787f263239ff97dd824636289286Jason Evans link_dirty); 428e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena->ndirty -= spare->ndirty; 429e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 430e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&arena->lock); 43112a488782681cbd740a5f54e0b7e74ea84858e21Jason Evans chunk_dealloc((void *)spare, chunksize, true); 432e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&arena->lock); 4337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 4347372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.mapped -= chunksize; 4358d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans } else 4368d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans arena->spare = chunk; 437e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 438e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 439e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 440e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) 441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 442e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 443e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm, key; 445e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 447e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 448e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Search the arena's chunks for the lowest best fit. */ 450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans key.bits = size | CHUNK_MAP_KEY; 45119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); 45219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (mapelm != NULL) { 45319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 4547393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 4557393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 4567393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 45719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 45819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 459ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 46019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_run_split(arena, run, size, large, zero); 46119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans return (run); 46219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 46319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); 464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 4667393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 4677393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 4687393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 470e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 471ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 472e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, run, size, large, zero); 473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 474e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 475e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 476e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 477e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * No usable runs. Create a new chunk from which to allocate the run. 478e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 479e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena_chunk_alloc(arena); 480e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (chunk != NULL) { 481ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); 482e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_split(arena, run, size, large, zero); 483e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 484e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 485e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 486e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 487e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_chunk_alloc() failed, but another thread may have made 488e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * sufficient memory available while this one dropped arena->lock in 489e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_chunk_alloc(), so search one more time. 490e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 49119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); 49219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (mapelm != NULL) { 49319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 4947393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 4957393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 4967393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 49719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 49819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 499ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 50019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_run_split(arena, run, size, large, zero); 50119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans return (run); 50219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 50319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); 504e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (mapelm != NULL) { 505e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 5067393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 5077393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 5087393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 509e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 510e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 511ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 512e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_split(arena, run, size, large, zero); 513e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 514e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 515e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 516e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (NULL); 517e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 518e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 51905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansstatic inline void 52005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansarena_maybe_purge(arena_t *arena) 52105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans{ 52205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 52305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Enforce opt_lg_dirty_mult. */ 524799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans if (opt_lg_dirty_mult >= 0 && arena->ndirty > arena->npurgatory && 525799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans (arena->ndirty - arena->npurgatory) > chunk_npages && 526799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans (arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - 527799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory)) 5286005f0710cf07d60659d91b20b7ff5592d310027Jason Evans arena_purge(arena, false); 52905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans} 53005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 53105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansstatic inline void 53205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansarena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) 53305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans{ 53405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_head(arena_chunk_map_t) mapelms; 53505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_map_t *mapelm; 5363377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans size_t pageind, flag_unzeroed; 53705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t ndirty; 53805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t nmadvise; 53905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 54005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_new(&mapelms); 54105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 5423377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans flag_unzeroed = 5432dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED 5442dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans /* 5452dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * madvise(..., MADV_DONTNEED) results in zero-filled pages for anonymous 5462dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * mappings, but not for file-backed mappings. 5472dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans */ 5484162627757889ea999264c2ddbc3c354768774e2Jason Evans 0 5493377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans#else 5504162627757889ea999264c2ddbc3c354768774e2Jason Evans CHUNK_MAP_UNZEROED 5512dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#endif 5524162627757889ea999264c2ddbc3c354768774e2Jason Evans ; 553c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 55405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 55505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * If chunk is the spare, temporarily re-allocate it, 1) so that its 55619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run is reinserted into runs_avail_dirty, and 2) so that it cannot be 55705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * completely discarded by another thread while arena->lock is dropped 55805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * by this thread. Note that the arena_run_dalloc() call will 55905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * implicitly deallocate the chunk, so no explicit action is required 56005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * in this function to deallocate the chunk. 56119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 56219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Note that once a chunk contains dirty pages, it cannot again contain 56319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * a single run unless 1) it is a dirty run, or 2) this function purges 56419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * dirty pages and causes the transition to a single clean run. Thus 56519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * (chunk == arena->spare) is possible, but it is not possible for 56619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * this function to be called on the spare unless it contains a dirty 56719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run. 56805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 56919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (chunk == arena->spare) { 5707393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) != 0); 57105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_alloc(arena); 57219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 57305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 57419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Temporarily allocate all free dirty runs within chunk. */ 5757393f44ff025ca67716fc53b68003fd65122fd97Jason Evans for (pageind = map_bias; pageind < chunk_npages;) { 5767393f44ff025ca67716fc53b68003fd65122fd97Jason Evans mapelm = &chunk->map[pageind-map_bias]; 57705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans if ((mapelm->bits & CHUNK_MAP_ALLOCATED) == 0) { 57819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t npages; 57905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 580ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans npages = mapelm->bits >> LG_PAGE; 581e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind + npages <= chunk_npages); 58219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (mapelm->bits & CHUNK_MAP_DIRTY) { 583c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans size_t i; 584c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 58519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_remove( 58619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans &arena->runs_avail_dirty, mapelm); 587c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 588ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans mapelm->bits = (npages << LG_PAGE) | 5898de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans flag_unzeroed | CHUNK_MAP_LARGE | 5908de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_ALLOCATED; 591c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans /* 592c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans * Update internal elements in the page map, so 5933377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * that CHUNK_MAP_UNZEROED is properly set. 594c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans */ 595c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans for (i = 1; i < npages - 1; i++) { 5967393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[pageind+i-map_bias].bits = 5973377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans flag_unzeroed; 598c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans } 599c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans if (npages > 1) { 6007393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[ 6017393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind+npages-1-map_bias].bits = 6028de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans flag_unzeroed | CHUNK_MAP_LARGE | 6038de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_ALLOCATED; 604c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans } 605c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 6067372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 6077372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans /* 6087372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * Update stats_cactive if nactive is 6097372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * crossing a chunk multiple. 6107372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans */ 6117372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t cactive_diff = 6127372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans CHUNK_CEILING((arena->nactive + 613ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans npages) << LG_PAGE) - 6147372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans CHUNK_CEILING(arena->nactive << 615ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE); 6167372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (cactive_diff != 0) 6177372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans stats_cactive_add(cactive_diff); 6187372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 61919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena->nactive += npages; 62019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Append to list for later processing. */ 62119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans ql_elm_new(mapelm, u.ql_link); 62219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans ql_tail_insert(&mapelms, mapelm, u.ql_link); 62305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 62419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 62505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans pageind += npages; 62605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } else { 62705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Skip allocated run. */ 628c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans if (mapelm->bits & CHUNK_MAP_LARGE) 629ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind += mapelm->bits >> LG_PAGE; 630c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans else { 63105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t) 632ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans chunk + (uintptr_t)(pageind << LG_PAGE)); 633e69bee01de62b56d3e585042d341743239568043Jason Evans 634ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans assert((mapelm->bits >> LG_PAGE) == 0); 63549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(arena, 63649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans run->bin); 63749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = 63849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans &arena_bin_info[binind]; 639ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind += bin_info->run_size >> LG_PAGE; 64005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 64105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 64205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 643e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind == chunk_npages); 64405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 6457372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) 6467372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ndirty = chunk->ndirty; 6477372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 6487372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.purged += chunk->ndirty; 64905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena->ndirty -= chunk->ndirty; 65005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans chunk->ndirty = 0; 65105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_remove(&arena->chunks_dirty, chunk, link_dirty); 65205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans chunk->dirtied = false; 65305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 65405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans malloc_mutex_unlock(&arena->lock); 6557372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 6567372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans nmadvise = 0; 65705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_foreach(mapelm, &mapelms, u.ql_link) { 6587393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 6597393f44ff025ca67716fc53b68003fd65122fd97Jason Evans sizeof(arena_chunk_map_t)) + map_bias; 660ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t npages = mapelm->bits >> LG_PAGE; 66105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 662e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind + npages <= chunk_npages); 66319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans assert(ndirty >= npages); 6647372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) 6657372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ndirty -= npages; 6662dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans 6672dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED 6687372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans# define MADV_PURGE MADV_DONTNEED 6692dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#elif defined(JEMALLOC_PURGE_MADVISE_FREE) 6707372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans# define MADV_PURGE MADV_FREE 6712dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#else 6722dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans# error "No method defined for purging unused dirty pages." 6732dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#endif 674ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans madvise((void *)((uintptr_t)chunk + (pageind << LG_PAGE)), 675ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (npages << LG_PAGE), MADV_PURGE); 6767372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans#undef MADV_PURGE 6777372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 6787372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans nmadvise++; 67905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 68005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans assert(ndirty == 0); 68105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans malloc_mutex_lock(&arena->lock); 6827372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 6837372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmadvise += nmadvise; 68405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 68505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Deallocate runs. */ 68605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans for (mapelm = ql_first(&mapelms); mapelm != NULL; 68705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans mapelm = ql_first(&mapelms)) { 6887393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 6897393f44ff025ca67716fc53b68003fd65122fd97Jason Evans sizeof(arena_chunk_map_t)) + map_bias; 69005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 691ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (uintptr_t)(pageind << LG_PAGE)); 69205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 69305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_remove(&mapelms, mapelm, u.ql_link); 69405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_run_dalloc(arena, run, false); 69505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 69605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans} 69705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 698e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 6996005f0710cf07d60659d91b20b7ff5592d310027Jason Evansarena_purge(arena_t *arena, bool all) 700e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 701e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 70205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t npurgatory; 7037372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) { 7047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t ndirty = 0; 705e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7067372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ql_foreach(chunk, &arena->chunks_dirty, link_dirty) { 7077372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans assert(chunk->dirtied); 7087372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ndirty += chunk->ndirty; 7097372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 7107372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans assert(ndirty == arena->ndirty); 7112caa4715ed4f787f263239ff97dd824636289286Jason Evans } 712af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans assert(arena->ndirty > arena->npurgatory || all); 713f9a8edbb50f8cbcaf8ed62b36e8d7191ed223d2aJason Evans assert(arena->ndirty - arena->npurgatory > chunk_npages || all); 714af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - 715f9a8edbb50f8cbcaf8ed62b36e8d7191ed223d2aJason Evans arena->npurgatory) || all); 716e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7177372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 7187372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.npurge++; 719e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 720e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 72105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * Compute the minimum number of pages that this thread should try to 722799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * purge, and add the result to arena->npurgatory. This will keep 723799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * multiple threads from racing to reduce ndirty below the threshold. 724e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 7256005f0710cf07d60659d91b20b7ff5592d310027Jason Evans npurgatory = arena->ndirty - arena->npurgatory; 726af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans if (all == false) { 727af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans assert(npurgatory >= arena->nactive >> opt_lg_dirty_mult); 7286005f0710cf07d60659d91b20b7ff5592d310027Jason Evans npurgatory -= arena->nactive >> opt_lg_dirty_mult; 729af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans } 730799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory += npurgatory; 731799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans 73205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans while (npurgatory > 0) { 73305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Get next chunk with dirty pages. */ 7342caa4715ed4f787f263239ff97dd824636289286Jason Evans chunk = ql_first(&arena->chunks_dirty); 73505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans if (chunk == NULL) { 73605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 73705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * This thread was unable to purge as many pages as 73805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * originally intended, due to races with other threads 739799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * that either did some of the purging work, or re-used 740799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * dirty pages. 74105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 742799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory -= npurgatory; 743799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans return; 744e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 74505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans while (chunk->ndirty == 0) { 7462caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_remove(&arena->chunks_dirty, chunk, link_dirty); 747e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = false; 74805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans chunk = ql_first(&arena->chunks_dirty); 74905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans if (chunk == NULL) { 75005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Same logic as for above. */ 751799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory -= npurgatory; 752799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans return; 75305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 754e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 75505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 756799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans if (chunk->ndirty > npurgatory) { 757799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans /* 758799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * This thread will, at a minimum, purge all the dirty 759799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * pages in chunk, so set npurgatory to reflect this 760799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * thread's commitment to purge the pages. This tends 761799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * to reduce the chances of the following scenario: 762799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 763799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 1) This thread sets arena->npurgatory such that 764799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * (arena->ndirty - arena->npurgatory) is at the 765799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * threshold. 766799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 2) This thread drops arena->lock. 767799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 3) Another thread causes one or more pages to be 768799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * dirtied, and immediately determines that it must 769799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * purge dirty pages. 770799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 771799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * If this scenario *does* play out, that's okay, 772799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * because all of the purging work being done really 773799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * needs to happen. 774799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans */ 775799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory += chunk->ndirty - npurgatory; 776799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans npurgatory = chunk->ndirty; 777799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans } 778799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans 779799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory -= chunk->ndirty; 780799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans npurgatory -= chunk->ndirty; 78105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_purge(arena, chunk); 782e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 783e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 784e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7856005f0710cf07d60659d91b20b7ff5592d310027Jason Evansvoid 7866005f0710cf07d60659d91b20b7ff5592d310027Jason Evansarena_purge_all(arena_t *arena) 7876005f0710cf07d60659d91b20b7ff5592d310027Jason Evans{ 7886005f0710cf07d60659d91b20b7ff5592d310027Jason Evans 7896005f0710cf07d60659d91b20b7ff5592d310027Jason Evans malloc_mutex_lock(&arena->lock); 7906005f0710cf07d60659d91b20b7ff5592d310027Jason Evans arena_purge(arena, true); 7916005f0710cf07d60659d91b20b7ff5592d310027Jason Evans malloc_mutex_unlock(&arena->lock); 7926005f0710cf07d60659d91b20b7ff5592d310027Jason Evans} 7936005f0710cf07d60659d91b20b7ff5592d310027Jason Evans 794e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 795e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) 796e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 797e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 79819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t size, run_ind, run_pages, flag_dirty; 79919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t *runs_avail; 800e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 801e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 802ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 8037393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(run_ind >= map_bias); 804e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run_ind < chunk_npages); 8058de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_LARGE) != 0) { 8067393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size = chunk->map[run_ind-map_bias].bits & ~PAGE_MASK; 807ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans assert(size == PAGE || 808ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (chunk->map[run_ind+(size>>LG_PAGE)-1-map_bias].bits & 8098de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans ~PAGE_MASK) == 0); 810ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans assert((chunk->map[run_ind+(size>>LG_PAGE)-1-map_bias].bits & 8118de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_LARGE) != 0); 812ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans assert((chunk->map[run_ind+(size>>LG_PAGE)-1-map_bias].bits & 8138de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_ALLOCATED) != 0); 81449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } else { 81549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(arena, run->bin); 81649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 81749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size = bin_info->run_size; 81849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 819ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run_pages = (size >> LG_PAGE); 8207372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 8217372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans /* 8227372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * Update stats_cactive if nactive is crossing a chunk 8237372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * multiple. 8247372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans */ 825ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t cactive_diff = CHUNK_CEILING(arena->nactive << LG_PAGE) - 826ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans CHUNK_CEILING((arena->nactive - run_pages) << LG_PAGE); 8277372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (cactive_diff != 0) 8287372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans stats_cactive_sub(cactive_diff); 8297372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 830e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive -= run_pages; 831e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 83219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 83319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * The run is dirty if the caller claims to have dirtied it, as well as 83419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * if it was already dirty before being allocated. 83519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 8367393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) != 0) 83719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans dirty = true; 83819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 83919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans runs_avail = dirty ? &arena->runs_avail_dirty : 84019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans &arena->runs_avail_clean; 84119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 842e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Mark pages as unallocated in the chunk map. */ 843e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (dirty) { 844940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[run_ind-map_bias].bits = size | CHUNK_MAP_DIRTY; 8457393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+run_pages-1-map_bias].bits = size | 846940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_DIRTY; 847e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 84819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans chunk->ndirty += run_pages; 84919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena->ndirty += run_pages; 850e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 8517393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind-map_bias].bits = size | 8523377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED); 8537393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+run_pages-1-map_bias].bits = size | 8547393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & 8553377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans CHUNK_MAP_UNZEROED); 856e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 857e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 858e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce forward. */ 859e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run_ind + run_pages < chunk_npages && 8607393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+run_pages-map_bias].bits & CHUNK_MAP_ALLOCATED) 8617393f44ff025ca67716fc53b68003fd65122fd97Jason Evans == 0 && (chunk->map[run_ind+run_pages-map_bias].bits & 8627393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_DIRTY) == flag_dirty) { 8637393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t nrun_size = chunk->map[run_ind+run_pages-map_bias].bits & 864e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ~PAGE_MASK; 865ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t nrun_pages = nrun_size >> LG_PAGE; 866e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 867e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 868e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove successor from runs_avail; the coalesced run is 869e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 870e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 87112ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits 87212ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans & ~PAGE_MASK) == nrun_size); 87312ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits 87412ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans & CHUNK_MAP_ALLOCATED) == 0); 87512ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits 87612ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans & CHUNK_MAP_DIRTY) == flag_dirty); 87719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_remove(runs_avail, 8787393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[run_ind+run_pages-map_bias]); 879e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 880e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += nrun_size; 88112ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_pages += nrun_pages; 882e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 8837393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind-map_bias].bits = size | 8847393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); 8857393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+run_pages-1-map_bias].bits = size | 8867393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & 887e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 888e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 889e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 890e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce backward. */ 8917393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if (run_ind > map_bias && (chunk->map[run_ind-1-map_bias].bits & 8927393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[run_ind-1-map_bias].bits & 89319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans CHUNK_MAP_DIRTY) == flag_dirty) { 8947393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t prun_size = chunk->map[run_ind-1-map_bias].bits & 8957393f44ff025ca67716fc53b68003fd65122fd97Jason Evans ~PAGE_MASK; 896ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t prun_pages = prun_size >> LG_PAGE; 897e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 89812ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_ind -= prun_pages; 899e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 900e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 901e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove predecessor from runs_avail; the coalesced run is 902e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 903e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 90412ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) 90512ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans == prun_size); 90612ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_ALLOCATED) 90712ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans == 0); 90812ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) 90912ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans == flag_dirty); 9107393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_remove(runs_avail, 9117393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[run_ind-map_bias]); 912e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 913e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += prun_size; 91412ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_pages += prun_pages; 915e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 9167393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind-map_bias].bits = size | 9177393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); 9187393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+run_pages-1-map_bias].bits = size | 9197393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & 920e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 921e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 922e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 923e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert into runs_avail, now that coalescing is complete. */ 9248de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) == 9258de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & ~PAGE_MASK)); 9268de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == 9278de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & CHUNK_MAP_DIRTY)); 9287393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_insert(runs_avail, &chunk->map[run_ind-map_bias]); 929e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 9308d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (dirty) { 9318d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans /* 9328d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * Insert into chunks_dirty before potentially calling 9338d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * arena_chunk_dealloc(), so that chunks_dirty and 9348d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * arena->ndirty are consistent. 9358d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans */ 9368d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (chunk->dirtied == false) { 9378d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty); 9388d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans chunk->dirtied = true; 9398d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans } 9408d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans } 9418d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans 9424fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 9434fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * Deallocate chunk if it is now completely unused. The bit 9444fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * manipulation checks whether the first run is unallocated and extends 9454fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * to the end of the chunk. 9464fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 9477393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[0].bits & (~PAGE_MASK | CHUNK_MAP_ALLOCATED)) == 9487393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_maxclass) 949e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_dealloc(arena, chunk); 950e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 9514fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 9528d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * It is okay to do dirty page processing here even if the chunk was 9534fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * deallocated above, since in that case it is the spare. Waiting 9544fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * until after possible chunk deallocation to do dirty processing 9554fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * allows for an old spare to be fully deallocated, thus decreasing the 9564fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * chances of spuriously crossing the dirty page purging threshold. 9574fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 9588d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (dirty) 95905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_maybe_purge(arena); 960e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 961e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 962e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 963e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 964e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize) 965e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 966ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 967ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t head_npages = (oldsize - newsize) >> LG_PAGE; 968940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t flag_dirty = chunk->map[pageind-map_bias].bits & CHUNK_MAP_DIRTY; 969e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 970e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 971e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 972e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 973e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 974940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * leading run as separately allocated. Set the last element of each 975940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * run first, in case of single-page runs. 976e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 977940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); 978940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); 979940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | 980940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+head_npages-1-map_bias].bits & 981940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 982940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind-map_bias].bits = (oldsize - newsize) 983940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans | flag_dirty | (chunk->map[pageind-map_bias].bits & 984940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 985940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 9867372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) { 987ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans UNUSED size_t tail_npages = newsize >> LG_PAGE; 988940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] 989940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans .bits & ~PAGE_MASK) == 0); 990940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] 991940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans .bits & CHUNK_MAP_DIRTY) == flag_dirty); 992940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] 993940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans .bits & CHUNK_MAP_LARGE) != 0); 994940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] 995940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans .bits & CHUNK_MAP_ALLOCATED) != 0); 996940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 997940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages-map_bias].bits = newsize | flag_dirty | 998940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+head_npages-map_bias].bits & 999940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_FLAGS_MASK) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1000e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1001e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, run, false); 1002e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1003e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1004e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1005e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1006e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize, bool dirty) 1007e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1008ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1009ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t head_npages = newsize >> LG_PAGE; 1010ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t tail_npages = (oldsize - newsize) >> LG_PAGE; 1011940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t flag_dirty = chunk->map[pageind-map_bias].bits & 1012940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_DIRTY; 1013e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1014e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 1015e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1016e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1017e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 1018940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * trailing run as separately allocated. Set the last element of each 1019940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * run first, in case of single-page runs. 1020e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1021940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); 1022940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); 1023940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | 1024940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+head_npages-1-map_bias].bits & 1025940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1026940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind-map_bias].bits = newsize | flag_dirty | 1027940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind-map_bias].bits & CHUNK_MAP_UNZEROED) | 1028940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1029940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1030940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & 1031940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans ~PAGE_MASK) == 0); 1032940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & 1033940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_LARGE) != 0); 1034940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & 1035940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_ALLOCATED) != 0); 1036940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits = 1037940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans flag_dirty | 1038940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & 1039940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1040940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages-map_bias].bits = (oldsize - newsize) | 1041940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans flag_dirty | (chunk->map[pageind+head_npages-map_bias].bits & 1042940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1043e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1044e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 1045e476f8a161d445211fd6e54fe370275196e66bcbJason Evans dirty); 1046e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1047e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1048e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 1049e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_first(arena_bin_t *bin) 1050e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1051e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); 1052e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 1053e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1054e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 1055e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1056e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 10577393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / 10587393f44ff025ca67716fc53b68003fd65122fd97Jason Evans sizeof(arena_chunk_map_t))) + map_bias; 1059e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 1060ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (uintptr_t)((pageind - (mapelm->bits >> LG_PAGE)) << 1061ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 1062e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1063e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans } 1064e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1065e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (NULL); 1066e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1067e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1068e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void 1069e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 1070e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1071e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); 1072ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1073e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; 1074e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1075e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); 1076e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1077e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_tree_insert(&bin->runs, mapelm); 1078e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1079e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1080e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void 1081e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) 1082e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1083e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1084ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; 1085e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; 1086e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1087e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); 1088e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1089e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_tree_remove(&bin->runs, mapelm); 1090e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1091e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1092e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t * 1093e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_nonfull_run_tryget(arena_bin_t *bin) 1094e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1095e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_t *run = arena_bin_runs_first(bin); 1096e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) { 1097e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_remove(bin, run); 10987372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 10997372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.reruns++; 1100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1101e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1102e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1103e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1104e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t * 1105e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 1106e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1107e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_t *run; 1108e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans size_t binind; 1109e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_info_t *bin_info; 1110e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1111e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans /* Look for a usable run. */ 1112e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans run = arena_bin_nonfull_run_tryget(bin); 1113e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) 1114e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* No existing runs have any space available. */ 1116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 111749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(arena, bin); 111849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 111949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 1120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Allocate a new run. */ 1121e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&bin->lock); 1122e69bee01de62b56d3e585042d341743239568043Jason Evans /******************************/ 112386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 112449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans run = arena_run_alloc(arena, bin_info->run_size, false, false); 1125e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run != NULL) { 112684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 112784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 112884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 1129e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* Initialize run internals. */ 1130e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run->bin = bin; 113184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans run->nextind = 0; 113249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans run->nfree = bin_info->nregs; 113384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_init(bitmap, &bin_info->bitmap_info); 1134e69bee01de62b56d3e585042d341743239568043Jason Evans } 1135e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_unlock(&arena->lock); 1136e69bee01de62b56d3e585042d341743239568043Jason Evans /********************************/ 1137e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 1138e69bee01de62b56d3e585042d341743239568043Jason Evans if (run != NULL) { 11397372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 11407372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nruns++; 11417372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.curruns++; 11427372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1143e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 1144e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1145e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1146e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 1147e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_run_alloc() failed, but another thread may have made 1148940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * sufficient memory available while this one dropped bin->lock above, 1149e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * so search one more time. 1150e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 1151e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans run = arena_bin_nonfull_run_tryget(bin); 1152e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) 1153e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 1154e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1155e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (NULL); 1156e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 11581e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 1159e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void * 1160e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1162e00572b384c81bd2aba57fac32f7077a34388915Jason Evans void *ret; 116349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 116449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1165e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_t *run; 1166e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 116749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(arena, bin); 116849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 1169e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->runcur = NULL; 1170e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = arena_bin_nonfull_run_get(arena, bin); 1171e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (bin->runcur != NULL && bin->runcur->nfree > 0) { 1172e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 1173e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * Another thread updated runcur while this one ran without the 1174e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * bin lock in arena_bin_nonfull_run_get(). 1175e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 1176e00572b384c81bd2aba57fac32f7077a34388915Jason Evans assert(bin->runcur->nfree > 0); 117749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_run_reg_alloc(bin->runcur, bin_info); 1178e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run != NULL) { 1179940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_chunk_t *chunk; 1180940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1181940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* 1182940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * arena_run_alloc() may have allocated run, or it may 118384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * have pulled run from the bin's run tree. Therefore 1184940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * it is unsafe to make any assumptions about how run 1185940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * has previously been used, and arena_bin_lower_run() 1186940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * must be called, as if a region were just deallocated 1187940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * from the run. 1188940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans */ 1189940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 119049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (run->nfree == bin_info->nregs) 11918de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 11928de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans else 11938de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_bin_lower_run(arena, chunk, run, bin); 1194e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1195e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (ret); 1196e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1197e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1198e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run == NULL) 1199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1200e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1201e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->runcur = run; 1202e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->runcur->nfree > 0); 1204e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 120549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (arena_run_reg_alloc(bin->runcur, bin_info)); 1206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 120886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evansvoid 120986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evansarena_prof_accum(arena_t *arena, uint64_t accumbytes) 121086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans{ 121186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 121286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (prof_interval != 0) { 121386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena->prof_accumbytes += accumbytes; 121486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (arena->prof_accumbytes >= prof_interval) { 121586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans prof_idump(); 121686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena->prof_accumbytes -= prof_interval; 121786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 121886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 121986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans} 122086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 1221e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 12227372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansarena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, 12237372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans uint64_t prof_accumbytes) 1224e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, nfill; 1226e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr; 1229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tbin->ncached == 0); 1231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 12327372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) { 12337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans malloc_mutex_lock(&arena->lock); 12347372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena_prof_accum(arena, prof_accumbytes); 12357372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans malloc_mutex_unlock(&arena->lock); 12367372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1237e69bee01de62b56d3e585042d341743239568043Jason Evans bin = &arena->bins[binind]; 1238e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 12391dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 12401dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans tbin->lg_fill_div); i < nfill; i++) { 1241e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 124249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ptr = arena_bin_malloc_hard(arena, bin); 12453fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (ptr == NULL) 1246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans break; 12479c43c13a35220c10d97a886616899189daceb359Jason Evans /* Insert such that low regions get used first. */ 12489c43c13a35220c10d97a886616899189daceb359Jason Evans tbin->avail[nfill - 1 - i] = ptr; 1249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 12507372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 12517372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated += i * arena_bin_info[binind].reg_size; 12527372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nmalloc += i; 12537372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 12547372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nfills++; 12557372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tbin->tstats.nrequests = 0; 12567372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 125786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1258e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->ncached = i; 1259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1260e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1261e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1262e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_small(arena_t *arena, size_t size, bool zero) 1263e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1264e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1265e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1266e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1267e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 1268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 126941ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans binind = SMALL_SIZE2BIN(size); 1270b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 1271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 127249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size = arena_bin_info[binind].reg_size; 1273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 127486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 1275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 127649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1277e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_hard(arena, bin); 1279e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 128186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 12857372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 12867372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated += size; 12877372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nmalloc++; 12887372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nrequests++; 12897372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 129086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 12917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof && isthreaded == false) { 129286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1293d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, size); 129486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 129586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 1296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 12987372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill) { 12997372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (opt_junk) 13007372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0xa5, size); 13017372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans else if (opt_zero) 13027372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0, size); 13037372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 1305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1310e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1311e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_large(arena_t *arena, size_t size, bool zero) 1312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1315e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1316e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = PAGE_CEILING(size); 1317e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1318e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)arena_run_alloc(arena, size, true, zero); 1319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 13237372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 13247372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 13257372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 13267372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 1327ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1328ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1329ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 13317372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) 13327372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena_prof_accum(arena, size); 1333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 13367372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill) { 13377372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (opt_junk) 13387372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0xa5, size); 13397372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans else if (opt_zero) 13407372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0, size); 13417372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Only handles large allocations that require more than page alignment. */ 1348e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 13498e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_palloc(arena_t *arena, size_t size, size_t alloc_size, size_t alignment, 13508e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 1351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t offset; 1354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1355e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1356e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 135793443689a4367cc6fe3de1c9e918adc13d8f9100Jason Evans 135893443689a4367cc6fe3de1c9e918adc13d8f9100Jason Evans alignment = PAGE_CEILING(alignment); 1359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 13618e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = (void *)arena_run_alloc(arena, alloc_size, true, zero); 1362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); 1368e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans offset = (uintptr_t)ret & (alignment - 1); 1370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((offset & PAGE_MASK) == 0); 1371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(offset < alloc_size); 1372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (offset == 0) 1373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, ret, alloc_size, size, false); 1374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 1375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t leadsize, trailsize; 1376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans leadsize = alignment - offset; 1378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (leadsize > 0) { 1379e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_head(arena, chunk, ret, alloc_size, 1380e476f8a161d445211fd6e54fe370275196e66bcbJason Evans alloc_size - leadsize); 1381e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)((uintptr_t)ret + leadsize); 1382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1384e476f8a161d445211fd6e54fe370275196e66bcbJason Evans trailsize = alloc_size - leadsize - size; 1385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (trailsize != 0) { 1386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Trim trailing space. */ 1387e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(trailsize < alloc_size); 1388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, ret, size + trailsize, 1389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size, false); 1390e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1392e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 13937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 13947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 13957372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 13967372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 1397ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1398ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1399ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 14037372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && zero == false) { 14048e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (opt_junk) 14058e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset(ret, 0xa5, size); 14068e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans else if (opt_zero) 14078e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset(ret, 0, size); 14088e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 1409e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1410e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1411e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1412e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Return the size of the allocation pointed to by ptr. */ 1413e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t 1414e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_salloc(const void *ptr) 1415e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1416e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ret; 1417e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1418e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind, mapbits; 1419e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1420e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1421e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1422e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1423e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1424ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 14257393f44ff025ca67716fc53b68003fd65122fd97Jason Evans mapbits = chunk->map[pageind-map_bias].bits; 1426e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 1427e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 1428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 1429ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << LG_PAGE)); 143049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, run->bin); 143149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 1432f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 143349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == 1434f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans 0); 143549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = bin_info->reg_size; 1436e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1437f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 1438e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = mapbits & ~PAGE_MASK; 1439e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ret != 0); 1440e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1442e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1443e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 14450b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansvoid 14460b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansarena_prof_promoted(const void *ptr, size_t size) 14470b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans{ 14480b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans arena_chunk_t *chunk; 14490b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t pageind, binind; 14500b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 14510b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(ptr != NULL); 14520b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1453ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans assert(isalloc(ptr) == PAGE); 1454b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(size <= SMALL_MAXCLASS); 14550b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 14560b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1457ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 145841ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans binind = SMALL_SIZE2BIN(size); 1459b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 14607393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits & 1461e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans ~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT); 14620b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans} 14630b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 14640b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evanssize_t 14650b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansarena_salloc_demote(const void *ptr) 14660b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans{ 14670b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t ret; 14680b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans arena_chunk_t *chunk; 14690b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t pageind, mapbits; 14700b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 14710b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(ptr != NULL); 14720b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 14730b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 14740b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1475ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 14767393f44ff025ca67716fc53b68003fd65122fd97Jason Evans mapbits = chunk->map[pageind-map_bias].bits; 14770b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 14780b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 14790b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 1480ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << LG_PAGE)); 148149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, run->bin); 148249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 14830b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 148449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == 14850b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 0); 148649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = bin_info->reg_size; 14870b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans } else { 14880b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 14890b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans ret = mapbits & ~PAGE_MASK; 1490ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans if (prof_promote && ret == PAGE && (mapbits & 1491e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans CHUNK_MAP_CLASS_MASK) != 0) { 14920b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >> 1493e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans CHUNK_MAP_CLASS_SHIFT) - 1; 1494b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 149549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_bin_info[binind].reg_size; 14960b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans } 14970b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(ret != 0); 14980b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans } 14990b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 15000b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans return (ret); 15010b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans} 15026109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1503e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1504088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansarena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 1505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin) 1506e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1507e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 150819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Dissociate run from bin. */ 1509e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run == bin->runcur) 1510e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 151149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans else { 151249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, bin); 151349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 151449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 151549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (bin_info->nregs != 1) { 151649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 151749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * This block's conditional is necessary because if the 151849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * run only contains one region, then it never gets 151949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * inserted into the non-full runs tree. 152049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 1521e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_remove(bin, run); 152249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 1523e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1524088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans} 1525088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans 1526088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansstatic void 1527088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansarena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1528088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_bin_t *bin) 1529088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans{ 153049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 153149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1532088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans size_t npages, run_ind, past; 1533088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans 1534088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans assert(run != bin->runcur); 1535088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans assert(arena_run_tree_search(&bin->runs, &chunk->map[ 1536ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)-map_bias]) == NULL); 153786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 153849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(chunk->arena, run->bin); 153949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 154049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 1541e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&bin->lock); 1542e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /******************************/ 1543ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans npages = bin_info->run_size >> LG_PAGE; 1544ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); 154584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans past = (size_t)(PAGE_CEILING((uintptr_t)run + 154684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * 1547ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans bin_info->reg_size) - (uintptr_t)chunk) >> LG_PAGE); 154886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 154919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 155019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 155119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * If the run was originally clean, and some pages were never touched, 155219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * trim the clean pages before deallocating the dirty portion of the 155319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run. 155419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 15557393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == 0 && past 15567393f44ff025ca67716fc53b68003fd65122fd97Jason Evans - run_ind < npages) { 155719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 155819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Trim clean pages. Convert to large run beforehand. Set the 155919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * last map element first, in case this is a one-page run. 156019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 15617393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+npages-1-map_bias].bits = CHUNK_MAP_LARGE | 1562940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[run_ind+npages-1-map_bias].bits & 1563940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_FLAGS_MASK); 156449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans chunk->map[run_ind-map_bias].bits = bin_info->run_size | 15657393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_LARGE | (chunk->map[run_ind-map_bias].bits & 15667393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_FLAGS_MASK); 1567ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), 1568ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans ((past - run_ind) << LG_PAGE), false); 1569940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* npages = past - run_ind; */ 15701e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans } 157119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_run_dalloc(arena, run, true); 157286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 1573e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /****************************/ 1574e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&bin->lock); 15757372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 15767372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.curruns--; 1577e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1578e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1579940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansstatic void 1580940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansarena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1581940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_bin_t *bin) 1582e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1583e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 15848de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* 1585e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * Make sure that if bin->runcur is non-NULL, it refers to the lowest 1586e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * non-full run. It is okay to NULL runcur out rather than proactively 1587e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * keeping it pointing at the lowest non-full run. 15888de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans */ 1589e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if ((uintptr_t)run < (uintptr_t)bin->runcur) { 15908de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* Switch runcur. */ 1591e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (bin->runcur->nfree > 0) 1592e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_insert(bin, bin->runcur); 15938de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans bin->runcur = run; 1594e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (config_stats) 1595e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans bin->stats.reruns++; 1596e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans } else 1597e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_insert(bin, run); 1598940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans} 1599940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1600940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansvoid 1601940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansarena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1602940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_chunk_map_t *mapelm) 1603940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans{ 1604940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t pageind; 1605940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_t *run; 1606940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_bin_t *bin; 1607940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t size; 1608940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1609ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1610940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1611ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (mapelm->bits >> LG_PAGE)) << LG_PAGE)); 1612940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans bin = run->bin; 161349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(arena, bin); 161449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 16157372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill || config_stats) 16167372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size = bin_info->reg_size; 1617940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 16187372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk) 1619940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans memset(ptr, 0x5a, size); 1620940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1621940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_reg_dalloc(run, ptr); 162249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (run->nfree == bin_info->nregs) { 1623088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_dissociate_bin_run(chunk, run, bin); 16248de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 1625088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans } else if (run->nfree == 1 && run != bin->runcur) 16268de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_bin_lower_run(arena, chunk, run, bin); 1627e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 16287372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 16297372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated -= size; 16307372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.ndalloc++; 16317372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1632e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1633e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1634e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1635b34e8684ec025aa780997c11f847c19fb269755bJason Evansarena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, 1636b34e8684ec025aa780997c11f847c19fb269755bJason Evans arena_stats_t *astats, malloc_bin_stats_t *bstats, 1637b34e8684ec025aa780997c11f847c19fb269755bJason Evans malloc_large_stats_t *lstats) 1638b34e8684ec025aa780997c11f847c19fb269755bJason Evans{ 16393c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans unsigned i; 1640b34e8684ec025aa780997c11f847c19fb269755bJason Evans 164186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1642b34e8684ec025aa780997c11f847c19fb269755bJason Evans *nactive += arena->nactive; 1643b34e8684ec025aa780997c11f847c19fb269755bJason Evans *ndirty += arena->ndirty; 1644b34e8684ec025aa780997c11f847c19fb269755bJason Evans 16454201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans astats->mapped += arena->stats.mapped; 1646b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->npurge += arena->stats.npurge; 1647b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmadvise += arena->stats.nmadvise; 1648b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->purged += arena->stats.purged; 1649b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->allocated_large += arena->stats.allocated_large; 1650b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmalloc_large += arena->stats.nmalloc_large; 1651b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->ndalloc_large += arena->stats.ndalloc_large; 1652dafde14e08ddfda747aabb2045b350848b601b2eJason Evans astats->nrequests_large += arena->stats.nrequests_large; 1653b34e8684ec025aa780997c11f847c19fb269755bJason Evans 16543c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans for (i = 0; i < nlclasses; i++) { 1655dafde14e08ddfda747aabb2045b350848b601b2eJason Evans lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 1656dafde14e08ddfda747aabb2045b350848b601b2eJason Evans lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 1657b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].nrequests += arena->stats.lstats[i].nrequests; 1658b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].curruns += arena->stats.lstats[i].curruns; 1659b34e8684ec025aa780997c11f847c19fb269755bJason Evans } 166086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 166186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 1662b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans for (i = 0; i < NBINS; i++) { 166386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin = &arena->bins[i]; 166486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 166586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 166686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].allocated += bin->stats.allocated; 166786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nmalloc += bin->stats.nmalloc; 166886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].ndalloc += bin->stats.ndalloc; 166986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nrequests += bin->stats.nrequests; 16707372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_tcache) { 16717372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bstats[i].nfills += bin->stats.nfills; 16727372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bstats[i].nflushes += bin->stats.nflushes; 16737372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 167486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nruns += bin->stats.nruns; 167586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].reruns += bin->stats.reruns; 167686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].curruns += bin->stats.curruns; 167786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 167886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 1679b34e8684ec025aa780997c11f847c19fb269755bJason Evans} 1680e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1681e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1682e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1683e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 168413668262d17fb5950e2441bc9d56a15db9c93877Jason Evans 16857372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill || config_stats) { 1686ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 16877393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t size = chunk->map[pageind-map_bias].bits & ~PAGE_MASK; 1688e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 16897372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && config_stats && opt_junk) 1690e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ptr, 0x5a, size); 16917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 16927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 16937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= size; 1694ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].ndalloc++; 1695ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns--; 16967372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1697e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1698e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1699e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)ptr, true); 1700e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1701e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1702e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1703e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 17048e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t oldsize, size_t size) 1705e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1706e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1707e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size < oldsize); 1708e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1709e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1710e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Shrink the run, and make trailing pages available for other 1711e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocations. 1712e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1713e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1714e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 1715e476f8a161d445211fd6e54fe370275196e66bcbJason Evans true); 17167372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 17177372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 17187372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= oldsize; 1719ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; 1720ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; 17217372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 17227372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 17237372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 17247372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 1725ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1726ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; 1727ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1728e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1729e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1730e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1731e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1732e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1733e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 17348e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t oldsize, size_t size, size_t extra, bool zero) 1735e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1736ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1737ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t npages = oldsize >> LG_PAGE; 17388e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t followsize; 1739e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17407393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(oldsize == (chunk->map[pageind-map_bias].bits & ~PAGE_MASK)); 1741e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1742e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to extend the run. */ 17438e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans assert(size + extra > oldsize); 1744e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 17457393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if (pageind + npages < chunk_npages && 17467393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[pageind+npages-map_bias].bits 17478e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans & CHUNK_MAP_ALLOCATED) == 0 && (followsize = 17487393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[pageind+npages-map_bias].bits & ~PAGE_MASK) >= size - 17497393f44ff025ca67716fc53b68003fd65122fd97Jason Evans oldsize) { 1750e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1751e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The next run is available and sufficiently large. Split the 1752e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * following run, then merge the first part with the existing 1753e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocation. 1754e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1755940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t flag_dirty; 17568e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t splitsize = (oldsize + followsize <= size + extra) 17578e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ? followsize : size + extra - oldsize; 1758e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + 1759ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans ((pageind+npages) << LG_PAGE)), splitsize, true, zero); 1760e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1761088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans size = oldsize + splitsize; 1762ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans npages = size >> LG_PAGE; 1763e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1764940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* 1765940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * Mark the extended run as dirty if either portion of the run 1766940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * was dirty before allocation. This is rather pedantic, 1767940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * because there's not actually any sequence of events that 1768940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * could cause the resulting run to be passed to 1769940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * arena_run_dalloc() with the dirty argument set to false 1770940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * (which is when dirty flag consistency would really matter). 1771940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans */ 1772940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans flag_dirty = (chunk->map[pageind-map_bias].bits & 1773940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_DIRTY) | 1774940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+npages-1-map_bias].bits & 1775940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_DIRTY); 1776940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind-map_bias].bits = size | flag_dirty 1777940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1778940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+npages-1-map_bias].bits = flag_dirty | 1779940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1780990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans 17817372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 17827372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 17837372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= oldsize; 1784ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) 17857372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans - 1].ndalloc++; 1786ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(oldsize >> LG_PAGE) 17877372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans - 1].curruns--; 17887372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 17897372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 17907372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 17917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 1792ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; 1793ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) 17947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans - 1].nrequests++; 1795ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; 1796940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 1797e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1798e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1799e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1800e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1801e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1802e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 1803e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1804e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1805e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 1806e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Try to resize a large allocation, in order to avoid copying. This will 1807e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * always fail if growing an object, and the following run is already in use. 1808e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1809e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 18108e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, 18118e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 1812e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1813e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t psize; 1814e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18158e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize = PAGE_CEILING(size + extra); 1816e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize == oldsize) { 1817e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Same size class. */ 18187372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk && size < oldsize) { 1819e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - 1820e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size); 1821e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1822e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1823e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1824e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1825e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 1826e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1827e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1828e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 1829e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1830e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize < oldsize) { 1831e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Fill before shrinking in order avoid a race. */ 18327372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk) { 1833e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, 1834e476f8a161d445211fd6e54fe370275196e66bcbJason Evans oldsize - size); 1835e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 18368e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, 18378e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize); 1838e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1839e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1840e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 18418e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans oldsize, PAGE_CEILING(size), 18428e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize - PAGE_CEILING(size), zero); 18437372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && ret == false && zero == false && 18447372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans opt_zero) { 1845e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + oldsize), 0, 1846e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size - oldsize); 1847e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1848e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1849e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1850e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1851e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1852e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1853e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 18548e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, 18558e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 1856e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1857e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18588e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* 18598e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * Avoid moving the allocation if the size class can be left the same. 18608e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans */ 1861e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= arena_maxclass) { 1862b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (oldsize <= SMALL_MAXCLASS) { 186349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size 186449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans == oldsize); 1865b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if ((size + extra <= SMALL_MAXCLASS && 186641ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans SMALL_SIZE2BIN(size + extra) == 186741ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && 18688e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size + extra >= oldsize)) { 18697372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk && size < oldsize) { 18708e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset((void *)((uintptr_t)ptr + size), 18718e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 0x5a, oldsize - size); 18728e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 18738e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (ptr); 18748e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 1875e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1876e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 1877b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (size + extra > SMALL_MAXCLASS) { 18788e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (arena_ralloc_large(ptr, oldsize, size, 18798e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans extra, zero) == false) 1880e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 1881e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1882e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1883e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1884e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18858e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Reallocation would require a move. */ 18868e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 18878e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans} 18888e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 18898e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansvoid * 18908e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, 18918e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t alignment, bool zero) 18928e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans{ 18938e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ret; 18948e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t copysize; 18958e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 18968e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Try to avoid moving the allocation. */ 18978e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = arena_ralloc_no_move(ptr, oldsize, size, extra, zero); 18988e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret != NULL) 18998e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (ret); 19008e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 1901e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 19028e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * size and oldsize are different enough that we need to move the 19038e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * object. In that case, fall back to allocating new space and 19048e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * copying. 1905e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 190638d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (alignment != 0) { 190738d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans size_t usize = sa2u(size + extra, alignment, NULL); 190838d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (usize == 0) 190938d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans return (NULL); 191038d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans ret = ipalloc(usize, alignment, zero); 191138d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans } else 19128e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = arena_malloc(size + extra, zero); 1913e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19148e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == NULL) { 19158e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (extra == 0) 19168e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 19178e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Try again, this time without extra. */ 191838d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (alignment != 0) { 191938d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans size_t usize = sa2u(size, alignment, NULL); 192038d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (usize == 0) 192138d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans return (NULL); 192238d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans ret = ipalloc(usize, alignment, zero); 192338d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans } else 19248e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = arena_malloc(size, zero); 19258e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 19268e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == NULL) 19278e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 19288e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 19298e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 19308e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ 19318e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 19328e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* 19338e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * Copy at most size bytes (not size+extra), since the caller has no 19348e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * expectation that the extra bytes will be reliably preserved. 19358e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans */ 1936e476f8a161d445211fd6e54fe370275196e66bcbJason Evans copysize = (size < oldsize) ? size : oldsize; 1937e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(ret, ptr, copysize); 1938e476f8a161d445211fd6e54fe370275196e66bcbJason Evans idalloc(ptr); 1939e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1940e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1941e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1942e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 1943e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_new(arena_t *arena, unsigned ind) 1944e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1945e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 1946e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1947e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19486109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena->ind = ind; 1949597632be188d2bcc135dad2145cc46ef44897aadJason Evans arena->nthreads = 0; 19506109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1951e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (malloc_mutex_init(&arena->lock)) 1952e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 1953e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19547372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 19557372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(&arena->stats, 0, sizeof(arena_stats_t)); 19567372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats = 19577372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans (malloc_large_stats_t *)base_alloc(nlclasses * 19587372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans sizeof(malloc_large_stats_t)); 19597372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (arena->stats.lstats == NULL) 19607372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans return (true); 19617372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(arena->stats.lstats, 0, nlclasses * 19627372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans sizeof(malloc_large_stats_t)); 19637372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_tcache) 19647372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ql_new(&arena->tcache_ql); 19657372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1966e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19677372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) 19687372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->prof_accumbytes = 0; 1969d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 1970e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize chunks. */ 19712caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_new(&arena->chunks_dirty); 1972e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 1973e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1974e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive = 0; 1975e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty = 0; 1976799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory = 0; 1977e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 197819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_new(&arena->runs_avail_clean); 197919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_new(&arena->runs_avail_dirty); 1980e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1981e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize bins. */ 1982b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans for (i = 0; i < NBINS; i++) { 1983e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 198486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 198586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 1986e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1987e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 19887372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 19897372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 1990e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1991e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1992e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1993e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1994e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 199549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 199649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Calculate bin_info->run_size such that it meets the following constraints: 199749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 199849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) bin_info->run_size >= min_run_size 199949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) bin_info->run_size <= arena_maxclass 200049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 200147e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans * *) bin_info->nregs <= RUN_MAXREGS 200249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 200384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also 200484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * calculated here, since these settings are all interdependent. 200549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 200649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstatic size_t 200749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansbin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) 200849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 200949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t try_run_size, good_run_size; 201049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_nregs, good_nregs; 201149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_hdr_size, good_hdr_size; 201284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans uint32_t try_bitmap_offset, good_bitmap_offset; 201349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_ctx0_offset, good_ctx0_offset; 201449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_reg0_offset, good_reg0_offset; 201549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2016ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans assert(min_run_size >= PAGE); 201749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(min_run_size <= arena_maxclass); 201849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 201949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 202049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Calculate known-valid settings before entering the run_size 202149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * expansion loop, so that the first part of the loop always copies 202249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * valid settings. 202349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 202449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * The do..while loop iteratively reduces the number of regions until 202549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * the run header and the regions no longer overlap. A closed formula 202649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * would be quite messy, since there is an interdependency between the 202749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * header's mask length and the number of regions. 202849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 202949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_run_size = min_run_size; 203049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_size) 203149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 203247e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans if (try_nregs > RUN_MAXREGS) { 203347e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans try_nregs = RUN_MAXREGS 203447e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 203547e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans } 203649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 203749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs--; 203849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = sizeof(arena_run_t); 203984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Pad to a long boundary. */ 204084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size = LONG_CEILING(try_hdr_size); 204184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_bitmap_offset = try_hdr_size; 204284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Add space for bitmap. */ 204384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size += bitmap_size(try_nregs); 20447372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof && opt_prof && prof_promote == false) { 204549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Pad to a quantum boundary. */ 204649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 204749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = try_hdr_size; 204849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Add space for one (prof_ctx_t *) per region. */ 204949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size += try_nregs * sizeof(prof_ctx_t *); 205049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } else 205149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = 0; 205249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_reg0_offset = try_run_size - (try_nregs * 205349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size); 205449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } while (try_hdr_size > try_reg0_offset); 205549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 205649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* run_size expansion loop. */ 205749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 205849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 205949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Copy valid settings before trying more aggressive settings. 206049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 206149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_run_size = try_run_size; 206249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_nregs = try_nregs; 206349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_hdr_size = try_hdr_size; 206484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans good_bitmap_offset = try_bitmap_offset; 206549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_ctx0_offset = try_ctx0_offset; 206649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_reg0_offset = try_reg0_offset; 206749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 206849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Try more aggressive settings. */ 2069ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans try_run_size += PAGE; 207049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / 207149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size) 207249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 207347e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans if (try_nregs > RUN_MAXREGS) { 207447e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans try_nregs = RUN_MAXREGS 207547e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 207647e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans } 207749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 207849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs--; 207949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = sizeof(arena_run_t); 208084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Pad to a long boundary. */ 208184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size = LONG_CEILING(try_hdr_size); 208284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_bitmap_offset = try_hdr_size; 208384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Add space for bitmap. */ 208484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size += bitmap_size(try_nregs); 20857372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof && opt_prof && prof_promote == false) { 208649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Pad to a quantum boundary. */ 208749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 208849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = try_hdr_size; 208949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 209049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Add space for one (prof_ctx_t *) per region. 209149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 209249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size += try_nregs * 209349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans sizeof(prof_ctx_t *); 209449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 209549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_reg0_offset = try_run_size - (try_nregs * 209649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size); 209749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } while (try_hdr_size > try_reg0_offset); 209849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } while (try_run_size <= arena_maxclass 209949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans && try_run_size <= arena_maxclass 210049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans && RUN_MAX_OVRHD * (bin_info->reg_size << 3) > RUN_MAX_OVRHD_RELAX 210147e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size 210247e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans && try_nregs < RUN_MAXREGS); 210349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 210449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(good_hdr_size <= good_reg0_offset); 210549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 210649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Copy final settings. */ 210749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->run_size = good_run_size; 210849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->nregs = good_nregs; 210984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bin_info->bitmap_offset = good_bitmap_offset; 211049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->ctx0_offset = good_ctx0_offset; 211149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg0_offset = good_reg0_offset; 211249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 211349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (good_run_size); 211449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 211549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2116b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansstatic void 211749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansbin_info_init(void) 211849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 211949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 2120ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans size_t prev_run_size = PAGE; 2121b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans 2122b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define SIZE_CLASS(bin, delta, size) \ 2123b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info = &arena_bin_info[bin]; \ 2124b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info->reg_size = size; \ 2125b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ 2126b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 2127b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans SIZE_CLASSES 2128b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef SIZE_CLASS 212949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 213049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2131b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansvoid 2132a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evansarena_boot(void) 2133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2134a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans size_t header_size; 21357393f44ff025ca67716fc53b68003fd65122fd97Jason Evans unsigned i; 2136e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Compute the header size such that it is large enough to contain the 21397393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * page map. The page map is biased to omit entries for the header 21407393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * itself, so some iteration is necessary to compute the map bias. 21417393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 21427393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 1) Compute safe header_size and map_bias values that include enough 21437393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * space for an unbiased page map. 21447393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 2) Refine map_bias based on (1) to omit the header pages in the page 21457393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * map. The resulting map_bias may be one too small. 21467393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 3) Refine map_bias based on (2). The result will be >= the result 21477393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * from (2), and will always be correct. 2148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 21497393f44ff025ca67716fc53b68003fd65122fd97Jason Evans map_bias = 0; 21507393f44ff025ca67716fc53b68003fd65122fd97Jason Evans for (i = 0; i < 3; i++) { 2151ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans header_size = offsetof(arena_chunk_t, map) + 2152ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); 2153ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) 2154ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans != 0); 21557393f44ff025ca67716fc53b68003fd65122fd97Jason Evans } 21567393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(map_bias > 0); 21577393f44ff025ca67716fc53b68003fd65122fd97Jason Evans 2158ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans arena_maxclass = chunksize - (map_bias << LG_PAGE); 2159a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans 2160b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info_init(); 2161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 21624e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 21634e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 21644e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansarena_prefork(arena_t *arena) 21654e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 21664e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans unsigned i; 21674e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 21684e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_prefork(&arena->lock); 21694e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans for (i = 0; i < NBINS; i++) 21704e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_prefork(&arena->bins[i].lock); 21714e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 21724e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 21734e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 21744e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansarena_postfork_parent(arena_t *arena) 21754e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 21764e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans unsigned i; 21774e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 21784e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans for (i = 0; i < NBINS; i++) 21794e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_parent(&arena->bins[i].lock); 21804e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_parent(&arena->lock); 21814e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 21824e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 21834e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid 21844e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansarena_postfork_child(arena_t *arena) 21854e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans{ 21864e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans unsigned i; 21874e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans 21884e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans for (i = 0; i < NBINS; i++) 21894e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_child(&arena->bins[i].lock); 21904e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans malloc_mutex_postfork_child(&arena->lock); 21914e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evans} 2192