arena.c revision bdcadf41e961a3c6bbb37d8d24e4b68a27f2b952
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define JEMALLOC_ARENA_C_ 2376b1529a383c39adf4674baf6db83a5e63f97acJason Evans#include "jemalloc/internal/jemalloc_internal.h" 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Data. */ 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evansssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; 8b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansarena_bin_info_t arena_bin_info[NBINS]; 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 10b172610317babc7f365584ddd7fdaf4eb8d9d04cJason EvansJEMALLOC_ATTR(aligned(CACHELINE)) 11b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansconst uint8_t small_size2bin[] = { 1241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#define S2B_8(i) i, 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_16(i) S2B_8(i) S2B_8(i) 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_32(i) S2B_16(i) S2B_16(i) 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_64(i) S2B_32(i) S2B_32(i) 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_128(i) S2B_64(i) S2B_64(i) 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define S2B_256(i) S2B_128(i) S2B_128(i) 18b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_512(i) S2B_256(i) S2B_256(i) 19b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_1024(i) S2B_512(i) S2B_512(i) 20b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_2048(i) S2B_1024(i) S2B_1024(i) 21b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_4096(i) S2B_2048(i) S2B_2048(i) 22b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define S2B_8192(i) S2B_4096(i) S2B_4096(i) 23b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define SIZE_CLASS(bin, delta, size) \ 24b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans S2B_##delta(bin) 25b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans SIZE_CLASSES 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_8 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_16 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_32 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_64 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_128 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#undef S2B_256 32b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_512 33b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_1024 34b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_2048 35b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_4096 36b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef S2B_8192 37b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef SIZE_CLASS 38b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans}; 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 41e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Function prototypes for non-inline static functions. */ 42e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 43e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, 44e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool large, bool zero); 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t *arena_chunk_alloc(arena_t *arena); 46e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool zero); 496005f0710cf07d60659d91b20b7ff5592d310027Jason Evansstatic void arena_purge(arena_t *arena, bool all); 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty); 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize); 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); 55e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t *arena_bin_runs_first(arena_bin_t *bin); 56e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run); 57e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run); 58e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t *arena_bin_nonfull_run_tryget(arena_bin_t *bin); 59e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); 61088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansstatic void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 62088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_bin_t *bin); 63e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, 64e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run, arena_bin_t *bin); 65940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansstatic void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, 66940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_t *run, arena_bin_t *bin); 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, 688e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ptr, size_t oldsize, size_t size); 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, 708e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); 718e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansstatic bool arena_ralloc_large(void *ptr, size_t oldsize, size_t size, 728e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t extra, bool zero); 7349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstatic size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, 7449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t min_run_size); 75b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansstatic void bin_info_init(void); 76e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm = (uintptr_t)a; 83e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t b_mapelm = (uintptr_t)b; 84e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 85e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(a != NULL); 86e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(b != NULL); 87e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); 89e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 90e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 91f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansrb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, 937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans u.rb_link, arena_run_comp) 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 95e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline int 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 98e476f8a161d445211fd6e54fe370275196e66bcbJason Evans int ret; 99e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t a_size = a->bits & ~PAGE_MASK; 100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t b_size = b->bits & ~PAGE_MASK; 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1020b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert((a->bits & CHUNK_MAP_KEY) == CHUNK_MAP_KEY || (a->bits & 1030b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans CHUNK_MAP_DIRTY) == (b->bits & CHUNK_MAP_DIRTY)); 10419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_size > b_size) - (a_size < b_size); 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == 0) { 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uintptr_t a_mapelm, b_mapelm; 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((a->bits & CHUNK_MAP_KEY) != CHUNK_MAP_KEY) 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = (uintptr_t)a; 111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Treat keys as though they are lower than anything 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * else. 115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans a_mapelm = 0; 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans b_mapelm = (uintptr_t)b; 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 126f3ff75289be32382fa455b4436871e4958fe6bf9Jason Evans/* Generate red-black tree functions. */ 1277372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansrb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, 1287372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans u.rb_link, arena_avail_comp) 129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 130e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic inline void * 13149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) 132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 13484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned regind; 13584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 13684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 137e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1381e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(run->nfree > 0); 13984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); 140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 14184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); 14284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + 14384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)(bin_info->reg_size * regind)); 1441e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree--; 14584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans if (regind == run->nextind) 14684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans run->nextind++; 14784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(regind < run->nextind); 1481e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans return (ret); 1496109fe07a14b7a619365977d9523db9f8b333792Jason Evans} 1506109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1516109fe07a14b7a619365977d9523db9f8b333792Jason Evansstatic inline void 1521e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evansarena_run_reg_dalloc(arena_run_t *run, void *ptr) 1536109fe07a14b7a619365977d9523db9f8b333792Jason Evans{ 15449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 15549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, run->bin); 15649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 15784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans unsigned regind = arena_run_regind(run, bin_info, ptr); 15884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 15984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 16084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 16149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(run->nfree < bin_info->nregs); 1621e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Freeing an interior pointer can cause assertion failure. */ 1631e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 16449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset)) % (uintptr_t)bin_info->reg_size 1651e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans == 0); 16621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans assert((uintptr_t)ptr >= (uintptr_t)run + 16749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset); 16884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Freeing an unallocated pointer can cause assertion failure. */ 16984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); 170e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_unset(bitmap, &bin_info->bitmap_info, regind); 1721e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans run->nfree++; 173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17521fb95bba6ea922e0523f269c0d9a32640047a29Jason Evansstatic inline void 17621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evansarena_chunk_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) 17721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans{ 178d4bab21756279db540866998099522dbd39c05f7Jason Evans size_t i; 1797372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << 1807372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans PAGE_SHIFT)); 181d4bab21756279db540866998099522dbd39c05f7Jason Evans 182d4bab21756279db540866998099522dbd39c05f7Jason Evans for (i = 0; i < PAGE_SIZE / sizeof(size_t); i++) 18321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans assert(p[i] == 0); 18421fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans} 18521fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans 186e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 187e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, 188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool zero) 189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 191bdcadf41e961a3c6bbb37d8d24e4b68a27f2b952Jason Evans size_t run_ind, total_pages, need_pages, rem_pages, i; 19219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t flag_dirty; 19319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t *runs_avail; 194e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 195e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) 197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> PAGE_SHIFT); 1987393f44ff025ca67716fc53b68003fd65122fd97Jason Evans flag_dirty = chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY; 19919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans runs_avail = (flag_dirty != 0) ? &arena->runs_avail_dirty : 20019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans &arena->runs_avail_clean; 2017393f44ff025ca67716fc53b68003fd65122fd97Jason Evans total_pages = (chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) >> 202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT; 2037393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert((chunk->map[run_ind+total_pages-1-map_bias].bits & 2047393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_DIRTY) == flag_dirty); 205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans need_pages = (size >> PAGE_SHIFT); 206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages > 0); 207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(need_pages <= total_pages); 208e476f8a161d445211fd6e54fe370275196e66bcbJason Evans rem_pages = total_pages - need_pages; 209e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2107393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_remove(runs_avail, &chunk->map[run_ind-map_bias]); 2117372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 2127372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans /* 2137372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * Update stats_cactive if nactive is crossing a chunk 2147372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * multiple. 2157372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans */ 2167372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t cactive_diff = CHUNK_CEILING((arena->nactive + 2177372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans need_pages) << PAGE_SHIFT) - CHUNK_CEILING(arena->nactive << 2187372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans PAGE_SHIFT); 2197372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (cactive_diff != 0) 2207372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans stats_cactive_add(cactive_diff); 2217372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive += need_pages; 223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 224e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Keep track of trailing unused pages for later use. */ 225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (rem_pages > 0) { 22619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (flag_dirty != 0) { 2277393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+need_pages-map_bias].bits = 2287393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (rem_pages << PAGE_SHIFT) | CHUNK_MAP_DIRTY; 2297393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+total_pages-1-map_bias].bits = 2307393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (rem_pages << PAGE_SHIFT) | CHUNK_MAP_DIRTY; 23119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 2327393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+need_pages-map_bias].bits = 2337393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (rem_pages << PAGE_SHIFT) | 2347393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+need_pages-map_bias].bits & 2353377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans CHUNK_MAP_UNZEROED); 2367393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+total_pages-1-map_bias].bits = 2377393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (rem_pages << PAGE_SHIFT) | 2387393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+total_pages-1-map_bias].bits & 2393377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans CHUNK_MAP_UNZEROED); 24019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 24119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_insert(runs_avail, 2427393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[run_ind+need_pages-map_bias]); 243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 24519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Update dirty page accounting. */ 24619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (flag_dirty != 0) { 24719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans chunk->ndirty -= need_pages; 24819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena->ndirty -= need_pages; 24919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 25019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 25119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 25219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Update the page map separately for large vs. small runs, since it is 25319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * possible to avoid iteration for large mallocs. 25419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 25519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (large) { 256e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero) { 25719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (flag_dirty == 0) { 25819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 25919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * The run is clean, so some pages may be 26019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * zeroed (i.e. never before touched). 26119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 26219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans for (i = 0; i < need_pages; i++) { 2637393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[run_ind+i-map_bias].bits 2643377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans & CHUNK_MAP_UNZEROED) != 0) { 26519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans memset((void *)((uintptr_t) 26621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans chunk + ((run_ind+i) << 26719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)), 0, 26819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SIZE); 2697372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } else if (config_debug) { 27021fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed( 27121fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans chunk, run_ind+i); 272940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 27319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 27419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 27519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 27619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * The run is dirty, so all pages must be 27719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * zeroed. 27819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 27919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans memset((void *)((uintptr_t)chunk + (run_ind << 28019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)), 0, (need_pages << 28119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)); 282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 28619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Set the last element first, in case the run only contains one 28719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * page (i.e. both statements set the same element). 288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 2897393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+need_pages-1-map_bias].bits = 2907393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED | flag_dirty; 291e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans chunk->map[run_ind-map_bias].bits = size | flag_dirty | 292e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 29319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } else { 29419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans assert(zero == false); 29519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 296940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * Propagate the dirty and unzeroed flags to the allocated 297940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * small run, so that arena_dalloc_bin_run() has the ability to 298940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * conditionally trim clean pages. 29919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 300397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans chunk->map[run_ind-map_bias].bits = 301397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) | 302397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans CHUNK_MAP_ALLOCATED | flag_dirty; 30321fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans /* 30421fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans * The first page will always be dirtied during small run 30521fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans * initialization, so a validation failure here would not 30621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans * actually cause an observable failure. 30721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans */ 3087372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug && flag_dirty == 0 && 30921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) 31021fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans == 0) 31121fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed(chunk, run_ind); 31219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans for (i = 1; i < need_pages - 1; i++) { 3137393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+i-map_bias].bits = (i << PAGE_SHIFT) 314397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans | (chunk->map[run_ind+i-map_bias].bits & 315397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED; 3167372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug && flag_dirty == 0 && 31721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans (chunk->map[run_ind+i-map_bias].bits & 31821fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans CHUNK_MAP_UNZEROED) == 0) 31921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed(chunk, run_ind+i); 32019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 321004ed142a66529ecf4a55e8f4fa42ff2e535f586Jason Evans chunk->map[run_ind+need_pages-1-map_bias].bits = ((need_pages 322397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans - 1) << PAGE_SHIFT) | 323397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans (chunk->map[run_ind+need_pages-1-map_bias].bits & 324397e5111b5efd49f61f73c1bad0375c7885a6128Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED | flag_dirty; 3257372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug && flag_dirty == 0 && 32621fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans (chunk->map[run_ind+need_pages-1-map_bias].bits & 32721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans CHUNK_MAP_UNZEROED) == 0) { 32821fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans arena_chunk_validate_zeroed(chunk, 32921fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans run_ind+need_pages-1); 33021fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans } 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_chunk_t * 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_alloc(arena_t *arena) 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t i; 339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (arena->spare != NULL) { 34119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t *runs_avail; 34219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena->spare; 344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 34519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 34619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Insert the run into the appropriate runs_avail_* tree. */ 3477393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) 34819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans runs_avail = &arena->runs_avail_clean; 34919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans else 35019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans runs_avail = &arena->runs_avail_dirty; 3518de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[0].bits & ~PAGE_MASK) == arena_maxclass); 3528de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[chunk_npages-1-map_bias].bits & ~PAGE_MASK) 3538de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans == arena_maxclass); 3548de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 3558de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (chunk->map[chunk_npages-1-map_bias].bits & 3568de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_DIRTY)); 3577393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_insert(runs_avail, &chunk->map[0]); 358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 35941631d00618d7262125e501c91d31b4d70e605faJason Evans bool zero; 3603377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans size_t unzeroed; 36141631d00618d7262125e501c91d31b4d70e605faJason Evans 36241631d00618d7262125e501c91d31b4d70e605faJason Evans zero = false; 363e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&arena->lock); 3642dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans chunk = (arena_chunk_t *)chunk_alloc(chunksize, false, &zero); 365e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&arena->lock); 366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (chunk == NULL) 367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 3687372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 3697372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.mapped += chunksize; 370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->arena = arena; 3722caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_elm_new(chunk, link_dirty); 373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = false; 374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Claim that no pages are in use, since the header is merely 377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * overhead. 378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 379e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->ndirty = 0; 380e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 381e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Initialize the map to contain one maximal free untouched run. 38341631d00618d7262125e501c91d31b4d70e605faJason Evans * Mark the pages as zeroed iff chunk_alloc() returned a zeroed 38441631d00618d7262125e501c91d31b4d70e605faJason Evans * chunk. 385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 3863377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; 3873377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans chunk->map[0].bits = arena_maxclass | unzeroed; 3883377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans /* 3893377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * There is no need to initialize the internal page map entries 3903377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * unless the chunk is not zeroed. 3913377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans */ 3923377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans if (zero == false) { 3933377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans for (i = map_bias+1; i < chunk_npages-1; i++) 3943377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans chunk->map[i-map_bias].bits = unzeroed; 3957372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } else if (config_debug) { 396940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans for (i = map_bias+1; i < chunk_npages-1; i++) 397940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert(chunk->map[i-map_bias].bits == unzeroed); 398940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 399004ed142a66529ecf4a55e8f4fa42ff2e535f586Jason Evans chunk->map[chunk_npages-1-map_bias].bits = arena_maxclass | 400004ed142a66529ecf4a55e8f4fa42ff2e535f586Jason Evans unzeroed; 401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 40219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Insert the run into the runs_avail_clean tree. */ 40319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_insert(&arena->runs_avail_clean, 4047393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[0]); 40519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 406e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 407e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (chunk); 408e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 409e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 410e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 411e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) 412e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 41319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t *runs_avail; 414e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4158d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans /* 4168d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * Remove run from the appropriate runs_avail_* tree, so that the arena 4178d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * does not use it. 4188d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans */ 4197393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) 4208d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans runs_avail = &arena->runs_avail_clean; 4218d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans else 4228d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans runs_avail = &arena->runs_avail_dirty; 4237393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_remove(runs_avail, &chunk->map[0]); 4248d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans 4258d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (arena->spare != NULL) { 426e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_chunk_t *spare = arena->spare; 427e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 4288d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans arena->spare = chunk; 429e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (spare->dirtied) { 430e00572b384c81bd2aba57fac32f7077a34388915Jason Evans ql_remove(&chunk->arena->chunks_dirty, spare, 4312caa4715ed4f787f263239ff97dd824636289286Jason Evans link_dirty); 432e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena->ndirty -= spare->ndirty; 433e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 434e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&arena->lock); 43512a488782681cbd740a5f54e0b7e74ea84858e21Jason Evans chunk_dealloc((void *)spare, chunksize, true); 436e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&arena->lock); 4377372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 4387372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.mapped -= chunksize; 4398d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans } else 4408d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans arena->spare = chunk; 441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 442e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 443e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 444e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) 445e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 447e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 448e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm, key; 449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 451e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 452e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 453e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Search the arena's chunks for the lowest best fit. */ 454e476f8a161d445211fd6e54fe370275196e66bcbJason Evans key.bits = size | CHUNK_MAP_KEY; 45519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); 45619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (mapelm != NULL) { 45719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 4587393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 4597393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 4607393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 46119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 46219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 46319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)); 46419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_run_split(arena, run, size, large, zero); 46519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans return (run); 46619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 46719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); 468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 4707393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 4717393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 4727393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 474e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 475e00572b384c81bd2aba57fac32f7077a34388915Jason Evans PAGE_SHIFT)); 476e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, run, size, large, zero); 477e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (run); 478e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 479e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 480e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 481e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * No usable runs. Create a new chunk from which to allocate the run. 482e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 483e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = arena_chunk_alloc(arena); 484e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (chunk != NULL) { 4857393f44ff025ca67716fc53b68003fd65122fd97Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (map_bias << 4867393f44ff025ca67716fc53b68003fd65122fd97Jason Evans PAGE_SHIFT)); 487e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_split(arena, run, size, large, zero); 488e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 489e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 490e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 491e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 492e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_chunk_alloc() failed, but another thread may have made 493e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * sufficient memory available while this one dropped arena->lock in 494e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_chunk_alloc(), so search one more time. 495e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 49619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); 49719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (mapelm != NULL) { 49819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 4997393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 5007393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 5017393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 50219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 50319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 50419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)); 50519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_run_split(arena, run, size, large, zero); 50619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans return (run); 50719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 50819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); 509e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (mapelm != NULL) { 510e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); 5117393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - 5127393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) 5137393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + map_bias; 514e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 515e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << 516e00572b384c81bd2aba57fac32f7077a34388915Jason Evans PAGE_SHIFT)); 517e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_split(arena, run, size, large, zero); 518e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 519e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 520e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 521e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (NULL); 522e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 523e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 52405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansstatic inline void 52505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansarena_maybe_purge(arena_t *arena) 52605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans{ 52705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 52805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Enforce opt_lg_dirty_mult. */ 529799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans if (opt_lg_dirty_mult >= 0 && arena->ndirty > arena->npurgatory && 530799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans (arena->ndirty - arena->npurgatory) > chunk_npages && 531799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans (arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - 532799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory)) 5336005f0710cf07d60659d91b20b7ff5592d310027Jason Evans arena_purge(arena, false); 53405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans} 53505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 53605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansstatic inline void 53705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evansarena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) 53805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans{ 53905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_head(arena_chunk_map_t) mapelms; 54005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_map_t *mapelm; 5413377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans size_t pageind, flag_unzeroed; 54205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t ndirty; 54305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t nmadvise; 54405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 54505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_new(&mapelms); 54605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 5473377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans flag_unzeroed = 5482dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED 5492dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans /* 5502dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * madvise(..., MADV_DONTNEED) results in zero-filled pages for anonymous 5512dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans * mappings, but not for file-backed mappings. 5522dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans */ 5534162627757889ea999264c2ddbc3c354768774e2Jason Evans 0 5543377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans#else 5554162627757889ea999264c2ddbc3c354768774e2Jason Evans CHUNK_MAP_UNZEROED 5562dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#endif 5574162627757889ea999264c2ddbc3c354768774e2Jason Evans ; 558c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 55905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 56005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * If chunk is the spare, temporarily re-allocate it, 1) so that its 56119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run is reinserted into runs_avail_dirty, and 2) so that it cannot be 56205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * completely discarded by another thread while arena->lock is dropped 56305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * by this thread. Note that the arena_run_dalloc() call will 56405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * implicitly deallocate the chunk, so no explicit action is required 56505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * in this function to deallocate the chunk. 56619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 56719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Note that once a chunk contains dirty pages, it cannot again contain 56819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * a single run unless 1) it is a dirty run, or 2) this function purges 56919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * dirty pages and causes the transition to a single clean run. Thus 57019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * (chunk == arena->spare) is possible, but it is not possible for 57119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * this function to be called on the spare unless it contains a dirty 57219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run. 57305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 57419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (chunk == arena->spare) { 5757393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) != 0); 57605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_alloc(arena); 57719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans } 57805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 57919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Temporarily allocate all free dirty runs within chunk. */ 5807393f44ff025ca67716fc53b68003fd65122fd97Jason Evans for (pageind = map_bias; pageind < chunk_npages;) { 5817393f44ff025ca67716fc53b68003fd65122fd97Jason Evans mapelm = &chunk->map[pageind-map_bias]; 58205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans if ((mapelm->bits & CHUNK_MAP_ALLOCATED) == 0) { 58319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t npages; 58405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 58519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans npages = mapelm->bits >> PAGE_SHIFT; 586e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind + npages <= chunk_npages); 58719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans if (mapelm->bits & CHUNK_MAP_DIRTY) { 588c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans size_t i; 589c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 59019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_remove( 59119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans &arena->runs_avail_dirty, mapelm); 592c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 593940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans mapelm->bits = (npages << PAGE_SHIFT) | 5948de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans flag_unzeroed | CHUNK_MAP_LARGE | 5958de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_ALLOCATED; 596c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans /* 597c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans * Update internal elements in the page map, so 5983377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * that CHUNK_MAP_UNZEROED is properly set. 599c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans */ 600c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans for (i = 1; i < npages - 1; i++) { 6017393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[pageind+i-map_bias].bits = 6023377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans flag_unzeroed; 603c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans } 604c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans if (npages > 1) { 6057393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[ 6067393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind+npages-1-map_bias].bits = 6078de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans flag_unzeroed | CHUNK_MAP_LARGE | 6088de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_ALLOCATED; 609c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans } 610c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans 6117372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 6127372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans /* 6137372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * Update stats_cactive if nactive is 6147372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * crossing a chunk multiple. 6157372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans */ 6167372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t cactive_diff = 6177372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans CHUNK_CEILING((arena->nactive + 6187372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans npages) << PAGE_SHIFT) - 6197372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans CHUNK_CEILING(arena->nactive << 6207372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans PAGE_SHIFT); 6217372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (cactive_diff != 0) 6227372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans stats_cactive_add(cactive_diff); 6237372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 62419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena->nactive += npages; 62519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Append to list for later processing. */ 62619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans ql_elm_new(mapelm, u.ql_link); 62719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans ql_tail_insert(&mapelms, mapelm, u.ql_link); 62805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 62919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 63005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans pageind += npages; 63105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } else { 63205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Skip allocated run. */ 633c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans if (mapelm->bits & CHUNK_MAP_LARGE) 63419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans pageind += mapelm->bits >> PAGE_SHIFT; 635c03a63d68df9ddbb80f997d8be7bc04e5f3d53ebJason Evans else { 63605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t) 637e69bee01de62b56d3e585042d341743239568043Jason Evans chunk + (uintptr_t)(pageind << PAGE_SHIFT)); 638e69bee01de62b56d3e585042d341743239568043Jason Evans 63919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans assert((mapelm->bits >> PAGE_SHIFT) == 0); 64049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(arena, 64149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans run->bin); 64249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = 64349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans &arena_bin_info[binind]; 64449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans pageind += bin_info->run_size >> PAGE_SHIFT; 64505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 64605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 64705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 648e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind == chunk_npages); 64905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 6507372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) 6517372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ndirty = chunk->ndirty; 6527372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 6537372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.purged += chunk->ndirty; 65405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena->ndirty -= chunk->ndirty; 65505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans chunk->ndirty = 0; 65605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_remove(&arena->chunks_dirty, chunk, link_dirty); 65705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans chunk->dirtied = false; 65805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 65905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans malloc_mutex_unlock(&arena->lock); 6607372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 6617372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans nmadvise = 0; 66205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_foreach(mapelm, &mapelms, u.ql_link) { 6637393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 6647393f44ff025ca67716fc53b68003fd65122fd97Jason Evans sizeof(arena_chunk_map_t)) + map_bias; 66519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t npages = mapelm->bits >> PAGE_SHIFT; 66605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 667e69bee01de62b56d3e585042d341743239568043Jason Evans assert(pageind + npages <= chunk_npages); 66819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans assert(ndirty >= npages); 6697372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) 6707372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ndirty -= npages; 6712dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans 6722dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED 6737372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans# define MADV_PURGE MADV_DONTNEED 6742dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#elif defined(JEMALLOC_PURGE_MADVISE_FREE) 6757372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans# define MADV_PURGE MADV_FREE 6762dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#else 6772dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans# error "No method defined for purging unused dirty pages." 6782dbecf1f6267fae7a161b9c39cfd4d04ce168a29Jason Evans#endif 6797372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans madvise((void *)((uintptr_t)chunk + (pageind << PAGE_SHIFT)), 6807372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans (npages << PAGE_SHIFT), MADV_PURGE); 6817372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans#undef MADV_PURGE 6827372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 6837372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans nmadvise++; 68405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 68505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans assert(ndirty == 0); 68605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans malloc_mutex_lock(&arena->lock); 6877372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 6887372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmadvise += nmadvise; 68905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 69005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Deallocate runs. */ 69105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans for (mapelm = ql_first(&mapelms); mapelm != NULL; 69205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans mapelm = ql_first(&mapelms)) { 6937393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / 6947393f44ff025ca67716fc53b68003fd65122fd97Jason Evans sizeof(arena_chunk_map_t)) + map_bias; 69505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 69605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans (uintptr_t)(pageind << PAGE_SHIFT)); 69705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 69805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_remove(&mapelms, mapelm, u.ql_link); 69905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_run_dalloc(arena, run, false); 70005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 70105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans} 70205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 703e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 7046005f0710cf07d60659d91b20b7ff5592d310027Jason Evansarena_purge(arena_t *arena, bool all) 705e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 706e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 70705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t npurgatory; 7087372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) { 7097372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t ndirty = 0; 710e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7117372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ql_foreach(chunk, &arena->chunks_dirty, link_dirty) { 7127372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans assert(chunk->dirtied); 7137372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ndirty += chunk->ndirty; 7147372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 7157372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans assert(ndirty == arena->ndirty); 7162caa4715ed4f787f263239ff97dd824636289286Jason Evans } 717af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans assert(arena->ndirty > arena->npurgatory || all); 718f9a8edbb50f8cbcaf8ed62b36e8d7191ed223d2aJason Evans assert(arena->ndirty - arena->npurgatory > chunk_npages || all); 719af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - 720f9a8edbb50f8cbcaf8ed62b36e8d7191ed223d2aJason Evans arena->npurgatory) || all); 721e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7227372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 7237372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.npurge++; 724e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 725e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 72605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * Compute the minimum number of pages that this thread should try to 727799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * purge, and add the result to arena->npurgatory. This will keep 728799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * multiple threads from racing to reduce ndirty below the threshold. 729e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 7306005f0710cf07d60659d91b20b7ff5592d310027Jason Evans npurgatory = arena->ndirty - arena->npurgatory; 731af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans if (all == false) { 732af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans assert(npurgatory >= arena->nactive >> opt_lg_dirty_mult); 7336005f0710cf07d60659d91b20b7ff5592d310027Jason Evans npurgatory -= arena->nactive >> opt_lg_dirty_mult; 734af8ad3ec6abe9840bd503535a604a6173ce73515Jason Evans } 735799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory += npurgatory; 736799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans 73705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans while (npurgatory > 0) { 73805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Get next chunk with dirty pages. */ 7392caa4715ed4f787f263239ff97dd824636289286Jason Evans chunk = ql_first(&arena->chunks_dirty); 74005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans if (chunk == NULL) { 74105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 74205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * This thread was unable to purge as many pages as 74305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * originally intended, due to races with other threads 744799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * that either did some of the purging work, or re-used 745799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * dirty pages. 74605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 747799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory -= npurgatory; 748799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans return; 749e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 75005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans while (chunk->ndirty == 0) { 7512caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_remove(&arena->chunks_dirty, chunk, link_dirty); 752e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk->dirtied = false; 75305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans chunk = ql_first(&arena->chunks_dirty); 75405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans if (chunk == NULL) { 75505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* Same logic as for above. */ 756799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory -= npurgatory; 757799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans return; 75805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } 759e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 76005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 761799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans if (chunk->ndirty > npurgatory) { 762799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans /* 763799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * This thread will, at a minimum, purge all the dirty 764799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * pages in chunk, so set npurgatory to reflect this 765799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * thread's commitment to purge the pages. This tends 766799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * to reduce the chances of the following scenario: 767799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 768799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 1) This thread sets arena->npurgatory such that 769799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * (arena->ndirty - arena->npurgatory) is at the 770799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * threshold. 771799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 2) This thread drops arena->lock. 772799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 3) Another thread causes one or more pages to be 773799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * dirtied, and immediately determines that it must 774799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * purge dirty pages. 775799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * 776799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * If this scenario *does* play out, that's okay, 777799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * because all of the purging work being done really 778799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * needs to happen. 779799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans */ 780799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory += chunk->ndirty - npurgatory; 781799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans npurgatory = chunk->ndirty; 782799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans } 783799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans 784799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory -= chunk->ndirty; 785799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans npurgatory -= chunk->ndirty; 78605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_chunk_purge(arena, chunk); 787e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 788e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 789e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 7906005f0710cf07d60659d91b20b7ff5592d310027Jason Evansvoid 7916005f0710cf07d60659d91b20b7ff5592d310027Jason Evansarena_purge_all(arena_t *arena) 7926005f0710cf07d60659d91b20b7ff5592d310027Jason Evans{ 7936005f0710cf07d60659d91b20b7ff5592d310027Jason Evans 7946005f0710cf07d60659d91b20b7ff5592d310027Jason Evans malloc_mutex_lock(&arena->lock); 7956005f0710cf07d60659d91b20b7ff5592d310027Jason Evans arena_purge(arena, true); 7966005f0710cf07d60659d91b20b7ff5592d310027Jason Evans malloc_mutex_unlock(&arena->lock); 7976005f0710cf07d60659d91b20b7ff5592d310027Jason Evans} 7986005f0710cf07d60659d91b20b7ff5592d310027Jason Evans 799e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 800e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) 801e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 802e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 80319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans size_t size, run_ind, run_pages, flag_dirty; 80419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t *runs_avail; 805e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 806e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 807e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) 808e476f8a161d445211fd6e54fe370275196e66bcbJason Evans >> PAGE_SHIFT); 8097393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(run_ind >= map_bias); 810e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(run_ind < chunk_npages); 8118de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_LARGE) != 0) { 8127393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size = chunk->map[run_ind-map_bias].bits & ~PAGE_MASK; 8138de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert(size == PAGE_SIZE || 8148de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & 8158de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans ~PAGE_MASK) == 0); 8168de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & 8178de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_LARGE) != 0); 8188de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & 8198de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans CHUNK_MAP_ALLOCATED) != 0); 82049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } else { 82149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(arena, run->bin); 82249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 82349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size = bin_info->run_size; 82449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 825e476f8a161d445211fd6e54fe370275196e66bcbJason Evans run_pages = (size >> PAGE_SHIFT); 8267372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 8277372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans /* 8287372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * Update stats_cactive if nactive is crossing a chunk 8297372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * multiple. 8307372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans */ 8317372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t cactive_diff = CHUNK_CEILING(arena->nactive << 8327372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans PAGE_SHIFT) - CHUNK_CEILING((arena->nactive - run_pages) << 8337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans PAGE_SHIFT); 8347372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (cactive_diff != 0) 8357372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans stats_cactive_sub(cactive_diff); 8367372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 837e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive -= run_pages; 838e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 83919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 84019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * The run is dirty if the caller claims to have dirtied it, as well as 84119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * if it was already dirty before being allocated. 84219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 8437393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) != 0) 84419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans dirty = true; 84519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; 84619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans runs_avail = dirty ? &arena->runs_avail_dirty : 84719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans &arena->runs_avail_clean; 84819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 849e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Mark pages as unallocated in the chunk map. */ 850e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (dirty) { 851940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[run_ind-map_bias].bits = size | CHUNK_MAP_DIRTY; 8527393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+run_pages-1-map_bias].bits = size | 853940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_DIRTY; 854e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 85519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans chunk->ndirty += run_pages; 85619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena->ndirty += run_pages; 857e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 8587393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind-map_bias].bits = size | 8593377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED); 8607393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+run_pages-1-map_bias].bits = size | 8617393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & 8623377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans CHUNK_MAP_UNZEROED); 863e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 864e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 865e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce forward. */ 866e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run_ind + run_pages < chunk_npages && 8677393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+run_pages-map_bias].bits & CHUNK_MAP_ALLOCATED) 8687393f44ff025ca67716fc53b68003fd65122fd97Jason Evans == 0 && (chunk->map[run_ind+run_pages-map_bias].bits & 8697393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_DIRTY) == flag_dirty) { 8707393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t nrun_size = chunk->map[run_ind+run_pages-map_bias].bits & 871e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ~PAGE_MASK; 87212ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans size_t nrun_pages = nrun_size >> PAGE_SHIFT; 873e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 874e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 875e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove successor from runs_avail; the coalesced run is 876e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 877e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 87812ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits 87912ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans & ~PAGE_MASK) == nrun_size); 88012ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits 88112ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans & CHUNK_MAP_ALLOCATED) == 0); 88212ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits 88312ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans & CHUNK_MAP_DIRTY) == flag_dirty); 88419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_remove(runs_avail, 8857393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[run_ind+run_pages-map_bias]); 886e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 887e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += nrun_size; 88812ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_pages += nrun_pages; 889e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 8907393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind-map_bias].bits = size | 8917393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); 8927393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+run_pages-1-map_bias].bits = size | 8937393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & 894e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 895e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 896e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 897e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to coalesce backward. */ 8987393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if (run_ind > map_bias && (chunk->map[run_ind-1-map_bias].bits & 8997393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[run_ind-1-map_bias].bits & 90019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans CHUNK_MAP_DIRTY) == flag_dirty) { 9017393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t prun_size = chunk->map[run_ind-1-map_bias].bits & 9027393f44ff025ca67716fc53b68003fd65122fd97Jason Evans ~PAGE_MASK; 90312ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans size_t prun_pages = prun_size >> PAGE_SHIFT; 904e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 90512ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_ind -= prun_pages; 906e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 907e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 908e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Remove predecessor from runs_avail; the coalesced run is 909e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * inserted later. 910e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 91112ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) 91212ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans == prun_size); 91312ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_ALLOCATED) 91412ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans == 0); 91512ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) 91612ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans == flag_dirty); 9177393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_remove(runs_avail, 9187393f44ff025ca67716fc53b68003fd65122fd97Jason Evans &chunk->map[run_ind-map_bias]); 919e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 920e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size += prun_size; 92112ca91402bc5d5c5a1cca495957463bb8e71fdcfJason Evans run_pages += prun_pages; 922e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 9237393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind-map_bias].bits = size | 9247393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); 9257393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+run_pages-1-map_bias].bits = size | 9267393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & 927e476f8a161d445211fd6e54fe370275196e66bcbJason Evans CHUNK_MAP_FLAGS_MASK); 928e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 929e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 930e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Insert into runs_avail, now that coalescing is complete. */ 9318de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) == 9328de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & ~PAGE_MASK)); 9338de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == 9348de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans (chunk->map[run_ind+run_pages-1-map_bias].bits & CHUNK_MAP_DIRTY)); 9357393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_avail_tree_insert(runs_avail, &chunk->map[run_ind-map_bias]); 936e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 9378d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (dirty) { 9388d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans /* 9398d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * Insert into chunks_dirty before potentially calling 9408d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * arena_chunk_dealloc(), so that chunks_dirty and 9418d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * arena->ndirty are consistent. 9428d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans */ 9438d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (chunk->dirtied == false) { 9448d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty); 9458d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans chunk->dirtied = true; 9468d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans } 9478d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans } 9488d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans 9494fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 9504fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * Deallocate chunk if it is now completely unused. The bit 9514fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * manipulation checks whether the first run is unallocated and extends 9524fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * to the end of the chunk. 9534fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 9547393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[0].bits & (~PAGE_MASK | CHUNK_MAP_ALLOCATED)) == 9557393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_maxclass) 956e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_dealloc(arena, chunk); 957e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 9584fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans /* 9598d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans * It is okay to do dirty page processing here even if the chunk was 9604fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * deallocated above, since in that case it is the spare. Waiting 9614fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * until after possible chunk deallocation to do dirty processing 9624fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * allows for an old spare to be fully deallocated, thus decreasing the 9634fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans * chances of spuriously crossing the dirty page purging threshold. 9644fb7f513376c0bb73fa1e4e1e89966af9cb2b9ecJason Evans */ 9658d4203c72de878c3976adc5db1e09d7ec0618d63Jason Evans if (dirty) 96605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans arena_maybe_purge(arena); 967e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 968e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 969e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 970e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 971e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize) 972e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 973e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; 974e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t head_npages = (oldsize - newsize) >> PAGE_SHIFT; 975940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t flag_dirty = chunk->map[pageind-map_bias].bits & CHUNK_MAP_DIRTY; 976e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 977e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 978e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 979e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 980e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 981940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * leading run as separately allocated. Set the last element of each 982940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * run first, in case of single-page runs. 983e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 984940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); 985940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); 986940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | 987940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+head_npages-1-map_bias].bits & 988940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 989940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind-map_bias].bits = (oldsize - newsize) 990940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans | flag_dirty | (chunk->map[pageind-map_bias].bits & 991940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 992940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 9937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_debug) { 9947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans UNUSED size_t tail_npages = newsize >> PAGE_SHIFT; 995940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] 996940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans .bits & ~PAGE_MASK) == 0); 997940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] 998940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans .bits & CHUNK_MAP_DIRTY) == flag_dirty); 999940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] 1000940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans .bits & CHUNK_MAP_LARGE) != 0); 1001940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] 1002940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans .bits & CHUNK_MAP_ALLOCATED) != 0); 1003940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 1004940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages-map_bias].bits = newsize | flag_dirty | 1005940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+head_npages-map_bias].bits & 1006940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_FLAGS_MASK) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1007e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1008e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, run, false); 1009e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1010e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1011e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1012e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1013e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t oldsize, size_t newsize, bool dirty) 1014e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1015e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; 1016940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t head_npages = newsize >> PAGE_SHIFT; 1017940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t tail_npages = (oldsize - newsize) >> PAGE_SHIFT; 1018940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t flag_dirty = chunk->map[pageind-map_bias].bits & 1019940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_DIRTY; 1020e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1021e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(oldsize > newsize); 1022e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1023e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1024e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Update the chunk map so that arena_run_dalloc() can treat the 1025940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * trailing run as separately allocated. Set the last element of each 1026940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * run first, in case of single-page runs. 1027e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1028940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); 1029940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); 1030940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | 1031940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+head_npages-1-map_bias].bits & 1032940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1033940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind-map_bias].bits = newsize | flag_dirty | 1034940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind-map_bias].bits & CHUNK_MAP_UNZEROED) | 1035940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1036940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1037940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & 1038940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans ~PAGE_MASK) == 0); 1039940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & 1040940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_LARGE) != 0); 1041940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & 1042940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_ALLOCATED) != 0); 1043940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits = 1044940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans flag_dirty | 1045940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & 1046940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1047940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+head_npages-map_bias].bits = (oldsize - newsize) | 1048940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans flag_dirty | (chunk->map[pageind+head_npages-map_bias].bits & 1049940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1050e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1051e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), 1052e476f8a161d445211fd6e54fe370275196e66bcbJason Evans dirty); 1053e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1054e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1055e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic arena_run_t * 1056e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_first(arena_bin_t *bin) 1057e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1058e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); 1059e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (mapelm != NULL) { 1060e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1061e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 1062e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1063e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); 10647393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / 10657393f44ff025ca67716fc53b68003fd65122fd97Jason Evans sizeof(arena_chunk_map_t))) + map_bias; 1066e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 1067e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans (uintptr_t)((pageind - (mapelm->bits >> PAGE_SHIFT)) << 1068e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans PAGE_SHIFT)); 1069e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1070e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans } 1071e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1072e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (NULL); 1073e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1074e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1075e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void 1076e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) 1077e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1078e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); 1079e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; 1080e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; 1081e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1082e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); 1083e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1084e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_tree_insert(&bin->runs, mapelm); 1085e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1086e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1087e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic void 1088e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) 1089e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1090e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 1091e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; 1092e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; 1093e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1094e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); 1095e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1096e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_tree_remove(&bin->runs, mapelm); 1097e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1098e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1099e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t * 1100e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_nonfull_run_tryget(arena_bin_t *bin) 1101e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1102e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_t *run = arena_bin_runs_first(bin); 1103e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) { 1104e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_remove(bin, run); 11057372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 11067372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.reruns++; 1107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1108e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1109e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans} 1110e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1111e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansstatic arena_run_t * 1112e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evansarena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) 1113e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans{ 1114e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_run_t *run; 1115e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans size_t binind; 1116e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_info_t *bin_info; 1117e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans 1118e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans /* Look for a usable run. */ 1119e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans run = arena_bin_nonfull_run_tryget(bin); 1120e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) 1121e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans return (run); 1122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* No existing runs have any space available. */ 1123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 112449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(arena, bin); 112549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 112649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 1127e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Allocate a new run. */ 1128e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&bin->lock); 1129e69bee01de62b56d3e585042d341743239568043Jason Evans /******************************/ 113086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 113149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans run = arena_run_alloc(arena, bin_info->run_size, false, false); 1132e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run != NULL) { 113384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + 113484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->bitmap_offset); 113584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 1136e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* Initialize run internals. */ 1137e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run->bin = bin; 113884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans run->nextind = 0; 113949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans run->nfree = bin_info->nregs; 114084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_init(bitmap, &bin_info->bitmap_info); 1141e69bee01de62b56d3e585042d341743239568043Jason Evans } 1142e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_unlock(&arena->lock); 1143e69bee01de62b56d3e585042d341743239568043Jason Evans /********************************/ 1144e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 1145e69bee01de62b56d3e585042d341743239568043Jason Evans if (run != NULL) { 11467372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 11477372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nruns++; 11487372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.curruns++; 11497372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1150e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 1151e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1152e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1153e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 1154e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * arena_run_alloc() failed, but another thread may have made 1155940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * sufficient memory available while this one dropped bin->lock above, 1156e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * so search one more time. 1157e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 1158e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans run = arena_bin_nonfull_run_tryget(bin); 1159e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (run != NULL) 1160e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (run); 1161e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1162e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (NULL); 1163e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1164e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 11651e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ 1166e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void * 1167e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) 1168e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1169e00572b384c81bd2aba57fac32f7077a34388915Jason Evans void *ret; 117049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 117149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1172e00572b384c81bd2aba57fac32f7077a34388915Jason Evans arena_run_t *run; 1173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 117449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(arena, bin); 117549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 1176e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->runcur = NULL; 1177e00572b384c81bd2aba57fac32f7077a34388915Jason Evans run = arena_bin_nonfull_run_get(arena, bin); 1178e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (bin->runcur != NULL && bin->runcur->nfree > 0) { 1179e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /* 1180e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * Another thread updated runcur while this one ran without the 1181e00572b384c81bd2aba57fac32f7077a34388915Jason Evans * bin lock in arena_bin_nonfull_run_get(). 1182e00572b384c81bd2aba57fac32f7077a34388915Jason Evans */ 1183e00572b384c81bd2aba57fac32f7077a34388915Jason Evans assert(bin->runcur->nfree > 0); 118449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_run_reg_alloc(bin->runcur, bin_info); 1185e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run != NULL) { 1186940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_chunk_t *chunk; 1187940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1188940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* 1189940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * arena_run_alloc() may have allocated run, or it may 119084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * have pulled run from the bin's run tree. Therefore 1191940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * it is unsafe to make any assumptions about how run 1192940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * has previously been used, and arena_bin_lower_run() 1193940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * must be called, as if a region were just deallocated 1194940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * from the run. 1195940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans */ 1196940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); 119749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (run->nfree == bin_info->nregs) 11988de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 11998de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans else 12008de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_bin_lower_run(arena, chunk, run, bin); 1201e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1202e00572b384c81bd2aba57fac32f7077a34388915Jason Evans return (ret); 1203e00572b384c81bd2aba57fac32f7077a34388915Jason Evans } 1204e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1205e00572b384c81bd2aba57fac32f7077a34388915Jason Evans if (run == NULL) 1206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1207e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1208e00572b384c81bd2aba57fac32f7077a34388915Jason Evans bin->runcur = run; 1209e00572b384c81bd2aba57fac32f7077a34388915Jason Evans 1210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(bin->runcur->nfree > 0); 1211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 121249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (arena_run_reg_alloc(bin->runcur, bin_info)); 1213e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 121586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evansvoid 121686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evansarena_prof_accum(arena_t *arena, uint64_t accumbytes) 121786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans{ 121886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 121986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (prof_interval != 0) { 122086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena->prof_accumbytes += accumbytes; 122186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (arena->prof_accumbytes >= prof_interval) { 122286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans prof_idump(); 122386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena->prof_accumbytes -= prof_interval; 122486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 122586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 122686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans} 122786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 1228e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 12297372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evansarena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, 12307372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans uint64_t prof_accumbytes) 1231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i, nfill; 1233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ptr; 1236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(tbin->ncached == 0); 1238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 12397372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) { 12407372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans malloc_mutex_lock(&arena->lock); 12417372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena_prof_accum(arena, prof_accumbytes); 12427372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans malloc_mutex_unlock(&arena->lock); 12437372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1244e69bee01de62b56d3e585042d341743239568043Jason Evans bin = &arena->bins[binind]; 1245e69bee01de62b56d3e585042d341743239568043Jason Evans malloc_mutex_lock(&bin->lock); 12461dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> 12471dcb4f86b23a5760f5a717ace716360b63b33fadJason Evans tbin->lg_fill_div); i < nfill; i++) { 1248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 124949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1251e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ptr = arena_bin_malloc_hard(arena, bin); 12523fa9a2fad83a3014d5069b5a2530a0cfb8d8d197Jason Evans if (ptr == NULL) 1253e476f8a161d445211fd6e54fe370275196e66bcbJason Evans break; 12549c43c13a35220c10d97a886616899189daceb359Jason Evans /* Insert such that low regions get used first. */ 12559c43c13a35220c10d97a886616899189daceb359Jason Evans tbin->avail[nfill - 1 - i] = ptr; 1256e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 12577372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 12587372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated += i * arena_bin_info[binind].reg_size; 12597372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nmalloc += i; 12607372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nrequests += tbin->tstats.nrequests; 12617372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nfills++; 12627372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans tbin->tstats.nrequests = 0; 12637372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 126486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1265e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tbin->ncached = i; 1266e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1267e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1268e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1269e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_small(arena_t *arena, size_t size, bool zero) 1270e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1272e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run; 1274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t binind; 1275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 127641ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans binind = SMALL_SIZE2BIN(size); 1277b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 1278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[binind]; 127949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size = arena_bin_info[binind].reg_size; 1280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 128186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 1282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((run = bin->runcur) != NULL && run->nfree > 0) 128349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); 1284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else 1285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = arena_bin_malloc_hard(arena, bin); 1286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1287e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 128886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 1289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 12927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 12937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated += size; 12947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nmalloc++; 12957372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.nrequests++; 12967372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 129786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 12987372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof && isthreaded == false) { 129986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1300d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans arena_prof_accum(arena, size); 130186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 130286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 1303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 13057372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill) { 13067372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (opt_junk) 13077372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0xa5, size); 13087372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans else if (opt_zero) 13097372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0, size); 13107372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else 1312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ret, 0, size); 1313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1315e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1316e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1317e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 1318e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_malloc_large(arena_t *arena, size_t size, bool zero) 1319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Large allocation. */ 1323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size = PAGE_CEILING(size); 1324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)arena_run_alloc(arena, size, true, zero); 1326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1328e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1329e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 13307372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 13317372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 13327372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 13337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 13347372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; 13357372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 13367372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 13387372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) 13397372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena_prof_accum(arena, size); 1340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (zero == false) { 13437372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill) { 13447372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (opt_junk) 13457372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0xa5, size); 13467372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans else if (opt_zero) 13477372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(ret, 0, size); 13487372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1350e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Only handles large allocations that require more than page alignment. */ 1355e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 13568e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_palloc(arena_t *arena, size_t size, size_t alloc_size, size_t alignment, 13578e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 1358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans void *ret; 1360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t offset; 1361e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((size & PAGE_MASK) == 0); 136493443689a4367cc6fe3de1c9e918adc13d8f9100Jason Evans 136593443689a4367cc6fe3de1c9e918adc13d8f9100Jason Evans alignment = PAGE_CEILING(alignment); 1366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 13688e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = (void *)arena_run_alloc(arena, alloc_size, true, zero); 1369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (ret == NULL) { 1370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (NULL); 1372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); 1375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans offset = (uintptr_t)ret & (alignment - 1); 1377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((offset & PAGE_MASK) == 0); 1378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(offset < alloc_size); 1379e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (offset == 0) 1380e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, ret, alloc_size, size, false); 1381e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 1382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t leadsize, trailsize; 1383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1384e476f8a161d445211fd6e54fe370275196e66bcbJason Evans leadsize = alignment - offset; 1385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (leadsize > 0) { 1386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_head(arena, chunk, ret, alloc_size, 1387e476f8a161d445211fd6e54fe370275196e66bcbJason Evans alloc_size - leadsize); 1388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = (void *)((uintptr_t)ret + leadsize); 1389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1390e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans trailsize = alloc_size - leadsize - size; 1392e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (trailsize != 0) { 1393e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Trim trailing space. */ 1394e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(trailsize < alloc_size); 1395e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, ret, size + trailsize, 1396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size, false); 1397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 14007372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 14017372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 14027372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 14037372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 14047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; 14057372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 14067372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1407e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1408e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1409e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 14107372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && zero == false) { 14118e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (opt_junk) 14128e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset(ret, 0xa5, size); 14138e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans else if (opt_zero) 14148e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset(ret, 0, size); 14158e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 1416e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1417e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1418e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1419e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Return the size of the allocation pointed to by ptr. */ 1420e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t 1421e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_salloc(const void *ptr) 1422e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1423e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ret; 1424e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1425e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind, mapbits; 1426e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1427e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1429e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1430e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 14317393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 14327393f44ff025ca67716fc53b68003fd65122fd97Jason Evans mapbits = chunk->map[pageind-map_bias].bits; 1433e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 1434e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 1435e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 143619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << 143719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)); 143849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, run->bin); 143949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 1440f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 144149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == 1442f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans 0); 144349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = bin_info->reg_size; 1444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1445f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 1446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ret = mapbits & ~PAGE_MASK; 1447e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ret != 0); 1448e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1451e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1452e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 14530b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansvoid 14540b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansarena_prof_promoted(const void *ptr, size_t size) 14550b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans{ 14560b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans arena_chunk_t *chunk; 14570b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t pageind, binind; 14580b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 14590b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(ptr != NULL); 14600b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 14610b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(isalloc(ptr) == PAGE_SIZE); 1462b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(size <= SMALL_MAXCLASS); 14630b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 14640b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 14657393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 146641ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans binind = SMALL_SIZE2BIN(size); 1467b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 14687393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits & 1469e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans ~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT); 14700b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans} 14710b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 14720b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evanssize_t 14730b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansarena_salloc_demote(const void *ptr) 14740b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans{ 14750b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t ret; 14760b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans arena_chunk_t *chunk; 14770b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t pageind, mapbits; 14780b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 14790b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(ptr != NULL); 14800b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 14810b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 14820b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 14837393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 14847393f44ff025ca67716fc53b68003fd65122fd97Jason Evans mapbits = chunk->map[pageind-map_bias].bits; 14850b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 14860b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 14870b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 14880b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << 14890b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans PAGE_SHIFT)); 149049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, run->bin); 149149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 14920b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 149349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == 14940b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 0); 149549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = bin_info->reg_size; 14960b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans } else { 14970b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 14980b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans ret = mapbits & ~PAGE_MASK; 14990b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans if (prof_promote && ret == PAGE_SIZE && (mapbits & 1500e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans CHUNK_MAP_CLASS_MASK) != 0) { 15010b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >> 1502e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans CHUNK_MAP_CLASS_SHIFT) - 1; 1503b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 150449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans ret = arena_bin_info[binind].reg_size; 15050b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans } 15060b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans assert(ret != 0); 15070b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans } 15080b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans 15090b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans return (ret); 15100b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans} 15116109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1512e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1513088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansarena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, 1514e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin) 1515e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1516e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 151719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* Dissociate run from bin. */ 1518e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (run == bin->runcur) 1519e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 152049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans else { 152149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, bin); 152249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 152349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 152449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (bin_info->nregs != 1) { 152549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 152649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * This block's conditional is necessary because if the 152749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * run only contains one region, then it never gets 152849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * inserted into the non-full runs tree. 152949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 1530e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_remove(bin, run); 153149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 1532e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1533088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans} 1534088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans 1535088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansstatic void 1536088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evansarena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1537088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_bin_t *bin) 1538088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans{ 153949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 154049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 1541088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans size_t npages, run_ind, past; 1542088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans 1543088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans assert(run != bin->runcur); 1544088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans assert(arena_run_tree_search(&bin->runs, &chunk->map[ 1545088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans (((uintptr_t)run-(uintptr_t)chunk)>>PAGE_SHIFT)-map_bias]) == NULL); 154686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 154749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(chunk->arena, run->bin); 154849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 154949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 1550e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_unlock(&bin->lock); 1551e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /******************************/ 155249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans npages = bin_info->run_size >> PAGE_SHIFT; 155319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT); 155484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans past = (size_t)(PAGE_CEILING((uintptr_t)run + 155584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * 155684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bin_info->reg_size) - (uintptr_t)chunk) >> PAGE_SHIFT); 155786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 155819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans 155919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 156019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * If the run was originally clean, and some pages were never touched, 156119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * trim the clean pages before deallocating the dirty portion of the 156219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * run. 156319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 15647393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == 0 && past 15657393f44ff025ca67716fc53b68003fd65122fd97Jason Evans - run_ind < npages) { 156619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans /* 156719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Trim clean pages. Convert to large run beforehand. Set the 156819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * last map element first, in case this is a one-page run. 156919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans */ 15707393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[run_ind+npages-1-map_bias].bits = CHUNK_MAP_LARGE | 1571940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[run_ind+npages-1-map_bias].bits & 1572940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_FLAGS_MASK); 157349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans chunk->map[run_ind-map_bias].bits = bin_info->run_size | 15747393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_LARGE | (chunk->map[run_ind-map_bias].bits & 15757393f44ff025ca67716fc53b68003fd65122fd97Jason Evans CHUNK_MAP_FLAGS_MASK); 157619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_run_trim_tail(arena, chunk, run, (npages << PAGE_SHIFT), 157721fb95bba6ea922e0523f269c0d9a32640047a29Jason Evans ((past - run_ind) << PAGE_SHIFT), false); 1578940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* npages = past - run_ind; */ 15791e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans } 158019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_run_dalloc(arena, run, true); 158186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 1582e00572b384c81bd2aba57fac32f7077a34388915Jason Evans /****************************/ 1583e00572b384c81bd2aba57fac32f7077a34388915Jason Evans malloc_mutex_lock(&bin->lock); 15847372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 15857372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.curruns--; 1586e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1587e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1588940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansstatic void 1589940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansarena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, 1590940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_bin_t *bin) 1591e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1592e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 15938de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* 1594e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * Make sure that if bin->runcur is non-NULL, it refers to the lowest 1595e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * non-full run. It is okay to NULL runcur out rather than proactively 1596e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans * keeping it pointing at the lowest non-full run. 15978de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans */ 1598e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if ((uintptr_t)run < (uintptr_t)bin->runcur) { 15998de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans /* Switch runcur. */ 1600e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (bin->runcur->nfree > 0) 1601e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_insert(bin, bin->runcur); 16028de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans bin->runcur = run; 1603e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans if (config_stats) 1604e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans bin->stats.reruns++; 1605e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans } else 1606e7a1058aaa6b2cbdd19da297bf2250f86dcdac89Jason Evans arena_bin_runs_insert(bin, run); 1607940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans} 1608940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1609940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansvoid 1610940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evansarena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 1611940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_chunk_map_t *mapelm) 1612940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans{ 1613940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t pageind; 1614940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_t *run; 1615940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_bin_t *bin; 1616940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t size; 1617940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1618940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 1619940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 1620940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (mapelm->bits >> PAGE_SHIFT)) << PAGE_SHIFT)); 1621940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans bin = run->bin; 162249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(arena, bin); 162349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 16247372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill || config_stats) 16257372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size = bin_info->reg_size; 1626940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 16277372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk) 1628940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans memset(ptr, 0x5a, size); 1629940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans 1630940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans arena_run_reg_dalloc(run, ptr); 163149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans if (run->nfree == bin_info->nregs) { 1632088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans arena_dissociate_bin_run(chunk, run, bin); 16338de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_dalloc_bin_run(arena, chunk, run, bin); 1634088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans } else if (run->nfree == 1 && run != bin->runcur) 16358de6a0282333ca742fb2870b1e859968d3c1b11fJason Evans arena_bin_lower_run(arena, chunk, run, bin); 1636e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 16377372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 16387372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.allocated -= size; 16397372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bin->stats.ndalloc++; 16407372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1641e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1642e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1643e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1644b34e8684ec025aa780997c11f847c19fb269755bJason Evansarena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, 1645b34e8684ec025aa780997c11f847c19fb269755bJason Evans arena_stats_t *astats, malloc_bin_stats_t *bstats, 1646b34e8684ec025aa780997c11f847c19fb269755bJason Evans malloc_large_stats_t *lstats) 1647b34e8684ec025aa780997c11f847c19fb269755bJason Evans{ 16483c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans unsigned i; 1649b34e8684ec025aa780997c11f847c19fb269755bJason Evans 165086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&arena->lock); 1651b34e8684ec025aa780997c11f847c19fb269755bJason Evans *nactive += arena->nactive; 1652b34e8684ec025aa780997c11f847c19fb269755bJason Evans *ndirty += arena->ndirty; 1653b34e8684ec025aa780997c11f847c19fb269755bJason Evans 16544201af05425b69ee37ffca437aca0cdd604d1e51Jason Evans astats->mapped += arena->stats.mapped; 1655b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->npurge += arena->stats.npurge; 1656b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmadvise += arena->stats.nmadvise; 1657b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->purged += arena->stats.purged; 1658b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->allocated_large += arena->stats.allocated_large; 1659b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->nmalloc_large += arena->stats.nmalloc_large; 1660b34e8684ec025aa780997c11f847c19fb269755bJason Evans astats->ndalloc_large += arena->stats.ndalloc_large; 1661dafde14e08ddfda747aabb2045b350848b601b2eJason Evans astats->nrequests_large += arena->stats.nrequests_large; 1662b34e8684ec025aa780997c11f847c19fb269755bJason Evans 16633c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans for (i = 0; i < nlclasses; i++) { 1664dafde14e08ddfda747aabb2045b350848b601b2eJason Evans lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; 1665dafde14e08ddfda747aabb2045b350848b601b2eJason Evans lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; 1666b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].nrequests += arena->stats.lstats[i].nrequests; 1667b34e8684ec025aa780997c11f847c19fb269755bJason Evans lstats[i].curruns += arena->stats.lstats[i].curruns; 1668b34e8684ec025aa780997c11f847c19fb269755bJason Evans } 166986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&arena->lock); 167086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 1671b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans for (i = 0; i < NBINS; i++) { 167286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin = &arena->bins[i]; 167386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 167486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 167586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].allocated += bin->stats.allocated; 167686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nmalloc += bin->stats.nmalloc; 167786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].ndalloc += bin->stats.ndalloc; 167886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nrequests += bin->stats.nrequests; 16797372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_tcache) { 16807372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bstats[i].nfills += bin->stats.nfills; 16817372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans bstats[i].nflushes += bin->stats.nflushes; 16827372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 168386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].nruns += bin->stats.nruns; 168486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].reruns += bin->stats.reruns; 168586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bstats[i].curruns += bin->stats.curruns; 168686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 168786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans } 1688b34e8684ec025aa780997c11f847c19fb269755bJason Evans} 1689e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1690e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid 1691e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) 1692e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 169313668262d17fb5950e2441bc9d56a15db9c93877Jason Evans 16947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill || config_stats) { 1695e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 1696e476f8a161d445211fd6e54fe370275196e66bcbJason Evans PAGE_SHIFT; 16977393f44ff025ca67716fc53b68003fd65122fd97Jason Evans size_t size = chunk->map[pageind-map_bias].bits & ~PAGE_MASK; 1698e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 16997372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && config_stats && opt_junk) 1700e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset(ptr, 0x5a, size); 17017372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 17027372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 17037372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= size; 17047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].ndalloc++; 17057372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns--; 17067372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1707e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1708e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1709e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_dalloc(arena, (arena_run_t *)ptr, true); 1710e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1711e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1712e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic void 1713e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, 17148e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t oldsize, size_t size) 1715e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1716e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1717e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size < oldsize); 1718e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1719e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1720e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Shrink the run, and make trailing pages available for other 1721e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocations. 1722e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1723e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 1724e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, 1725e476f8a161d445211fd6e54fe370275196e66bcbJason Evans true); 17267372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 17277372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 17287372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= oldsize; 17297372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++; 17307372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; 17317372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 17327372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 17337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 17347372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 17357372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; 17367372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; 17377372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1738e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1739e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1740e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1741e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1742e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 1743e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, 17448e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t oldsize, size_t size, size_t extra, bool zero) 1745e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1746e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 1747e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t npages = oldsize >> PAGE_SHIFT; 17488e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t followsize; 1749e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17507393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(oldsize == (chunk->map[pageind-map_bias].bits & ~PAGE_MASK)); 1751e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1752e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Try to extend the run. */ 17538e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans assert(size + extra > oldsize); 1754e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_lock(&arena->lock); 17557393f44ff025ca67716fc53b68003fd65122fd97Jason Evans if (pageind + npages < chunk_npages && 17567393f44ff025ca67716fc53b68003fd65122fd97Jason Evans (chunk->map[pageind+npages-map_bias].bits 17578e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans & CHUNK_MAP_ALLOCATED) == 0 && (followsize = 17587393f44ff025ca67716fc53b68003fd65122fd97Jason Evans chunk->map[pageind+npages-map_bias].bits & ~PAGE_MASK) >= size - 17597393f44ff025ca67716fc53b68003fd65122fd97Jason Evans oldsize) { 1760e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1761e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The next run is available and sufficiently large. Split the 1762e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * following run, then merge the first part with the existing 1763e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * allocation. 1764e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1765940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans size_t flag_dirty; 17668e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t splitsize = (oldsize + followsize <= size + extra) 17678e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ? followsize : size + extra - oldsize; 1768e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + 17698e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ((pageind+npages) << PAGE_SHIFT)), splitsize, true, zero); 1770e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1771088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans size = oldsize + splitsize; 1772088e6a0a37e197c8db16e3d4c518a5eb762e7657Jason Evans npages = size >> PAGE_SHIFT; 1773e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1774940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans /* 1775940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * Mark the extended run as dirty if either portion of the run 1776940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * was dirty before allocation. This is rather pedantic, 1777940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * because there's not actually any sequence of events that 1778940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * could cause the resulting run to be passed to 1779940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * arena_run_dalloc() with the dirty argument set to false 1780940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans * (which is when dirty flag consistency would really matter). 1781940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans */ 1782940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans flag_dirty = (chunk->map[pageind-map_bias].bits & 1783940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_DIRTY) | 1784940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans (chunk->map[pageind+npages-1-map_bias].bits & 1785940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_DIRTY); 1786940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind-map_bias].bits = size | flag_dirty 1787940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1788940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans chunk->map[pageind+npages-1-map_bias].bits = flag_dirty | 1789940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 1790990d10cefb4b2ff4458086aedba8fc70975a9adcJason Evans 17917372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 17927372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.ndalloc_large++; 17937372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large -= oldsize; 17947372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(oldsize >> PAGE_SHIFT) 17957372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans - 1].ndalloc++; 17967372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(oldsize >> PAGE_SHIFT) 17977372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans - 1].curruns--; 17987372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans 17997372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nmalloc_large++; 18007372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.nrequests_large++; 18017372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.allocated_large += size; 18027372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; 18037372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) 18047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans - 1].nrequests++; 18057372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; 1806940a2e02b27b264cc92e8ecbf186a711ce05ad04Jason Evans } 1807e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1808e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1809e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1810e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_unlock(&arena->lock); 1811e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1812e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 1813e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1814e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1815e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 1816e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Try to resize a large allocation, in order to avoid copying. This will 1817e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * always fail if growing an object, and the following run is already in use. 1818e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 1819e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstatic bool 18208e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, 18218e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 1822e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1823e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t psize; 1824e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18258e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize = PAGE_CEILING(size + extra); 1826e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize == oldsize) { 1827e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Same size class. */ 18287372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk && size < oldsize) { 1829e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - 1830e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size); 1831e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1832e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1833e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1834e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *chunk; 1835e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 1836e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1837e476f8a161d445211fd6e54fe370275196e66bcbJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1838e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena = chunk->arena; 1839e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1840e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (psize < oldsize) { 1841e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Fill before shrinking in order avoid a race. */ 18427372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk) { 1843e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + size), 0x5a, 1844e476f8a161d445211fd6e54fe370275196e66bcbJason Evans oldsize - size); 1845e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 18468e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, 18478e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize); 1848e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 1849e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1850e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool ret = arena_ralloc_large_grow(arena, chunk, ptr, 18518e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans oldsize, PAGE_CEILING(size), 18528e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans psize - PAGE_CEILING(size), zero); 18537372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && ret == false && zero == false && 18547372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans opt_zero) { 1855e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memset((void *)((uintptr_t)ptr + oldsize), 0, 1856e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size - oldsize); 1857e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1858e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1859e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1860e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1861e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1862e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1863e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid * 18648e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, 18658e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans bool zero) 1866e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1867e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18688e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* 18698e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * Avoid moving the allocation if the size class can be left the same. 18708e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans */ 1871e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (oldsize <= arena_maxclass) { 1872b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (oldsize <= SMALL_MAXCLASS) { 187349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size 187449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans == oldsize); 1875b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if ((size + extra <= SMALL_MAXCLASS && 187641ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans SMALL_SIZE2BIN(size + extra) == 187741ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && 18788e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size + extra >= oldsize)) { 18797372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_fill && opt_junk && size < oldsize) { 18808e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans memset((void *)((uintptr_t)ptr + size), 18818e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 0x5a, oldsize - size); 18828e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 18838e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (ptr); 18848e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 1885e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } else { 1886e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(size <= arena_maxclass); 1887b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (size + extra > SMALL_MAXCLASS) { 18888e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (arena_ralloc_large(ptr, oldsize, size, 18898e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans extra, zero) == false) 1890e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ptr); 1891e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1892e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1893e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 1894e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18958e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Reallocation would require a move. */ 18968e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 18978e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans} 18988e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 18998e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansvoid * 19008e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansarena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, 19018e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t alignment, bool zero) 19028e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans{ 19038e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans void *ret; 19048e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t copysize; 19058e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 19068e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Try to avoid moving the allocation. */ 19078e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = arena_ralloc_no_move(ptr, oldsize, size, extra, zero); 19088e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret != NULL) 19098e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (ret); 19108e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 1911e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 19128e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * size and oldsize are different enough that we need to move the 19138e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * object. In that case, fall back to allocating new space and 19148e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * copying. 1915e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 191638d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (alignment != 0) { 191738d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans size_t usize = sa2u(size + extra, alignment, NULL); 191838d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (usize == 0) 191938d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans return (NULL); 192038d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans ret = ipalloc(usize, alignment, zero); 192138d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans } else 19228e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = arena_malloc(size + extra, zero); 1923e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19248e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == NULL) { 19258e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (extra == 0) 19268e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 19278e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Try again, this time without extra. */ 192838d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (alignment != 0) { 192938d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans size_t usize = sa2u(size, alignment, NULL); 193038d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans if (usize == 0) 193138d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans return (NULL); 193238d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans ret = ipalloc(usize, alignment, zero); 193338d9210c464c4ad49655a4da6bc84ea4fbec83d2Jason Evans } else 19348e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans ret = arena_malloc(size, zero); 19358e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 19368e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans if (ret == NULL) 19378e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans return (NULL); 19388e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans } 19398e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 19408e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ 19418e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans 19428e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans /* 19438e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * Copy at most size bytes (not size+extra), since the caller has no 19448e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans * expectation that the extra bytes will be reliably preserved. 19458e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans */ 1946e476f8a161d445211fd6e54fe370275196e66bcbJason Evans copysize = (size < oldsize) ? size : oldsize; 1947e476f8a161d445211fd6e54fe370275196e66bcbJason Evans memcpy(ret, ptr, copysize); 1948e476f8a161d445211fd6e54fe370275196e66bcbJason Evans idalloc(ptr); 1949e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (ret); 1950e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1951e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1952e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool 1953e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_new(arena_t *arena, unsigned ind) 1954e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1955e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned i; 1956e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 1957e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19586109fe07a14b7a619365977d9523db9f8b333792Jason Evans arena->ind = ind; 1959597632be188d2bcc135dad2145cc46ef44897aadJason Evans arena->nthreads = 0; 19606109fe07a14b7a619365977d9523db9f8b333792Jason Evans 1961e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if (malloc_mutex_init(&arena->lock)) 1962e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (true); 1963e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19647372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) { 19657372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(&arena->stats, 0, sizeof(arena_stats_t)); 19667372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->stats.lstats = 19677372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans (malloc_large_stats_t *)base_alloc(nlclasses * 19687372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans sizeof(malloc_large_stats_t)); 19697372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (arena->stats.lstats == NULL) 19707372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans return (true); 19717372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(arena->stats.lstats, 0, nlclasses * 19727372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans sizeof(malloc_large_stats_t)); 19737372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_tcache) 19747372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans ql_new(&arena->tcache_ql); 19757372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans } 1976e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19777372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof) 19787372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans arena->prof_accumbytes = 0; 1979d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 1980e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize chunks. */ 19812caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_new(&arena->chunks_dirty); 1982e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->spare = NULL; 1983e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1984e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->nactive = 0; 1985e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena->ndirty = 0; 1986799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans arena->npurgatory = 0; 1987e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 198819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_new(&arena->runs_avail_clean); 198919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_new(&arena->runs_avail_dirty); 1990e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1991e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Initialize bins. */ 1992b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans for (i = 0; i < NBINS; i++) { 1993e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin = &arena->bins[i]; 199486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans if (malloc_mutex_init(&bin->lock)) 199586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans return (true); 1996e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bin->runcur = NULL; 1997e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_new(&bin->runs); 19987372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_stats) 19997372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); 2000e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 2001e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2002e476f8a161d445211fd6e54fe370275196e66bcbJason Evans return (false); 2003e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2004e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 200549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 200649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Calculate bin_info->run_size such that it meets the following constraints: 200749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 200849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) bin_info->run_size >= min_run_size 200949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) bin_info->run_size <= arena_maxclass 201049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). 201147e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans * *) bin_info->nregs <= RUN_MAXREGS 201249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 201384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also 201484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * calculated here, since these settings are all interdependent. 201549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 201649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstatic size_t 201749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansbin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) 201849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 201949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t try_run_size, good_run_size; 202049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_nregs, good_nregs; 202149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_hdr_size, good_hdr_size; 202284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans uint32_t try_bitmap_offset, good_bitmap_offset; 202349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_ctx0_offset, good_ctx0_offset; 202449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t try_reg0_offset, good_reg0_offset; 202549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 202649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(min_run_size >= PAGE_SIZE); 202749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(min_run_size <= arena_maxclass); 202849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 202949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 203049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Calculate known-valid settings before entering the run_size 203149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * expansion loop, so that the first part of the loop always copies 203249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * valid settings. 203349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * 203449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * The do..while loop iteratively reduces the number of regions until 203549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * the run header and the regions no longer overlap. A closed formula 203649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * would be quite messy, since there is an interdependency between the 203749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * header's mask length and the number of regions. 203849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 203949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_run_size = min_run_size; 204049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_size) 204149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 204247e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans if (try_nregs > RUN_MAXREGS) { 204347e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans try_nregs = RUN_MAXREGS 204447e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 204547e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans } 204649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 204749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs--; 204849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = sizeof(arena_run_t); 204984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Pad to a long boundary. */ 205084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size = LONG_CEILING(try_hdr_size); 205184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_bitmap_offset = try_hdr_size; 205284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Add space for bitmap. */ 205384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size += bitmap_size(try_nregs); 20547372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof && opt_prof && prof_promote == false) { 205549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Pad to a quantum boundary. */ 205649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 205749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = try_hdr_size; 205849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Add space for one (prof_ctx_t *) per region. */ 205949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size += try_nregs * sizeof(prof_ctx_t *); 206049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } else 206149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = 0; 206249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_reg0_offset = try_run_size - (try_nregs * 206349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size); 206449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } while (try_hdr_size > try_reg0_offset); 206549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 206649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* run_size expansion loop. */ 206749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 206849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 206949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Copy valid settings before trying more aggressive settings. 207049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 207149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_run_size = try_run_size; 207249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_nregs = try_nregs; 207349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_hdr_size = try_hdr_size; 207484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans good_bitmap_offset = try_bitmap_offset; 207549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_ctx0_offset = try_ctx0_offset; 207649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans good_reg0_offset = try_reg0_offset; 207749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 207849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Try more aggressive settings. */ 207949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_run_size += PAGE_SIZE; 208049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs = ((try_run_size - sizeof(arena_run_t)) / 208149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size) 208249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 208347e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans if (try_nregs > RUN_MAXREGS) { 208447e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans try_nregs = RUN_MAXREGS 208547e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans + 1; /* Counter-act try_nregs-- in loop. */ 208647e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans } 208749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans do { 208849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_nregs--; 208949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = sizeof(arena_run_t); 209084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Pad to a long boundary. */ 209184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size = LONG_CEILING(try_hdr_size); 209284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_bitmap_offset = try_hdr_size; 209384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Add space for bitmap. */ 209484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans try_hdr_size += bitmap_size(try_nregs); 20957372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans if (config_prof && opt_prof && prof_promote == false) { 209649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Pad to a quantum boundary. */ 209749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size = QUANTUM_CEILING(try_hdr_size); 209849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_ctx0_offset = try_hdr_size; 209949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 210049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Add space for one (prof_ctx_t *) per region. 210149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 210249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_hdr_size += try_nregs * 210349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans sizeof(prof_ctx_t *); 210449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 210549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans try_reg0_offset = try_run_size - (try_nregs * 210649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size); 210749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } while (try_hdr_size > try_reg0_offset); 210849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } while (try_run_size <= arena_maxclass 210949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans && try_run_size <= arena_maxclass 211049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans && RUN_MAX_OVRHD * (bin_info->reg_size << 3) > RUN_MAX_OVRHD_RELAX 211147e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size 211247e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans && try_nregs < RUN_MAXREGS); 211349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 211449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(good_hdr_size <= good_reg0_offset); 211549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 211649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Copy final settings. */ 211749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->run_size = good_run_size; 211849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->nregs = good_nregs; 211984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bin_info->bitmap_offset = good_bitmap_offset; 212049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->ctx0_offset = good_ctx0_offset; 212149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg0_offset = good_reg0_offset; 212249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 212349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (good_run_size); 212449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 212549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2126b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansstatic void 212749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansbin_info_init(void) 212849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 212949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 2130b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans size_t prev_run_size = PAGE_SIZE; 2131b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans 2132b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#define SIZE_CLASS(bin, delta, size) \ 2133b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info = &arena_bin_info[bin]; \ 2134b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info->reg_size = size; \ 2135b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ 2136b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); 2137b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans SIZE_CLASSES 2138b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans#undef SIZE_CLASS 213949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 214049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 2141b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansvoid 2142a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evansarena_boot(void) 2143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 2144a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans size_t header_size; 21457393f44ff025ca67716fc53b68003fd65122fd97Jason Evans unsigned i; 2146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2147e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 2148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Compute the header size such that it is large enough to contain the 21497393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * page map. The page map is biased to omit entries for the header 21507393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * itself, so some iteration is necessary to compute the map bias. 21517393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 21527393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 1) Compute safe header_size and map_bias values that include enough 21537393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * space for an unbiased page map. 21547393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 2) Refine map_bias based on (1) to omit the header pages in the page 21557393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * map. The resulting map_bias may be one too small. 21567393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * 3) Refine map_bias based on (2). The result will be >= the result 21577393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * from (2), and will always be correct. 2158e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 21597393f44ff025ca67716fc53b68003fd65122fd97Jason Evans map_bias = 0; 21607393f44ff025ca67716fc53b68003fd65122fd97Jason Evans for (i = 0; i < 3; i++) { 21617393f44ff025ca67716fc53b68003fd65122fd97Jason Evans header_size = offsetof(arena_chunk_t, map) 21627393f44ff025ca67716fc53b68003fd65122fd97Jason Evans + (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); 21637393f44ff025ca67716fc53b68003fd65122fd97Jason Evans map_bias = (header_size >> PAGE_SHIFT) + ((header_size & 21647393f44ff025ca67716fc53b68003fd65122fd97Jason Evans PAGE_MASK) != 0); 21657393f44ff025ca67716fc53b68003fd65122fd97Jason Evans } 21667393f44ff025ca67716fc53b68003fd65122fd97Jason Evans assert(map_bias > 0); 21677393f44ff025ca67716fc53b68003fd65122fd97Jason Evans 21687393f44ff025ca67716fc53b68003fd65122fd97Jason Evans arena_maxclass = chunksize - (map_bias << PAGE_SHIFT); 2169a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evans 2170b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans bin_info_init(); 2171e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 2172