arena.h revision a3b3386ddde8048b9d6b54c397bb93da5e806cef
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 2e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_TYPES 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * as small as possible such that this setting is still honored, without 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * violating other constraints. The goal is to make runs as small as possible 8e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * without exceeding a per run external fragmentation threshold. 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * We use binary fixed point math for overhead computations, where the binary 11e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * point is implicitly RUN_BFP bits to the left. 12e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be 148ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * honored for some/all object sizes, since when heap profiling is enabled 158ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * there is one pointer of header overhead per object (plus a constant). This 168ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * constraint is relaxed (ignored) for runs that are so small that the 178ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * per-region overhead is greater than: 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 19122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * (RUN_MAX_OVRHD / (reg_interval << (3+RUN_BFP)) 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 21e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define RUN_BFP 12 22e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* \/ Implicit binary fixed point. */ 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define RUN_MAX_OVRHD 0x0000003dU 24e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define RUN_MAX_OVRHD_RELAX 0x00001800U 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2647e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans/* Maximum number of regions in one run. */ 2747e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define LG_RUN_MAXREGS 11 2847e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) 2947e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 31122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Minimum redzone size. Redzones may be larger than this if necessary to 32122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * preserve region alignment. 33122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans */ 34122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans#define REDZONE_MINSIZE 16 35122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 36122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans/* 37e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The minimum ratio of active:dirty pages per arena is computed as: 38e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * (nactive >> opt_lg_dirty_mult) >= ndirty 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 41e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * So, supposing that opt_lg_dirty_mult is 3, there can be no less than 8 times 42e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * as many active pages as dirty pages. 43e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 44e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans#define LG_DIRTY_MULT_DEFAULT 3 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 46e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_chunk_map_s arena_chunk_map_t; 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_chunk_s arena_chunk_t; 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_run_s arena_run_t; 4949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evanstypedef struct arena_bin_info_s arena_bin_info_t; 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_bin_s arena_bin_t; 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_s arena_t; 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_TYPES */ 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_STRUCTS 56e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 57e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Each element of the chunk map corresponds to one page within the chunk. */ 58e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_chunk_map_s { 597372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans#ifndef JEMALLOC_PROF 607372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans /* 617372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * Overlay prof_ctx in order to allow it to be referenced by dead code. 627372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * Such antics aren't warranted for per arena data structures, but 637372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * chunk map overhead accounts for a percentage of memory, rather than 647372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * being just a fixed cost. 657372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans */ 667372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans union { 677372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans#endif 6805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans union { 6905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 7005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * Linkage for run trees. There are two disjoint uses: 7105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * 72e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * 1) arena_t's runs_avail tree. 7305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * 2) arena_run_t conceptually uses this linkage for in-use 7405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * non-full runs, rather than directly embedding linkage. 7505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 7605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans rb_node(arena_chunk_map_t) rb_link; 7705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 7805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * List of runs currently in purgatory. arena_chunk_purge() 7905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * temporarily allocates runs that contain dirty pages while 8005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * purging, so that other threads cannot use the runs while the 8105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * purging thread is operating without the arena lock held. 8205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 8305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_elm(arena_chunk_map_t) ql_link; 8405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } u; 85e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 866109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Profile counters, used for large object runs. */ 875065156f3f90e421ba2b1a914e47eeb30d83d994Jason Evans prof_ctx_t *prof_ctx; 887372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans#ifndef JEMALLOC_PROF 897372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans }; /* union { ... }; */ 906109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 916109fe07a14b7a619365977d9523db9f8b333792Jason Evans 92e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 93e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Run address (or size) and various flags are stored together. The bit 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * layout looks like (assuming 32-bit system): 95e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 9653bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ???????? ???????? ????nnnn nnnndula 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 98e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * ? : Unallocated: Run address for first/last pages, unset for internal 99e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * pages. 10019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Small: Run page offset. 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: Run size for first page, unset for trailing pages. 10253bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * n : binind for small size class, BININD_INVALID for large size class. 103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * d : dirty? 1048ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * u : unzeroed? 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * l : large? 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a : allocated? 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Following are example bit patterns for the three types of runs. 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * p : run page offset 111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * s : run size 112203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * n : binind for size class; large objects set these to BININD_INVALID 113203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * except for promoted allocations (see prof_promote) 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * x : don't care 115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * - : 0 1160b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * + : 1 1173377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * [DULA] : bit set 1183377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * [dula] : bit unset 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 12019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (clean): 12153bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++du-a 122203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxx-Uxx 12353bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++dU-a 12419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 12519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (dirty): 12653bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D--a 127203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 12853bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D--a 129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 130dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Small: 131203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * pppppppp pppppppp ppppnnnn nnnnd--A 132203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * pppppppp pppppppp ppppnnnn nnnn---A 133203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * pppppppp pppppppp ppppnnnn nnnnd--A 134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: 13653bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D-LA 137203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 13853bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * -------- -------- ----++++ ++++D-LA 1390b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 140ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans * Large (sampled, size <= PAGE): 141203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * ssssssss ssssssss ssssnnnn nnnnD-LA 1420b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 143ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans * Large (not sampled, size == PAGE): 14453bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D-LA 145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 146e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t bits; 147203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_SHIFT 4 148203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define BININD_INVALID ((size_t)0xffU) 149203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans/* CHUNK_MAP_BININD_MASK == (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) */ 150203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_MASK ((size_t)0xff0U) 151203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK 152203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_FLAGS_MASK ((size_t)0xcU) 1530b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_DIRTY ((size_t)0x8U) 1543377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans#define CHUNK_MAP_UNZEROED ((size_t)0x4U) 1550b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_LARGE ((size_t)0x2U) 1560b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_ALLOCATED ((size_t)0x1U) 1570b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED 158e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 159e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef rb_tree(arena_chunk_map_t) arena_avail_tree_t; 160e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef rb_tree(arena_chunk_map_t) arena_run_tree_t; 161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 162e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Arena chunk header. */ 163e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_chunk_s { 164e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Arena that owns the chunk. */ 165e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_t *arena; 166e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 167e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Linkage for tree of arena chunks that contain dirty runs. */ 168e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans rb_node(arena_chunk_t) dirty_link; 169e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 170e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Number of dirty pages. */ 171e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t ndirty; 172e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 173e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Number of available runs. */ 174e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t nruns_avail; 175e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans 176e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* 177e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * Number of available run adjacencies. Clean and dirty available runs 178e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * are not coalesced, which causes virtual memory fragmentation. The 179e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * ratio of (nruns_avail-nruns_adjac):nruns_adjac is used for tracking 180e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * this fragmentation. 181e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * */ 182e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans size_t nruns_adjac; 18313668262d17fb5950e2441bc9d56a15db9c93877Jason Evans 1847393f44ff025ca67716fc53b68003fd65122fd97Jason Evans /* 1857393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * Map of pages within chunk that keeps track of free/large/small. The 1867393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * first map_bias entries are omitted, since the chunk header does not 1877393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * need to be tracked in the map. This omission saves a header page 1887393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * for common chunk sizes (e.g. 4 MiB). 1897393f44ff025ca67716fc53b68003fd65122fd97Jason Evans */ 190e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_map_t map[1]; /* Dynamically sized. */ 191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 192e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef rb_tree(arena_chunk_t) arena_chunk_tree_t; 193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 194e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_run_s { 195e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Bin this run is associated with. */ 196e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Index of next region that has never been allocated, or nregs. */ 19984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans uint32_t nextind; 200e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Number of free regions in run. */ 202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned nfree; 203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 204e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 20549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 20684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Read-only information associated with each element of arena_t's bins array 20749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * is stored separately, partly to reduce memory usage (only one copy, rather 20849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * than one per arena), but mainly to avoid false cacheline sharing. 209122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 210122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Each run has the following layout: 211122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 212122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * /--------------------\ 213122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | arena_run_t header | 214122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 215122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * bitmap_offset | bitmap | 216122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 217122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * ctx0_offset | ctx map | 218122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 219122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 220122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 221122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * reg0_offset | region 0 | 222122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 223122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| \ 224122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | | 225122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | region 1 | > reg_interval 226122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | / 227122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 228122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 229122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 230122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 231122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 232122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 233122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | region nregs-1 | 234122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 235122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 236122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | alignment pad? | 237122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * \--------------------/ 238122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 239122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * reg_interval has at least the same minimum alignment as reg_size; this 240122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * preserves the alignment constraint that sa2u() depends on. Alignment pad is 241122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * either 0 or redzone_size; it is present only if needed to align reg0_offset. 24249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 24349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstruct arena_bin_info_s { 24449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Size of regions in a run for this bin's size class. */ 24549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t reg_size; 24649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 247122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Redzone size. */ 248122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t redzone_size; 249122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 250122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Interval between regions (reg_size + (redzone_size << 1)). */ 251122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t reg_interval; 252122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 25349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total size of a run for this bin's size class. */ 25449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t run_size; 25549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 25649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total number of regions in a run for this bin's size class. */ 25749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t nregs; 25849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 25984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 26084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Offset of first bitmap_t element in a run header for this bin's size 26184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * class. 26284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 26384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans uint32_t bitmap_offset; 26484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 26584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 26684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Metadata used to manipulate bitmaps for runs associated with this 26784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin. 26884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 26984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_info_t bitmap_info; 27084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 27149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 27249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Offset of first (prof_ctx_t *) in a run header for this bin's size 2737372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans * class, or 0 if (config_prof == false || opt_prof == false). 27449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 27549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t ctx0_offset; 27649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 27749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Offset of first region in a run for this bin's size class. */ 27849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t reg0_offset; 27949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans}; 28049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 281e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_bin_s { 282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 28386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * All operations on runcur, runs, and stats require that lock be 28486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. Run allocation/deallocation are protected by the arena lock, 28586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * which may be acquired while holding one or more bin locks, but not 28686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * vise versa. 28786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 28886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_t lock; 28986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 29086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current run being used to service allocations of this bin's size 292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * class. 293e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 294e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *runcur; 295e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Tree of non-full runs. This tree is used when looking for an 298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * existing run when runcur is no longer usable. We choose the 299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * non-full run that is lowest in memory; this policy tends to keep 300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * objects packed well, and it can also help reduce the number of 301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * almost-empty chunks. 302e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_t runs; 304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Bin statistics. */ 306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_bin_stats_t stats; 307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 309e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_s { 3106109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* This arena's index within the arenas array. */ 3116109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned ind; 3126109fe07a14b7a619365977d9523db9f8b333792Jason Evans 31386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 314597632be188d2bcc135dad2145cc46ef44897aadJason Evans * Number of threads currently assigned to this arena. This field is 315597632be188d2bcc135dad2145cc46ef44897aadJason Evans * protected by arenas_lock. 316597632be188d2bcc135dad2145cc46ef44897aadJason Evans */ 317597632be188d2bcc135dad2145cc46ef44897aadJason Evans unsigned nthreads; 318597632be188d2bcc135dad2145cc46ef44897aadJason Evans 319597632be188d2bcc135dad2145cc46ef44897aadJason Evans /* 320597632be188d2bcc135dad2145cc46ef44897aadJason Evans * There are three classes of arena operations from a locking 321597632be188d2bcc135dad2145cc46ef44897aadJason Evans * perspective: 322597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 1) Thread asssignment (modifies nthreads) is protected by 323597632be188d2bcc135dad2145cc46ef44897aadJason Evans * arenas_lock. 324597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 2) Bin-related operations are protected by bin locks. 325597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 3) Chunk- and run-related operations are protected by this mutex. 32686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_t lock; 328e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 329e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_stats_t stats; 330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * List of tcaches for extant threads associated with this arena. 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Stats from these are merged incrementally, and at exit. 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_head(tcache_t) tcache_ql; 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 336d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans uint64_t prof_accumbytes; 337d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 338609ae595f0358157b19311b0f9f9591db7cee705Jason Evans dss_prec_t dss_prec; 339609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 340e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans /* Tree of dirty-page-containing chunks this arena manages. */ 341e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_chunk_tree_t chunks_dirty; 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * In order to avoid rapid chunk allocation/deallocation when an arena 345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * oscillates right on the cusp of needing a new chunk, cache the most 346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * recently freed chunk. The spare is left in the arena's chunk trees 347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * until it is deleted. 348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * There is one spare chunk per arena, rather than one spare total, in 350e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * order to avoid interactions between multiple threads that could make 351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a single spare inadequate. 352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *spare; 354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 355e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Number of pages in active runs. */ 356bc25a47ee0e2ac8e10a94d5fa070f0dbbdeb7e7eJason Evans size_t nactive; 357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current count of pages within unused runs that are potentially 360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * dirty, and for which madvise(... MADV_DONTNEED) has not been called. 361e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * By tracking this, we can institute a limit on how much dirty unused 362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * memory is mapped for each arena. 363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty; 365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 367799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * Approximate number of pages being purged. It is possible for 368799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * multiple threads to purge dirty pages concurrently, and they use 369799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * npurgatory to indicate the total number of pages all threads are 370799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * attempting to purge. 37105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 372799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans size_t npurgatory; 37305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 37405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 37519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Size/address-ordered trees of this arena's available runs. The trees 376e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * are used for first-best-fit run allocation. 377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 378e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_t runs_avail; 379e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 380b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans /* bins is used to store trees of free regions. */ 381b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans arena_bin_t bins[NBINS]; 382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 384e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_STRUCTS */ 385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_EXTERNS 387e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 38884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evansextern ssize_t opt_lg_dirty_mult; 38941ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans/* 39041ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans * small_size2bin is a compact lookup table that rounds request sizes up to 39141ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans * size classes. In order to reduce cache footprint, the table is compressed, 39241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans * and all accesses are via the SMALL_SIZE2BIN macro. 39341ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans */ 394b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansextern uint8_t const small_size2bin[]; 39541ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN]) 396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 397b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansextern arena_bin_info_t arena_bin_info[NBINS]; 398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 399b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans/* Number of large size classes. */ 4007393f44ff025ca67716fc53b68003fd65122fd97Jason Evans#define nlclasses (chunk_npages - map_bias) 4013c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans 4026005f0710cf07d60659d91b20b7ff5592d310027Jason Evansvoid arena_purge_all(arena_t *arena); 403dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, 4047372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t binind, uint64_t prof_accumbytes); 405122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, 406122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bool zero); 407122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); 408e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid *arena_malloc_small(arena_t *arena, size_t size, bool zero); 409dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid *arena_malloc_large(arena_t *arena, size_t size, bool zero); 4105ff709c264e52651de25b788692c62ff1f6f389cJason Evansvoid *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); 4110b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansvoid arena_prof_promoted(const void *ptr, size_t size); 412203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 413e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm); 414203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 415203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind, arena_chunk_map_t *mapelm); 416203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 417203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind); 418203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, 419203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans void *ptr); 420e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); 4218e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansvoid *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, 4228e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t extra, bool zero); 423609ae595f0358157b19311b0f9f9591db7cee705Jason Evansvoid *arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, 424609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, 425609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bool try_tcache_dalloc); 426609ae595f0358157b19311b0f9f9591db7cee705Jason Evansdss_prec_t arena_dss_prec_get(arena_t *arena); 427609ae595f0358157b19311b0f9f9591db7cee705Jason Evansvoid arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); 428609ae595f0358157b19311b0f9f9591db7cee705Jason Evansvoid arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, 429609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 430609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_large_stats_t *lstats); 431e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool arena_new(arena_t *arena, unsigned ind); 432b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansvoid arena_boot(void); 4334e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_prefork(arena_t *arena); 4344e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_postfork_parent(arena_t *arena); 4354e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_postfork_child(arena_t *arena); 436e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 437e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_EXTERNS */ 438e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 439e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_INLINES 440e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifndef JEMALLOC_ENABLE_INLINE 442203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_chunk_map_t *arena_mapp_get(arena_chunk_t *chunk, size_t pageind); 443203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); 444203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); 445203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, 446203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind); 447203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); 448203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); 44980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evanssize_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); 450203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); 451203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); 452203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); 453203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind); 454203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, 455203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size, size_t flags); 456203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 457203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size); 458203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, 459203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size, size_t flags); 460203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 461203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind); 462203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, 463203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t runind, size_t binind, size_t flags); 464203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, 465203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t unzeroed); 466a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansvoid arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); 467a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansvoid arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); 468a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansvoid arena_prof_accum(arena_t *arena, uint64_t accumbytes); 46980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evanssize_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); 47049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evanssize_t arena_bin_index(arena_t *arena, arena_bin_t *bin); 47149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansunsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, 472b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans const void *ptr); 47381b4e6eb6f06cac048e3743787a70676f1534269Jason Evansprof_ctx_t *arena_prof_ctx_get(const void *ptr); 474e4f7846f1fd279a039ffa2a41707348187219de4Jason Evansvoid arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); 47501b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evansvoid *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); 476f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evanssize_t arena_salloc(const void *ptr, bool demote); 47701b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evansvoid arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, 47801b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans bool try_tcache); 479e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 480e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 481e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) 482203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans# ifdef JEMALLOC_ARENA_INLINE_A 483203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE arena_chunk_map_t * 484203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapp_get(arena_chunk_t *chunk, size_t pageind) 485203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 486203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 487203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind >= map_bias); 488203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind < chunk_npages); 489203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 490203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (&chunk->map[pageind-map_bias]); 491203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 492203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 493203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE size_t * 494203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) 495203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 496203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 497203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (&arena_mapp_get(chunk, pageind)->bits); 498203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 499203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 500203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE size_t 501203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_get(arena_chunk_t *chunk, size_t pageind) 502203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 503203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 504203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (*arena_mapbitsp_get(chunk, pageind)); 505203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 506203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 507203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE size_t 508203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) 509203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 510203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 511203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 512203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 513203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 514203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & ~PAGE_MASK); 515203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 516203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 517203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE size_t 518203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) 519203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 520203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 521203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 522203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 523203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 524203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); 525203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & ~PAGE_MASK); 526203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 527203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 528203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE size_t 529203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) 530203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 531203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 532203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 533203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 534203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 535203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans CHUNK_MAP_ALLOCATED); 536203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits >> LG_PAGE); 537203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 538203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 539203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE size_t 54080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evansarena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) 54180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans{ 54280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t mapbits; 54380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t binind; 54480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 54580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans mapbits = arena_mapbits_get(chunk, pageind); 54680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 54780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind < NBINS || binind == BININD_INVALID); 54880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans return (binind); 54980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans} 55080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 55180737c3323dabc45232affcaeb99ac2bad6ea647Jason EvansJEMALLOC_INLINE size_t 552203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) 553203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 554203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 555203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 556203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 557203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_DIRTY); 558203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 559203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 560203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE size_t 561203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind) 562203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 563203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 564203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 565203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 566203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_UNZEROED); 567203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 568203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 569203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE size_t 570203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind) 571203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 572203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 573203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 574203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 575203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_LARGE); 576203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 577203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 578203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE size_t 579203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind) 580203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 581203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 582203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 583203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 584203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_ALLOCATED); 585203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 586203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 587203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE void 588203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, 589203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flags) 590203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 591203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t *mapbitsp; 592203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 593203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbitsp = arena_mapbitsp_get(chunk, pageind); 594203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((size & PAGE_MASK) == 0); 595203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0); 596d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans assert((flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == flags); 597203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans *mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags; 598203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 599203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 600203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE void 601203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 602203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size) 603203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 604203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t *mapbitsp; 605203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 606203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbitsp = arena_mapbitsp_get(chunk, pageind); 607203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((size & PAGE_MASK) == 0); 608203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((*mapbitsp & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 609203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans *mapbitsp = size | (*mapbitsp & PAGE_MASK); 610203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 611203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 612203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE void 613203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, 614203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flags) 615203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 616203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t *mapbitsp; 617d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans size_t unzeroed; 618203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 619203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbitsp = arena_mapbitsp_get(chunk, pageind); 620203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((size & PAGE_MASK) == 0); 621d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans assert((flags & CHUNK_MAP_DIRTY) == flags); 622d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans unzeroed = *mapbitsp & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ 623d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans *mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags | unzeroed | 624d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 625203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 626203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 627203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE void 628203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 629203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind) 630203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 631203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t *mapbitsp; 632203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 633203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind <= BININD_INVALID); 634203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbitsp = arena_mapbitsp_get(chunk, pageind); 635203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, pageind) == PAGE); 636203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans *mapbitsp = (*mapbitsp & ~CHUNK_MAP_BININD_MASK) | (binind << 637203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans CHUNK_MAP_BININD_SHIFT); 638203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 639203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 640203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE void 641203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, 642203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind, size_t flags) 643203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 644203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t *mapbitsp; 645d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans size_t unzeroed; 646203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 647203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind < BININD_INVALID); 648203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbitsp = arena_mapbitsp_get(chunk, pageind); 649203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind - runind >= map_bias); 650d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans assert((flags & CHUNK_MAP_DIRTY) == flags); 651d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans unzeroed = *mapbitsp & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ 652203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans *mapbitsp = (runind << LG_PAGE) | (binind << CHUNK_MAP_BININD_SHIFT) | 653d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans flags | unzeroed | CHUNK_MAP_ALLOCATED; 654203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 655203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 656203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE void 657203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, 658203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t unzeroed) 659203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 660203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t *mapbitsp; 661203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 662203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbitsp = arena_mapbitsp_get(chunk, pageind); 663203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans *mapbitsp = (*mapbitsp & ~CHUNK_MAP_UNZEROED) | unzeroed; 664203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 665203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 666a3b3386ddde8048b9d6b54c397bb93da5e806cefJason EvansJEMALLOC_INLINE void 667a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) 668a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 669a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 670a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 671a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans assert(prof_interval != 0); 672a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 673a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena->prof_accumbytes += accumbytes; 674a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans if (arena->prof_accumbytes >= prof_interval) { 675a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans prof_idump(); 676a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena->prof_accumbytes -= prof_interval; 677a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans } 678a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 679a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 680a3b3386ddde8048b9d6b54c397bb93da5e806cefJason EvansJEMALLOC_INLINE void 681a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum_locked(arena_t *arena, uint64_t accumbytes) 682a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 683a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 684a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 685a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 686a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans if (prof_interval == 0) 687a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans return; 688a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena_prof_accum_impl(arena, accumbytes); 689a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 690a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 691a3b3386ddde8048b9d6b54c397bb93da5e806cefJason EvansJEMALLOC_INLINE void 692a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum(arena_t *arena, uint64_t accumbytes) 693a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 694a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 695a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 696a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 697a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans if (prof_interval == 0) 698a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans return; 699a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans malloc_mutex_lock(&arena->lock); 700a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena_prof_accum_impl(arena, accumbytes); 701a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans malloc_mutex_unlock(&arena->lock); 702a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 703a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 704203484e2ea267e068a68fd2922263f0ff1d5ac6fJason EvansJEMALLOC_INLINE size_t 70580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evansarena_ptr_small_binind_get(const void *ptr, size_t mapbits) 706203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 707203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind; 708203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 709203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 710203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 711203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (config_debug) { 71280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_chunk_t *chunk; 71380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_t *arena; 71480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t pageind; 71580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t actual_mapbits; 71680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_run_t *run; 71780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_bin_t *bin; 71880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t actual_binind; 71980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_bin_info_t *bin_info; 72080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 72180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind != BININD_INVALID); 72280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind < NBINS); 72380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 72480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena = chunk->arena; 72580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 72680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans actual_mapbits = arena_mapbits_get(chunk, pageind); 727203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(mapbits == actual_mapbits); 72880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_large_get(chunk, pageind) == 0); 72980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 73080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 73180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans (actual_mapbits >> LG_PAGE)) << LG_PAGE)); 73280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans bin = run->bin; 73380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans actual_binind = bin - arena->bins; 734203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind == actual_binind); 73580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans bin_info = &arena_bin_info[actual_binind]; 736203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 737203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval 738203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans == 0); 739203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 740203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 741203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (binind); 742203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 743203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans# endif /* JEMALLOC_ARENA_INLINE_A */ 744203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 745203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans# ifdef JEMALLOC_ARENA_INLINE_B 74649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason EvansJEMALLOC_INLINE size_t 74749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_bin_index(arena_t *arena, arena_bin_t *bin) 74849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 74949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = bin - arena->bins; 750b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 75149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (binind); 75249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 75349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 75481b4e6eb6f06cac048e3743787a70676f1534269Jason EvansJEMALLOC_INLINE unsigned 755b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evansarena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) 75681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 75781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans unsigned shift, diff, regind; 758122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t interval; 75981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 76084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 76184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Freeing a pointer lower than region zero can cause assertion 76284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * failure. 76384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 76484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert((uintptr_t)ptr >= (uintptr_t)run + 76584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset); 76681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 76781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 76881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * Avoid doing division with a variable divisor if possible. Using 76981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * actual division here can reduce allocator throughput by over 20%! 77081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 77149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - 77249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg0_offset); 77381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 77481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* Rescale (factor powers of 2 out of the numerator and denominator). */ 775122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans interval = bin_info->reg_interval; 776122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans shift = ffs(interval) - 1; 77781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans diff >>= shift; 778122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans interval >>= shift; 77981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 780122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (interval == 1) { 78181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* The divisor was a power of 2. */ 78281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans regind = diff; 78381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } else { 78481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 78581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * To divide by a number D that is not a power of two we 78681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * multiply by (2^21 / D) and then right shift by 21 positions. 78781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 78881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * X / D 78981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 79081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * becomes 79181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 792122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT 79381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 79481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * We can omit the first three elements, because we never 79581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * divide by 0, and 1 and 2 are both powers of two, which are 79681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * handled above. 79781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 79847e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) 79947e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) 800122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans static const unsigned interval_invs[] = { 80181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(3), 80281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 80381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 80481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 80581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 80681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 80781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 80881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 80981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans }; 81081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 811122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (interval <= ((sizeof(interval_invs) / sizeof(unsigned)) + 812122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 2)) { 813122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans regind = (diff * interval_invs[interval - 3]) >> 814122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans SIZE_INV_SHIFT; 815122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else 816122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans regind = diff / interval; 81781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV 81881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV_SHIFT 81981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } 820122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(diff == regind * interval); 82149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(regind < bin_info->nregs); 82281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 82381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (regind); 82481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 82581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 82681b4e6eb6f06cac048e3743787a70676f1534269Jason EvansJEMALLOC_INLINE prof_ctx_t * 82781b4e6eb6f06cac048e3743787a70676f1534269Jason Evansarena_prof_ctx_get(const void *ptr) 82881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 82981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans prof_ctx_t *ret; 83081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans arena_chunk_t *chunk; 83181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans size_t pageind, mapbits; 83281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 8337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 83481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert(ptr != NULL); 83581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 83681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 83781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 838ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 839203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 84081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 84181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 84281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans if (prof_promote) 84381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans ret = (prof_ctx_t *)(uintptr_t)1U; 84481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans else { 84581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 846ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << 847ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 84880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t binind = arena_ptr_small_binind_get(ptr, 84980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans mapbits); 85049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 85181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans unsigned regind; 85281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 853b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans regind = arena_run_regind(run, bin_info, ptr); 85481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans ret = *(prof_ctx_t **)((uintptr_t)run + 85549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->ctx0_offset + (regind * 85681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans sizeof(prof_ctx_t *))); 85781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } 85881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } else 859203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans ret = arena_mapp_get(chunk, pageind)->prof_ctx; 86081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 86181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (ret); 86281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 863e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 864e4f7846f1fd279a039ffa2a41707348187219de4Jason EvansJEMALLOC_INLINE void 865e4f7846f1fd279a039ffa2a41707348187219de4Jason Evansarena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) 866e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans{ 867e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans arena_chunk_t *chunk; 868e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans size_t pageind, mapbits; 869e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 8707372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 871e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert(ptr != NULL); 872e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 873e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 874e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 875ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 876203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 877e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 878e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 879e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans if (prof_promote == false) { 880e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 881ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << 882ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans LG_PAGE)); 88349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 88449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 885b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans unsigned regind; 886e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 88780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = arena_ptr_small_binind_get(ptr, mapbits); 88849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 889b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans regind = arena_run_regind(run, bin_info, ptr); 890e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 89149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans *((prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset 892e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans + (regind * sizeof(prof_ctx_t *)))) = ctx; 893e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans } else 894e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert((uintptr_t)ctx == (uintptr_t)1U); 895e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans } else 896203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_mapp_get(chunk, pageind)->prof_ctx = ctx; 897e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans} 89881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 899962463d9b57bcc65de2fa108a691b4183b9b2fafJason EvansJEMALLOC_INLINE void * 90001b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evansarena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) 901962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans{ 902ef8897b4b938111fcc9b54725067f1dbb33a4c20Jason Evans tcache_t *tcache; 903962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 904962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans assert(size != 0); 905cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans assert(size <= arena_maxclass); 906962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 907b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (size <= SMALL_MAXCLASS) { 90801b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans if (try_tcache && (tcache = tcache_get(true)) != NULL) 909962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans return (tcache_alloc_small(tcache, size, zero)); 91001b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans else { 91101b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans return (arena_malloc_small(choose_arena(arena), size, 91201b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans zero)); 91301b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans } 914962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans } else { 915746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans /* 916746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans * Initialize tcache after checking size in order to avoid 917746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans * infinite recursion during tcache initialization. 918746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans */ 91901b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans if (try_tcache && size <= tcache_maxclass && (tcache = 92001b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans tcache_get(true)) != NULL) 921962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans return (tcache_alloc_large(tcache, size, zero)); 92201b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans else { 92301b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans return (arena_malloc_large(choose_arena(arena), size, 92401b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans zero)); 92501b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans } 926962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans } 927962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans} 928962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 929f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans/* Return the size of the allocation pointed to by ptr. */ 930f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason EvansJEMALLOC_INLINE size_t 931f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evansarena_salloc(const void *ptr, bool demote) 932f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans{ 933f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans size_t ret; 934f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans arena_chunk_t *chunk; 93580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t pageind, binind; 936f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 937f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(ptr != NULL); 938f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 939f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 940f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 941f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 942203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 94380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = arena_mapbits_binind_get(chunk, pageind); 94480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans if (binind == BININD_INVALID || (config_prof && demote == false && 94580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans prof_promote && arena_mapbits_large_get(chunk, pageind) != 0)) { 94680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans /* 94780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * Large allocation. In the common case (demote == true), and 94880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * as this is an inline function, most callers will only end up 94980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * looking at binind to determine that ptr is a small 95080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * allocation. 95180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans */ 952f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 953203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans ret = arena_mapbits_large_size_get(chunk, pageind); 954f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(ret != 0); 95580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(pageind + (ret>>LG_PAGE) <= chunk_npages); 95680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(ret == PAGE || arena_mapbits_large_size_get(chunk, 95780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind+(ret>>LG_PAGE)-1) == 0); 95880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind == arena_mapbits_binind_get(chunk, 95980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind+(ret>>LG_PAGE)-1)); 96080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_dirty_get(chunk, pageind) == 96180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1)); 96280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans } else { 96380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans /* 96480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * Small allocation (possibly promoted to a large object due to 96580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * prof_promote). 96680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans */ 96780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_large_get(chunk, pageind) != 0 || 96880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 96980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind)) == binind); 97080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans ret = arena_bin_info[binind].reg_size; 971f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans } 972f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 973f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans return (ret); 974f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans} 975f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 976e476f8a161d445211fd6e54fe370275196e66bcbJason EvansJEMALLOC_INLINE void 97701b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evansarena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache) 978e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 979203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind, mapbits; 98001b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans tcache_t *tcache; 981e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 982e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(arena != NULL); 983e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(chunk->arena == arena); 984e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 985e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 986e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 987ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 988203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 989203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 990203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 991e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Small allocation. */ 992203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (try_tcache && (tcache = tcache_get(false)) != NULL) { 993203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind; 99486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 99580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = arena_ptr_small_binind_get(ptr, mapbits); 996203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tcache_dalloc_small(tcache, ptr, binind); 997203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } else 998203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans arena_dalloc_small(arena, chunk, ptr, pageind); 999f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans } else { 1000203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size = arena_mapbits_large_size_get(chunk, pageind); 1001962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1002962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 1003962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 100401b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans if (try_tcache && size <= tcache_maxclass && (tcache = 100501b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans tcache_get(false)) != NULL) { 1006962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans tcache_dalloc_large(tcache, ptr, size); 1007203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } else 1008dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_dalloc_large(arena, chunk, ptr); 1009f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans } 1010e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 1011203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans# endif /* JEMALLOC_ARENA_INLINE_B */ 1012e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1013e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1014e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_INLINES */ 1015e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 1016