arena.h revision ff6a31d3b92b7c63446ce645341d2bbd77b67dc6
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 2e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_TYPES 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * as small as possible such that this setting is still honored, without 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * violating other constraints. The goal is to make runs as small as possible 8e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * without exceeding a per run external fragmentation threshold. 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * We use binary fixed point math for overhead computations, where the binary 11e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * point is implicitly RUN_BFP bits to the left. 12e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be 148ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * honored for some/all object sizes, since when heap profiling is enabled 158ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * there is one pointer of header overhead per object (plus a constant). This 168ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * constraint is relaxed (ignored) for runs that are so small that the 178ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * per-region overhead is greater than: 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 19122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * (RUN_MAX_OVRHD / (reg_interval << (3+RUN_BFP)) 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 21e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define RUN_BFP 12 22e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* \/ Implicit binary fixed point. */ 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define RUN_MAX_OVRHD 0x0000003dU 24e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define RUN_MAX_OVRHD_RELAX 0x00001800U 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2647e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans/* Maximum number of regions in one run. */ 2747e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define LG_RUN_MAXREGS 11 2847e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) 2947e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 31122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Minimum redzone size. Redzones may be larger than this if necessary to 32122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * preserve region alignment. 33122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans */ 34122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans#define REDZONE_MINSIZE 16 35122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 36122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans/* 37e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The minimum ratio of active:dirty pages per arena is computed as: 38e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * (nactive >> opt_lg_dirty_mult) >= ndirty 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 41e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * So, supposing that opt_lg_dirty_mult is 3, there can be no less than 8 times 42e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * as many active pages as dirty pages. 43e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 44e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans#define LG_DIRTY_MULT_DEFAULT 3 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 46ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; 47ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_chunk_s arena_chunk_t; 49e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_run_s arena_run_t; 5049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evanstypedef struct arena_bin_info_s arena_bin_info_t; 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_bin_s arena_bin_t; 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_s arena_t; 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_TYPES */ 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 56e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_STRUCTS 57e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 58e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Each element of the chunk map corresponds to one page within the chunk. */ 59ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wustruct arena_chunk_map_bits_s { 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 61e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Run address (or size) and various flags are stored together. The bit 62e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * layout looks like (assuming 32-bit system): 63e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 6453bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ???????? ???????? ????nnnn nnnndula 65e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 66e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * ? : Unallocated: Run address for first/last pages, unset for internal 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * pages. 6819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Small: Run page offset. 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: Run size for first page, unset for trailing pages. 7053bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * n : binind for small size class, BININD_INVALID for large size class. 71e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * d : dirty? 728ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * u : unzeroed? 73e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * l : large? 74e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a : allocated? 75e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 76e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Following are example bit patterns for the three types of runs. 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * p : run page offset 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * s : run size 80203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * n : binind for size class; large objects set these to BININD_INVALID 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * x : don't care 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * - : 0 830b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * + : 1 843377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * [DULA] : bit set 853377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * [dula] : bit unset 86e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 8719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (clean): 8853bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++du-a 89203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxx-Uxx 9053bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++dU-a 9119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 9219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (dirty): 9353bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D--a 94203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 9553bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D--a 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 97dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Small: 98203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * pppppppp pppppppp ppppnnnn nnnnd--A 99203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * pppppppp pppppppp ppppnnnn nnnn---A 100203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * pppppppp pppppppp ppppnnnn nnnnd--A 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 102e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: 10353bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D-LA 104203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 10553bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * -------- -------- ----++++ ++++D-LA 1060b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 107ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans * Large (sampled, size <= PAGE): 108203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * ssssssss ssssssss ssssnnnn nnnnD-LA 1090b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 110ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans * Large (not sampled, size == PAGE): 11153bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D-LA 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t bits; 114203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_SHIFT 4 115203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define BININD_INVALID ((size_t)0xffU) 116203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans/* CHUNK_MAP_BININD_MASK == (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) */ 117203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_MASK ((size_t)0xff0U) 118203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK 119203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_FLAGS_MASK ((size_t)0xcU) 1200b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_DIRTY ((size_t)0x8U) 1213377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans#define CHUNK_MAP_UNZEROED ((size_t)0x4U) 1220b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_LARGE ((size_t)0x2U) 1230b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_ALLOCATED ((size_t)0x1U) 1240b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED 125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 126ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 127ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu/* 128ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just 129ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * like arena_chunk_map_bits_t. Two separate arrays are stored within each 130ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * chunk header in order to improve cache locality. 131ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu */ 132ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wustruct arena_chunk_map_misc_s { 133ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu#ifndef JEMALLOC_PROF 134ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu /* 135ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * Overlay prof_tctx in order to allow it to be referenced by dead code. 136ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * Such antics aren't warranted for per arena data structures, but 137ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * chunk map overhead accounts for a percentage of memory, rather than 138ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * being just a fixed cost. 139ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu */ 140ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu union { 141ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu#endif 142ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu /* 143ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * Linkage for run trees. There are two disjoint uses: 144ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 145ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 1) arena_t's runs_avail tree. 146ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 2) arena_run_t conceptually uses this linkage for in-use non-full 147ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * runs, rather than directly embedding linkage. 148ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu */ 149ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu rb_node(arena_chunk_map_misc_t) rb_link; 150ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 151ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu /* Profile counters, used for large object runs. */ 152ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu prof_tctx_t *prof_tctx; 153ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu#ifndef JEMALLOC_PROF 154ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu }; /* union { ... }; */ 155ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu#endif 156ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 157ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu /* Linkage for list of dirty runs. */ 158ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu ql_elm(arena_chunk_map_misc_t) dr_link; 159ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu}; 160ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef rb_tree(arena_chunk_map_misc_t) arena_avail_tree_t; 161ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef rb_tree(arena_chunk_map_misc_t) arena_run_tree_t; 162ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef ql_head(arena_chunk_map_misc_t) arena_chunk_miscelms_t; 163e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 164e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Arena chunk header. */ 165e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_chunk_s { 166e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Arena that owns the chunk. */ 167e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_t *arena; 168e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1697393f44ff025ca67716fc53b68003fd65122fd97Jason Evans /* 1707393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * Map of pages within chunk that keeps track of free/large/small. The 1717393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * first map_bias entries are omitted, since the chunk header does not 1727393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * need to be tracked in the map. This omission saves a header page 1737393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * for common chunk sizes (e.g. 4 MiB). 1747393f44ff025ca67716fc53b68003fd65122fd97Jason Evans */ 175ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu arena_chunk_map_bits_t map_bits[1]; /* Dynamically sized. */ 176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 177e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 178e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_run_s { 179e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Bin this run is associated with. */ 180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 181e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Index of next region that has never been allocated, or nregs. */ 18384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans uint32_t nextind; 184e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 185e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Number of free regions in run. */ 186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned nfree; 187e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 18949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 19084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Read-only information associated with each element of arena_t's bins array 19149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * is stored separately, partly to reduce memory usage (only one copy, rather 19249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * than one per arena), but mainly to avoid false cacheline sharing. 193122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 194122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Each run has the following layout: 195122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 196122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * /--------------------\ 197122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | arena_run_t header | 198122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 199122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * bitmap_offset | bitmap | 200122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 201122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 202122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 203122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * reg0_offset | region 0 | 204122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 205122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| \ 206122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | | 207122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | region 1 | > reg_interval 208122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | / 209122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 210122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 211122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 212122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 213122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 214122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 215122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | region nregs-1 | 216122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 217122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 218122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | alignment pad? | 219122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * \--------------------/ 220122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 221122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * reg_interval has at least the same minimum alignment as reg_size; this 222122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * preserves the alignment constraint that sa2u() depends on. Alignment pad is 223122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * either 0 or redzone_size; it is present only if needed to align reg0_offset. 22449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 22549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstruct arena_bin_info_s { 22649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Size of regions in a run for this bin's size class. */ 22749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t reg_size; 22849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 229122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Redzone size. */ 230122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t redzone_size; 231122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 232122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Interval between regions (reg_size + (redzone_size << 1)). */ 233122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t reg_interval; 234122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 23549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total size of a run for this bin's size class. */ 23649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t run_size; 23749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 23849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total number of regions in a run for this bin's size class. */ 23949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t nregs; 24049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 24184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 24284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Offset of first bitmap_t element in a run header for this bin's size 24384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * class. 24484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 24584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans uint32_t bitmap_offset; 24684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 24784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 24884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Metadata used to manipulate bitmaps for runs associated with this 24984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin. 25084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 25184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_info_t bitmap_info; 25284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 25349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Offset of first region in a run for this bin's size class. */ 25449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t reg0_offset; 25549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans}; 25649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 257e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_bin_s { 258e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 25986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * All operations on runcur, runs, and stats require that lock be 26086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. Run allocation/deallocation are protected by the arena lock, 26186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * which may be acquired while holding one or more bin locks, but not 26286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * vise versa. 26386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 26486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_t lock; 26586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 26686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 267e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current run being used to service allocations of this bin's size 268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * class. 269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 270e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *runcur; 271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 272e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Tree of non-full runs. This tree is used when looking for an 274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * existing run when runcur is no longer usable. We choose the 275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * non-full run that is lowest in memory; this policy tends to keep 276e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * objects packed well, and it can also help reduce the number of 277e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * almost-empty chunks. 278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 279e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_t runs; 280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Bin statistics. */ 282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_bin_stats_t stats; 283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 285e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_s { 2866109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* This arena's index within the arenas array. */ 2876109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned ind; 2886109fe07a14b7a619365977d9523db9f8b333792Jason Evans 28986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 290597632be188d2bcc135dad2145cc46ef44897aadJason Evans * Number of threads currently assigned to this arena. This field is 291597632be188d2bcc135dad2145cc46ef44897aadJason Evans * protected by arenas_lock. 292597632be188d2bcc135dad2145cc46ef44897aadJason Evans */ 293597632be188d2bcc135dad2145cc46ef44897aadJason Evans unsigned nthreads; 294597632be188d2bcc135dad2145cc46ef44897aadJason Evans 295597632be188d2bcc135dad2145cc46ef44897aadJason Evans /* 296597632be188d2bcc135dad2145cc46ef44897aadJason Evans * There are three classes of arena operations from a locking 297597632be188d2bcc135dad2145cc46ef44897aadJason Evans * perspective: 298597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 1) Thread asssignment (modifies nthreads) is protected by 299597632be188d2bcc135dad2145cc46ef44897aadJason Evans * arenas_lock. 300597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 2) Bin-related operations are protected by bin locks. 301597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 3) Chunk- and run-related operations are protected by this mutex. 30286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_t lock; 304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_stats_t stats; 306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * List of tcaches for extant threads associated with this arena. 308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Stats from these are merged incrementally, and at exit. 309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_head(tcache_t) tcache_ql; 311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 312d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans uint64_t prof_accumbytes; 313d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 314609ae595f0358157b19311b0f9f9591db7cee705Jason Evans dss_prec_t dss_prec; 315609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 316e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 317e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * In order to avoid rapid chunk allocation/deallocation when an arena 318e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * oscillates right on the cusp of needing a new chunk, cache the most 319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * recently freed chunk. The spare is left in the arena's chunk trees 320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * until it is deleted. 321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * There is one spare chunk per arena, rather than one spare total, in 323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * order to avoid interactions between multiple threads that could make 324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a single spare inadequate. 325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *spare; 327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 328e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evans /* Number of pages in active runs and huge regions. */ 329bc25a47ee0e2ac8e10a94d5fa070f0dbbdeb7e7eJason Evans size_t nactive; 330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current count of pages within unused runs that are potentially 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * dirty, and for which madvise(... MADV_DONTNEED) has not been called. 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * By tracking this, we can institute a limit on how much dirty unused 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * memory is mapped for each arena. 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty; 338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 34019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Size/address-ordered trees of this arena's available runs. The trees 341e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * are used for first-best-fit run allocation. 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 343e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_t runs_avail; 344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 345070b3c3fbd90296610005c111ec6060e8bb23d31Jason Evans /* List of dirty runs this arena manages. */ 346ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu arena_chunk_miscelms_t runs_dirty; 347070b3c3fbd90296610005c111ec6060e8bb23d31Jason Evans 348fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind /* 349fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind * user-configureable chunk allocation and deallocation functions. 350fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind */ 351fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind chunk_alloc_t *chunk_alloc; 352e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evans chunk_dalloc_t *chunk_dalloc; 353fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind 354b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans /* bins is used to store trees of free regions. */ 355b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans arena_bin_t bins[NBINS]; 356e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_STRUCTS */ 359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_EXTERNS 361e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 36284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evansextern ssize_t opt_lg_dirty_mult; 36341ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans/* 3643541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans * small_size2bin_tab is a compact lookup table that rounds request sizes up to 36541ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans * size classes. In order to reduce cache footprint, the table is compressed, 3663541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans * and all accesses are via small_size2bin(). 36741ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans */ 3683541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evansextern uint8_t const small_size2bin_tab[]; 3693541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans/* 3703541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans * small_bin2size_tab duplicates information in arena_bin_info, but in a const 3713541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans * array, for which it is easier for the compiler to optimize repeated 3723541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans * dereferences. 3733541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans */ 3743541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evansextern uint32_t const small_bin2size_tab[NBINS]; 375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 376b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansextern arena_bin_info_t arena_bin_info[NBINS]; 377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 378b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans/* Number of large size classes. */ 3797393f44ff025ca67716fc53b68003fd65122fd97Jason Evans#define nlclasses (chunk_npages - map_bias) 3803c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans 381e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evansvoid *arena_chunk_alloc_huge(arena_t *arena, size_t size, size_t alignment, 382e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evans bool *zero); 383e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evansvoid arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t size); 3846005f0710cf07d60659d91b20b7ff5592d310027Jason Evansvoid arena_purge_all(arena_t *arena); 385dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, 3867372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t binind, uint64_t prof_accumbytes); 387122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, 388122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bool zero); 3890d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#ifdef JEMALLOC_JET 3900d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evanstypedef void (arena_redzone_corruption_t)(void *, size_t, bool, size_t, 3910d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans uint8_t); 3926b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_redzone_corruption_t *arena_redzone_corruption; 3936b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_dalloc_junk_small_t)(void *, arena_bin_info_t *); 3946b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_dalloc_junk_small_t *arena_dalloc_junk_small; 3956b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#else 396122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); 3976b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 3980d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansvoid arena_quarantine_junk_small(void *ptr, size_t usize); 399e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid *arena_malloc_small(arena_t *arena, size_t size, bool zero); 400dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid *arena_malloc_large(arena_t *arena, size_t size, bool zero); 4015ff709c264e52651de25b788692c62ff1f6f389cJason Evansvoid *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); 4020b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansvoid arena_prof_promoted(const void *ptr, size_t size); 403203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 404ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu arena_chunk_map_bits_t *bitselm); 405203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 406ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind, arena_chunk_map_bits_t *bitselm); 407203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 408203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind); 4096b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 4106b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_dalloc_junk_large_t)(void *, size_t); 4116b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_dalloc_junk_large_t *arena_dalloc_junk_large; 4126b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 413203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, 414203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans void *ptr); 415e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); 4166b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 4176b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t); 4186b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_ralloc_junk_large_t *arena_ralloc_junk_large; 4196b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 420b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evansbool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, 4218e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t extra, bool zero); 422609ae595f0358157b19311b0f9f9591db7cee705Jason Evansvoid *arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, 423609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, 424609ae595f0358157b19311b0f9f9591db7cee705Jason Evans bool try_tcache_dalloc); 425609ae595f0358157b19311b0f9f9591db7cee705Jason Evansdss_prec_t arena_dss_prec_get(arena_t *arena); 4264d434adb146375ad17f0d5e994ed5728d2942e3fJason Evansbool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); 427609ae595f0358157b19311b0f9f9591db7cee705Jason Evansvoid arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, 428609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 429609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_large_stats_t *lstats); 430e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool arena_new(arena_t *arena, unsigned ind); 431b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansvoid arena_boot(void); 4324e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_prefork(arena_t *arena); 4334e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_postfork_parent(arena_t *arena); 4344e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_postfork_child(arena_t *arena); 435e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 436e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_EXTERNS */ 437e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 438e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_INLINES 439e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 440e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifndef JEMALLOC_ENABLE_INLINE 441d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_size2bin_compute(size_t size); 442d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_size2bin_lookup(size_t size); 4433541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evanssize_t small_size2bin(size_t size); 444d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_bin2size_compute(size_t binind); 445d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_bin2size_lookup(size_t binind); 4463541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evanssize_t small_bin2size(size_t binind); 447d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_s2u_compute(size_t size); 448d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_s2u_lookup(size_t size); 449d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_s2u(size_t size); 450ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_chunk_map_bits_t *arena_bitselm_get(arena_chunk_t *chunk, 451ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind); 452ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_chunk_map_misc_t *arena_miscelm_get(arena_chunk_t *chunk, 453ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind); 454203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); 45587a02d2bb18dbcb2955541b849bc95862e864803Jason Evanssize_t arena_mapbitsp_read(size_t *mapbitsp); 456203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); 457203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, 458203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind); 459203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); 460203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); 46180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evanssize_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); 462203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); 463203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); 464203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); 465203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind); 46687a02d2bb18dbcb2955541b849bc95862e864803Jason Evansvoid arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits); 467203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, 468203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size, size_t flags); 469203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 470203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size); 471203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, 472203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size, size_t flags); 473203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 474203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind); 475203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, 476203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t runind, size_t binind, size_t flags); 477203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, 478203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t unzeroed); 47988c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); 48088c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); 48188c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum(arena_t *arena, uint64_t accumbytes); 48280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evanssize_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); 48349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evanssize_t arena_bin_index(arena_t *arena, arena_bin_t *bin); 48449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansunsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, 485b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans const void *ptr); 486602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansprof_tctx_t *arena_prof_tctx_get(const void *ptr); 487602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansvoid arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); 48801b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evansvoid *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); 489f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evanssize_t arena_salloc(const void *ptr, bool demote); 490be8e59f5a64ef775c9694aee0d6a87d92336d303Ben Maurervoid arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache); 491e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 492e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 493e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) 494203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans# ifdef JEMALLOC_ARENA_INLINE_A 495d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_INLINE size_t 496d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_size2bin_compute(size_t size) 497d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 498d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#if (NTBINS != 0) 499d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { 500d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; 501d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_ceil = lg_floor(pow2_ceil(size)); 502d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); 503d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } else 504d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#endif 505d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans { 506d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t x = lg_floor((size<<1)-1); 507d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : 508d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); 509d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t grp = shift << LG_SIZE_CLASS_GROUP; 510d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 511d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) 512d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; 513d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 514d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t delta_inverse_mask = ZI(-1) << lg_delta; 515d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & 516d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); 517d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 518d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t bin = NTBINS + grp + mod; 519d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (bin); 520d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } 521d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 522d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 523d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_ALWAYS_INLINE size_t 524d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_size2bin_lookup(size_t size) 525d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 526d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 527d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(size <= LOOKUP_MAXCLASS); 528d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans { 529d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t ret = ((size_t)(small_size2bin_tab[(size-1) >> 530d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans LG_TINY_MIN])); 531d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(ret == small_size2bin_compute(size)); 532d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (ret); 533d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } 534d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 535d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 5363541a904d6fb949f3f0aea05418ccce7cbd4b705Jason EvansJEMALLOC_ALWAYS_INLINE size_t 5373541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evanssmall_size2bin(size_t size) 5383541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans{ 5393541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans 540d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(size > 0); 541d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans if (size <= LOOKUP_MAXCLASS) 542d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (small_size2bin_lookup(size)); 543d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans else 544d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (small_size2bin_compute(size)); 545d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 546d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 547d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_INLINE size_t 548d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_bin2size_compute(size_t binind) 549d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 550d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#if (NTBINS > 0) 551d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans if (binind < NTBINS) 552d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + binind)); 553d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans else 554d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#endif 555d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans { 556d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t reduced_binind = binind - NTBINS; 557d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t grp = reduced_binind >> LG_SIZE_CLASS_GROUP; 558d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t mod = reduced_binind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 559d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 1); 560d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 561d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t grp_size_mask = ~((!!grp)-1); 562d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t grp_size = ((ZU(1) << (LG_QUANTUM + 563d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; 564d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 565d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t shift = (grp == 0) ? 1 : grp; 566d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_delta = shift + (LG_QUANTUM-1); 567d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t mod_size = (mod+1) << lg_delta; 568d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 569d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t usize = grp_size + mod_size; 570d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (usize); 571d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } 572d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 573d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 574d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_ALWAYS_INLINE size_t 575d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_bin2size_lookup(size_t binind) 576d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 577d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 578d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(binind < NBINS); 579d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans { 580d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t ret = ((size_t)(small_bin2size_tab[binind])); 581d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(ret == small_bin2size_compute(binind)); 582d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (ret); 583d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } 5843541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans} 5853541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans 5863541a904d6fb949f3f0aea05418ccce7cbd4b705Jason EvansJEMALLOC_ALWAYS_INLINE size_t 5873541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evanssmall_bin2size(size_t binind) 5883541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans{ 5893541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans 590d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (small_bin2size_lookup(binind)); 591d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 592d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 593d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_ALWAYS_INLINE size_t 594d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_s2u_compute(size_t size) 595d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 596d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#if (NTBINS > 0) 597d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { 598d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; 599d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_ceil = lg_floor(pow2_ceil(size)); 600d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : 601d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans (ZU(1) << lg_ceil)); 602d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } else 603d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#endif 604d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans { 605d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t x = lg_floor((size<<1)-1); 606d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) 607d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; 608d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t delta = ZU(1) << lg_delta; 609d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t delta_mask = delta - 1; 610d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t usize = (size + delta_mask) & ~delta_mask; 611d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (usize); 612d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } 613d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 614d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 615d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_ALWAYS_INLINE size_t 616d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_s2u_lookup(size_t size) 617d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 618d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t ret = (small_bin2size(small_size2bin(size))); 619d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 620d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(ret == small_s2u_compute(size)); 621d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (ret); 622d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 623d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 624d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_ALWAYS_INLINE size_t 625d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_s2u(size_t size) 626d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 627d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 628d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(size > 0); 629d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans if (size <= LOOKUP_MAXCLASS) 630d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (small_s2u_lookup(size)); 631d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans else 632d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (small_s2u_compute(size)); 6333541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans} 6343541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans# endif /* JEMALLOC_ARENA_INLINE_A */ 6353541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans 6363541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans# ifdef JEMALLOC_ARENA_INLINE_B 637ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan WuJEMALLOC_ALWAYS_INLINE arena_chunk_map_bits_t * 638ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_bitselm_get(arena_chunk_t *chunk, size_t pageind) 639ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu{ 640ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 641ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu assert(pageind >= map_bias); 642ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu assert(pageind < chunk_npages); 643ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 644ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return (&chunk->map_bits[pageind-map_bias]); 645ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu} 646ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 647ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan WuJEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 648ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_miscelm_get(arena_chunk_t *chunk, size_t pageind) 649203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 650203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 651203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind >= map_bias); 652203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind < chunk_npages); 653203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 654ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return ((arena_chunk_map_misc_t *)((uintptr_t)chunk + 655ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu (uintptr_t)map_misc_offset) + pageind-map_bias); 656203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 657203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 65888393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t * 659203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) 660203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 661203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 662ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return (&arena_bitselm_get(chunk, pageind)->bits); 663203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 664203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 66588393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 66687a02d2bb18dbcb2955541b849bc95862e864803Jason Evansarena_mapbitsp_read(size_t *mapbitsp) 66787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans{ 66887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 66987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans return (*mapbitsp); 67087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans} 67187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 67287a02d2bb18dbcb2955541b849bc95862e864803Jason EvansJEMALLOC_ALWAYS_INLINE size_t 673203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_get(arena_chunk_t *chunk, size_t pageind) 674203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 675203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 67687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans return (arena_mapbitsp_read(arena_mapbitsp_get(chunk, pageind))); 677203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 678203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 67988393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 680203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) 681203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 682203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 683203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 684203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 685203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 686203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & ~PAGE_MASK); 687203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 688203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 68988393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 690203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) 691203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 692203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 693203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 694203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 695203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 696203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); 697203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & ~PAGE_MASK); 698203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 699203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 70088393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 701203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) 702203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 703203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 704203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 705203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 706203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 707203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans CHUNK_MAP_ALLOCATED); 708203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits >> LG_PAGE); 709203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 710203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 71188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 71280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evansarena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) 71380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans{ 71480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t mapbits; 71580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t binind; 71680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 71780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans mapbits = arena_mapbits_get(chunk, pageind); 71880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 71980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind < NBINS || binind == BININD_INVALID); 72080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans return (binind); 72180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans} 72280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 72388393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 724203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) 725203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 726203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 727203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 728203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 729203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_DIRTY); 730203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 731203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 73288393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 733203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind) 734203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 735203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 736203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 737203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 738203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_UNZEROED); 739203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 740203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 74188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 742203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind) 743203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 744203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 745203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 746203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 747203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_LARGE); 748203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 749203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 75088393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 751203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind) 752203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 753203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 754203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 755203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 756203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_ALLOCATED); 757203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 758203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 75988393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 76087a02d2bb18dbcb2955541b849bc95862e864803Jason Evansarena_mapbitsp_write(size_t *mapbitsp, size_t mapbits) 76187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans{ 76287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 76387a02d2bb18dbcb2955541b849bc95862e864803Jason Evans *mapbitsp = mapbits; 76487a02d2bb18dbcb2955541b849bc95862e864803Jason Evans} 76587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 76687a02d2bb18dbcb2955541b849bc95862e864803Jason EvansJEMALLOC_ALWAYS_INLINE void 767203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, 768203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flags) 769203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 77087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 771203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 772203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((size & PAGE_MASK) == 0); 773203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0); 774d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans assert((flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == flags); 77587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags); 776203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 777203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 77888393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 779203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 780203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size) 781203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 78287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 78387a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 784203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 785203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((size & PAGE_MASK) == 0); 78687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 78787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, size | (mapbits & PAGE_MASK)); 788203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 789203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 79088393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 791203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, 792203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flags) 793203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 79487a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 79587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 796d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans size_t unzeroed; 797203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 798203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((size & PAGE_MASK) == 0); 799d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans assert((flags & CHUNK_MAP_DIRTY) == flags); 80087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ 80187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags 80287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans | unzeroed | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED); 803203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 804203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 80588393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 806203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 807203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind) 808203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 80987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 81087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 811203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 812203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind <= BININD_INVALID); 813203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, pageind) == PAGE); 81487a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) | 81587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans (binind << CHUNK_MAP_BININD_SHIFT)); 816203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 817203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 81888393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 819203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, 820203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind, size_t flags) 821203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 82287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 82387a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 824d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans size_t unzeroed; 825203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 826203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind < BININD_INVALID); 827203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind - runind >= map_bias); 828d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans assert((flags & CHUNK_MAP_DIRTY) == flags); 82987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ 83087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, (runind << LG_PAGE) | (binind << 83187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans CHUNK_MAP_BININD_SHIFT) | flags | unzeroed | CHUNK_MAP_ALLOCATED); 832203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 833203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 83488393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 835203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, 836203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t unzeroed) 837203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 83887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 83987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 840203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 84187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_UNZEROED) | 84287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans unzeroed); 843203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 844203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 84588c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 846a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) 847a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 848a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 849a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 850a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans assert(prof_interval != 0); 851a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 852a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena->prof_accumbytes += accumbytes; 853a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans if (arena->prof_accumbytes >= prof_interval) { 854a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena->prof_accumbytes -= prof_interval; 85588c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (true); 856a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans } 85788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 858a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 859a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 86088c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 861a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum_locked(arena_t *arena, uint64_t accumbytes) 862a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 863a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 864a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 865a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 866a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans if (prof_interval == 0) 86788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 86888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (arena_prof_accum_impl(arena, accumbytes)); 869a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 870a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 87188c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 872a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum(arena_t *arena, uint64_t accumbytes) 873a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 874a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 875a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 876a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 877a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans if (prof_interval == 0) 87888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 87988c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans 88088c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans { 88188c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans bool ret; 88288c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans 88388c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans malloc_mutex_lock(&arena->lock); 88488c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans ret = arena_prof_accum_impl(arena, accumbytes); 88588c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans malloc_mutex_unlock(&arena->lock); 88688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (ret); 88788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans } 888a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 889a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 89088393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 89180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evansarena_ptr_small_binind_get(const void *ptr, size_t mapbits) 892203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 893203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind; 894203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 895203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 896203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 897203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (config_debug) { 89880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_chunk_t *chunk; 89980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_t *arena; 90080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t pageind; 90180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t actual_mapbits; 90280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_run_t *run; 90380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_bin_t *bin; 90480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t actual_binind; 90580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_bin_info_t *bin_info; 90680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 90780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind != BININD_INVALID); 90880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind < NBINS); 90980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 91080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena = chunk->arena; 91180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 91280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans actual_mapbits = arena_mapbits_get(chunk, pageind); 913203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(mapbits == actual_mapbits); 91480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_large_get(chunk, pageind) == 0); 91580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 91680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 91780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans (actual_mapbits >> LG_PAGE)) << LG_PAGE)); 91880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans bin = run->bin; 91980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans actual_binind = bin - arena->bins; 920203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind == actual_binind); 92180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans bin_info = &arena_bin_info[actual_binind]; 922203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 923203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval 924203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans == 0); 925203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 926203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 927203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (binind); 928203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 9293541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans# endif /* JEMALLOC_ARENA_INLINE_B */ 930203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 9313541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans# ifdef JEMALLOC_ARENA_INLINE_C 93249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason EvansJEMALLOC_INLINE size_t 93349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_bin_index(arena_t *arena, arena_bin_t *bin) 93449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 93549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = bin - arena->bins; 936b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 93749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (binind); 93849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 93949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 94081b4e6eb6f06cac048e3743787a70676f1534269Jason EvansJEMALLOC_INLINE unsigned 941b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evansarena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) 94281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 94381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans unsigned shift, diff, regind; 944122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t interval; 94581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 94684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 94784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Freeing a pointer lower than region zero can cause assertion 94884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * failure. 94984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 95084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert((uintptr_t)ptr >= (uintptr_t)run + 95184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset); 95281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 95381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 95481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * Avoid doing division with a variable divisor if possible. Using 95581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * actual division here can reduce allocator throughput by over 20%! 95681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 95749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - 95849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg0_offset); 95981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 96081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* Rescale (factor powers of 2 out of the numerator and denominator). */ 961122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans interval = bin_info->reg_interval; 9629c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond shift = jemalloc_ffs(interval) - 1; 96381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans diff >>= shift; 964122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans interval >>= shift; 96581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 966122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (interval == 1) { 96781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* The divisor was a power of 2. */ 96881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans regind = diff; 96981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } else { 97081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 97181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * To divide by a number D that is not a power of two we 97281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * multiply by (2^21 / D) and then right shift by 21 positions. 97381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 97481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * X / D 97581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 97681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * becomes 97781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 978122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT 97981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 98081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * We can omit the first three elements, because we never 98181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * divide by 0, and 1 and 2 are both powers of two, which are 98281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * handled above. 98381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 98447e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) 98547e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) 986122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans static const unsigned interval_invs[] = { 98781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(3), 98881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 98981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 99081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 99181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 99281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 99381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 99481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 99581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans }; 99681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 997122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (interval <= ((sizeof(interval_invs) / sizeof(unsigned)) + 998122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 2)) { 999122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans regind = (diff * interval_invs[interval - 3]) >> 1000122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans SIZE_INV_SHIFT; 1001122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else 1002122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans regind = diff / interval; 100381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV 100481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV_SHIFT 100581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } 1006122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(diff == regind * interval); 100749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(regind < bin_info->nregs); 100881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 100981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (regind); 101081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 101181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 1012602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason EvansJEMALLOC_INLINE prof_tctx_t * 1013602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansarena_prof_tctx_get(const void *ptr) 101481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 1015602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *ret; 101681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans arena_chunk_t *chunk; 101781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans size_t pageind, mapbits; 101881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 10197372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 102081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert(ptr != NULL); 102181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 102281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 102381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1024ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1025203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 102681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 10279b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) 1028602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans ret = (prof_tctx_t *)(uintptr_t)1U; 10299b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evans else 1030ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu ret = arena_miscelm_get(chunk, pageind)->prof_tctx; 103181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 103281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (ret); 103381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 1034e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 1035e4f7846f1fd279a039ffa2a41707348187219de4Jason EvansJEMALLOC_INLINE void 1036602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansarena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) 1037e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans{ 1038e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans arena_chunk_t *chunk; 10395fbad0902b845b1a6b311994468d0b9962e4fd30Jason Evans size_t pageind; 1040e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 10417372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 1042e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert(ptr != NULL); 1043e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1044e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 1045e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1046ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 10475fbad0902b845b1a6b311994468d0b9962e4fd30Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1048665769357cd77b74e00a146f196fff19243b33c4Jason Evans 10499b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evans if (arena_mapbits_large_get(chunk, pageind) != 0) 1050ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu arena_miscelm_get(chunk, pageind)->prof_tctx = tctx; 1051e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans} 105281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 105388393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void * 105401b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evansarena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) 1055962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans{ 1056ef8897b4b938111fcc9b54725067f1dbb33a4c20Jason Evans tcache_t *tcache; 1057962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1058962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans assert(size != 0); 1059cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans assert(size <= arena_maxclass); 1060962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1061b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans if (size <= SMALL_MAXCLASS) { 106201b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans if (try_tcache && (tcache = tcache_get(true)) != NULL) 1063962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans return (tcache_alloc_small(tcache, size, zero)); 106401b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans else { 106501b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans return (arena_malloc_small(choose_arena(arena), size, 106601b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans zero)); 106701b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans } 1068962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans } else { 1069746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans /* 1070746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans * Initialize tcache after checking size in order to avoid 1071746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans * infinite recursion during tcache initialization. 1072746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans */ 107301b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans if (try_tcache && size <= tcache_maxclass && (tcache = 107401b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans tcache_get(true)) != NULL) 1075962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans return (tcache_alloc_large(tcache, size, zero)); 107601b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans else { 107701b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans return (arena_malloc_large(choose_arena(arena), size, 107801b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans zero)); 107901b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans } 1080962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans } 1081962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans} 1082962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1083f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans/* Return the size of the allocation pointed to by ptr. */ 108488393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 1085f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evansarena_salloc(const void *ptr, bool demote) 1086f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans{ 1087f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans size_t ret; 1088f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans arena_chunk_t *chunk; 108980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t pageind, binind; 1090f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1091f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(ptr != NULL); 1092f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1093f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1094f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1095f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1096203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 109780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = arena_mapbits_binind_get(chunk, pageind); 109880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans if (binind == BININD_INVALID || (config_prof && demote == false && 10999b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evans arena_mapbits_large_get(chunk, pageind) != 0)) { 110080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans /* 110180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * Large allocation. In the common case (demote == true), and 110280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * as this is an inline function, most callers will only end up 110380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * looking at binind to determine that ptr is a small 110480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * allocation. 110580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans */ 1106f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 1107203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans ret = arena_mapbits_large_size_get(chunk, pageind); 1108f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(ret != 0); 110980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(pageind + (ret>>LG_PAGE) <= chunk_npages); 111080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(ret == PAGE || arena_mapbits_large_size_get(chunk, 111180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind+(ret>>LG_PAGE)-1) == 0); 111280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind == arena_mapbits_binind_get(chunk, 111380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind+(ret>>LG_PAGE)-1)); 111480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_dirty_get(chunk, pageind) == 111580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1)); 111680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans } else { 11179b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evans /* Small allocation (possibly promoted to a large object). */ 111880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_large_get(chunk, pageind) != 0 || 111980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 112080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind)) == binind); 11213541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans ret = small_bin2size(binind); 1122f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans } 1123f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1124f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans return (ret); 1125f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans} 1126f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 112788393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 1128be8e59f5a64ef775c9694aee0d6a87d92336d303Ben Maurerarena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache) 1129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1130203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind, mapbits; 113101b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans tcache_t *tcache; 1132e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1136ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1137203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 1138203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1139203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 1140e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Small allocation. */ 1141203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (try_tcache && (tcache = tcache_get(false)) != NULL) { 1142203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind; 114386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 114480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = arena_ptr_small_binind_get(ptr, mapbits); 1145203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tcache_dalloc_small(tcache, ptr, binind); 1146203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } else 1147be8e59f5a64ef775c9694aee0d6a87d92336d303Ben Maurer arena_dalloc_small(chunk->arena, chunk, ptr, pageind); 1148f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans } else { 1149203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size = arena_mapbits_large_size_get(chunk, pageind); 1150962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1151962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 1152962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 115301b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans if (try_tcache && size <= tcache_maxclass && (tcache = 115401b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans tcache_get(false)) != NULL) { 1155962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans tcache_dalloc_large(tcache, ptr, size); 1156203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } else 1157be8e59f5a64ef775c9694aee0d6a87d92336d303Ben Maurer arena_dalloc_large(chunk->arena, chunk, ptr); 1158f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans } 1159e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 11603541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans# endif /* JEMALLOC_ARENA_INLINE_C */ 1161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1162e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1163e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_INLINES */ 1164e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 1165