arena.h revision 0c5dd03e889d0269170b5db9fa872738d906eb78
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 2e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_TYPES 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 447e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans/* Maximum number of regions in one run. */ 50c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans#define LG_RUN_MAXREGS (LG_PAGE - LG_TINY_MIN) 647e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) 747e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans 8e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 9122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Minimum redzone size. Redzones may be larger than this if necessary to 10122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * preserve region alignment. 11122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans */ 12122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans#define REDZONE_MINSIZE 16 13122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 14122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans/* 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The minimum ratio of active:dirty pages per arena is computed as: 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * (nactive >> opt_lg_dirty_mult) >= ndirty 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 19e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * So, supposing that opt_lg_dirty_mult is 3, there can be no less than 8 times 20e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * as many active pages as dirty pages. 21e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 22e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans#define LG_DIRTY_MULT_DEFAULT 3 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 240c5dd03e889d0269170b5db9fa872738d906eb78Jason Evanstypedef struct arena_run_s arena_run_t; 25ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; 26ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_chunk_s arena_chunk_t; 2849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evanstypedef struct arena_bin_info_s arena_bin_info_t; 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_bin_s arena_bin_t; 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_s arena_t; 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 32e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_TYPES */ 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 34e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_STRUCTS 35e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 360c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansstruct arena_run_s { 370c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Bin this run is associated with. */ 380c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_bin_t *bin; 390c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 400c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Index of next region that has never been allocated, or nregs. */ 410c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans uint32_t nextind; 420c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 430c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Number of free regions in run. */ 440c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans unsigned nfree; 450c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 460c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Per region allocated/deallocated bitmap. */ 470c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans bitmap_t bitmap[BITMAP_GROUPS_MAX]; 480c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans}; 490c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Each element of the chunk map corresponds to one page within the chunk. */ 51ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wustruct arena_chunk_map_bits_s { 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Run address (or size) and various flags are stored together. The bit 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * layout looks like (assuming 32-bit system): 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 5653bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ???????? ???????? ????nnnn nnnndula 57e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 58e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * ? : Unallocated: Run address for first/last pages, unset for internal 59e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * pages. 6019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Small: Run page offset. 61e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: Run size for first page, unset for trailing pages. 6253bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * n : binind for small size class, BININD_INVALID for large size class. 63e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * d : dirty? 648ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * u : unzeroed? 65e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * l : large? 66e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a : allocated? 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 68e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Following are example bit patterns for the three types of runs. 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 70e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * p : run page offset 71e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * s : run size 72203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * n : binind for size class; large objects set these to BININD_INVALID 73e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * x : don't care 74e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * - : 0 750b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * + : 1 763377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * [DULA] : bit set 773377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * [dula] : bit unset 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 7919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (clean): 8053bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++du-a 81203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxx-Uxx 8253bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++dU-a 8319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 8419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (dirty): 8553bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D--a 86203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 8753bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D--a 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 89dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Small: 90203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * pppppppp pppppppp ppppnnnn nnnnd--A 91203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * pppppppp pppppppp ppppnnnn nnnn---A 92203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * pppppppp pppppppp ppppnnnn nnnnd--A 93e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: 9553bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D-LA 96203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 9753bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * -------- -------- ----++++ ++++D-LA 980b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 99ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans * Large (sampled, size <= PAGE): 100203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * ssssssss ssssssss ssssnnnn nnnnD-LA 1010b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 102ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans * Large (not sampled, size == PAGE): 10353bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D-LA 104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t bits; 106203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_SHIFT 4 107203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define BININD_INVALID ((size_t)0xffU) 108203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans/* CHUNK_MAP_BININD_MASK == (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) */ 109203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_MASK ((size_t)0xff0U) 110203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK 111203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_FLAGS_MASK ((size_t)0xcU) 1120b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_DIRTY ((size_t)0x8U) 1133377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans#define CHUNK_MAP_UNZEROED ((size_t)0x4U) 1140b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_LARGE ((size_t)0x2U) 1150b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_ALLOCATED ((size_t)0x1U) 1160b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 118ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 119ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu/* 120ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just 121ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * like arena_chunk_map_bits_t. Two separate arrays are stored within each 122ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * chunk header in order to improve cache locality. 123ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu */ 124ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wustruct arena_chunk_map_misc_s { 125ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu /* 126ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * Linkage for run trees. There are two disjoint uses: 127ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 128ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 1) arena_t's runs_avail tree. 129ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 2) arena_run_t conceptually uses this linkage for in-use non-full 130ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * runs, rather than directly embedding linkage. 131ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu */ 1320c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans rb_node(arena_chunk_map_misc_t) rb_link; 133ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 1340c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans union { 1350c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Linkage for list of dirty runs. */ 1360c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans ql_elm(arena_chunk_map_misc_t) dr_link; 137ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 1380c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Profile counters, used for large object runs. */ 1390c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans prof_tctx_t *prof_tctx; 1400c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 1410c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Small region run metadata. */ 1420c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_run_t run; 1430c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans }; 144ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu}; 145ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef rb_tree(arena_chunk_map_misc_t) arena_avail_tree_t; 146ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef rb_tree(arena_chunk_map_misc_t) arena_run_tree_t; 147ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef ql_head(arena_chunk_map_misc_t) arena_chunk_miscelms_t; 148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Arena chunk header. */ 150e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_chunk_s { 151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Arena that owns the chunk. */ 152e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_t *arena; 153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1547393f44ff025ca67716fc53b68003fd65122fd97Jason Evans /* 1557393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * Map of pages within chunk that keeps track of free/large/small. The 1567393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * first map_bias entries are omitted, since the chunk header does not 1577393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * need to be tracked in the map. This omission saves a header page 1587393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * for common chunk sizes (e.g. 4 MiB). 1597393f44ff025ca67716fc53b68003fd65122fd97Jason Evans */ 160ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu arena_chunk_map_bits_t map_bits[1]; /* Dynamically sized. */ 161e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 162e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 16349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 16484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Read-only information associated with each element of arena_t's bins array 16549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * is stored separately, partly to reduce memory usage (only one copy, rather 16649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * than one per arena), but mainly to avoid false cacheline sharing. 167122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 168122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Each run has the following layout: 169122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 170122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * /--------------------\ 1710c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans * | pad? | 172122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 173122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 174122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * reg0_offset | region 0 | 175122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 176122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| \ 177122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | | 178122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | region 1 | > reg_interval 179122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | / 180122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 181122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 182122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 183122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 184122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 185122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 186122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | region nregs-1 | 187122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 188122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 189122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | alignment pad? | 190122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * \--------------------/ 191122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 192122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * reg_interval has at least the same minimum alignment as reg_size; this 193122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * preserves the alignment constraint that sa2u() depends on. Alignment pad is 194122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * either 0 or redzone_size; it is present only if needed to align reg0_offset. 19549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 19649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstruct arena_bin_info_s { 19749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Size of regions in a run for this bin's size class. */ 19849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t reg_size; 19949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 200122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Redzone size. */ 201122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t redzone_size; 202122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 203122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Interval between regions (reg_size + (redzone_size << 1)). */ 204122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t reg_interval; 205122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 20649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total size of a run for this bin's size class. */ 20749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t run_size; 20849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 20949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total number of regions in a run for this bin's size class. */ 21049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t nregs; 21149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 21284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 21384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Metadata used to manipulate bitmaps for runs associated with this 21484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin. 21584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 21684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_info_t bitmap_info; 21784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 21849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Offset of first region in a run for this bin's size class. */ 21949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t reg0_offset; 22049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans}; 22149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 222e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_bin_s { 223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 22486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * All operations on runcur, runs, and stats require that lock be 22586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. Run allocation/deallocation are protected by the arena lock, 22686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * which may be acquired while holding one or more bin locks, but not 22786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * vise versa. 22886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 22986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_t lock; 23086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 23186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current run being used to service allocations of this bin's size 233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * class. 234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 235e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *runcur; 236e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 238e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Tree of non-full runs. This tree is used when looking for an 239e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * existing run when runcur is no longer usable. We choose the 240e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * non-full run that is lowest in memory; this policy tends to keep 241e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * objects packed well, and it can also help reduce the number of 242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * almost-empty chunks. 243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_t runs; 245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Bin statistics. */ 247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_bin_stats_t stats; 248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 250e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_s { 2516109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* This arena's index within the arenas array. */ 2526109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned ind; 2536109fe07a14b7a619365977d9523db9f8b333792Jason Evans 25486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 255597632be188d2bcc135dad2145cc46ef44897aadJason Evans * Number of threads currently assigned to this arena. This field is 256597632be188d2bcc135dad2145cc46ef44897aadJason Evans * protected by arenas_lock. 257597632be188d2bcc135dad2145cc46ef44897aadJason Evans */ 258597632be188d2bcc135dad2145cc46ef44897aadJason Evans unsigned nthreads; 259597632be188d2bcc135dad2145cc46ef44897aadJason Evans 260597632be188d2bcc135dad2145cc46ef44897aadJason Evans /* 261597632be188d2bcc135dad2145cc46ef44897aadJason Evans * There are three classes of arena operations from a locking 262597632be188d2bcc135dad2145cc46ef44897aadJason Evans * perspective: 263597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 1) Thread asssignment (modifies nthreads) is protected by 264597632be188d2bcc135dad2145cc46ef44897aadJason Evans * arenas_lock. 265597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 2) Bin-related operations are protected by bin locks. 266597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 3) Chunk- and run-related operations are protected by this mutex. 26786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_t lock; 269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 270e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_stats_t stats; 271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 272e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * List of tcaches for extant threads associated with this arena. 273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Stats from these are merged incrementally, and at exit. 274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_head(tcache_t) tcache_ql; 276e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 277d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans uint64_t prof_accumbytes; 278d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 279609ae595f0358157b19311b0f9f9591db7cee705Jason Evans dss_prec_t dss_prec; 280609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * In order to avoid rapid chunk allocation/deallocation when an arena 283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * oscillates right on the cusp of needing a new chunk, cache the most 284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * recently freed chunk. The spare is left in the arena's chunk trees 285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * until it is deleted. 286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 287e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * There is one spare chunk per arena, rather than one spare total, in 288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * order to avoid interactions between multiple threads that could make 289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a single spare inadequate. 290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *spare; 292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 293e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evans /* Number of pages in active runs and huge regions. */ 294bc25a47ee0e2ac8e10a94d5fa070f0dbbdeb7e7eJason Evans size_t nactive; 295e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current count of pages within unused runs that are potentially 298e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * dirty, and for which madvise(... MADV_DONTNEED) has not been called. 299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * By tracking this, we can institute a limit on how much dirty unused 300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * memory is mapped for each arena. 301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 302e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty; 303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 30519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Size/address-ordered trees of this arena's available runs. The trees 306e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans * are used for first-best-fit run allocation. 307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 308e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_t runs_avail; 309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 310070b3c3fbd90296610005c111ec6060e8bb23d31Jason Evans /* List of dirty runs this arena manages. */ 311ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu arena_chunk_miscelms_t runs_dirty; 312070b3c3fbd90296610005c111ec6060e8bb23d31Jason Evans 313fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind /* 314fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind * user-configureable chunk allocation and deallocation functions. 315fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind */ 316fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind chunk_alloc_t *chunk_alloc; 317e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evans chunk_dalloc_t *chunk_dalloc; 318fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind 319b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans /* bins is used to store trees of free regions. */ 320b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans arena_bin_t bins[NBINS]; 321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_STRUCTS */ 324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_EXTERNS 326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 32784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evansextern ssize_t opt_lg_dirty_mult; 32841ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans/* 3293541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans * small_size2bin_tab is a compact lookup table that rounds request sizes up to 33041ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans * size classes. In order to reduce cache footprint, the table is compressed, 3313541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans * and all accesses are via small_size2bin(). 33241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans */ 3333541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evansextern uint8_t const small_size2bin_tab[]; 3343541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans/* 3353541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans * small_bin2size_tab duplicates information in arena_bin_info, but in a const 3363541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans * array, for which it is easier for the compiler to optimize repeated 3373541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans * dereferences. 3383541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans */ 3393541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evansextern uint32_t const small_bin2size_tab[NBINS]; 340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 341b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansextern arena_bin_info_t arena_bin_info[NBINS]; 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 343b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans/* Number of large size classes. */ 3447393f44ff025ca67716fc53b68003fd65122fd97Jason Evans#define nlclasses (chunk_npages - map_bias) 3453c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans 346e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evansvoid *arena_chunk_alloc_huge(arena_t *arena, size_t size, size_t alignment, 347e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evans bool *zero); 348e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evansvoid arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t size); 3496005f0710cf07d60659d91b20b7ff5592d310027Jason Evansvoid arena_purge_all(arena_t *arena); 350dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, 3517372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans size_t binind, uint64_t prof_accumbytes); 352122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, 353122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bool zero); 3540d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#ifdef JEMALLOC_JET 3550d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evanstypedef void (arena_redzone_corruption_t)(void *, size_t, bool, size_t, 3560d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans uint8_t); 3576b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_redzone_corruption_t *arena_redzone_corruption; 3586b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_dalloc_junk_small_t)(void *, arena_bin_info_t *); 3596b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_dalloc_junk_small_t *arena_dalloc_junk_small; 3606b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#else 361122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); 3626b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 3630d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansvoid arena_quarantine_junk_small(void *ptr, size_t usize); 364e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid *arena_malloc_small(arena_t *arena, size_t size, bool zero); 365dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid *arena_malloc_large(arena_t *arena, size_t size, bool zero); 3665ff709c264e52651de25b788692c62ff1f6f389cJason Evansvoid *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); 3670b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansvoid arena_prof_promoted(const void *ptr, size_t size); 368203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 369ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu arena_chunk_map_bits_t *bitselm); 370203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 371ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind, arena_chunk_map_bits_t *bitselm); 372203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 373203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind); 3746b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 3756b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_dalloc_junk_large_t)(void *, size_t); 3766b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_dalloc_junk_large_t *arena_dalloc_junk_large; 3776b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 378203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, 379203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans void *ptr); 380e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); 3816b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 3826b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t); 3836b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_ralloc_junk_large_t *arena_ralloc_junk_large; 3846b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 385b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evansbool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, 3868e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t extra, bool zero); 3875460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, 3885460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans size_t size, size_t extra, size_t alignment, bool zero, 3895460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans bool try_tcache_alloc, bool try_tcache_dalloc); 390609ae595f0358157b19311b0f9f9591db7cee705Jason Evansdss_prec_t arena_dss_prec_get(arena_t *arena); 3914d434adb146375ad17f0d5e994ed5728d2942e3fJason Evansbool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); 392609ae595f0358157b19311b0f9f9591db7cee705Jason Evansvoid arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, 393609ae595f0358157b19311b0f9f9591db7cee705Jason Evans size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 394609ae595f0358157b19311b0f9f9591db7cee705Jason Evans malloc_large_stats_t *lstats); 395e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool arena_new(arena_t *arena, unsigned ind); 396b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansvoid arena_boot(void); 3974e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_prefork(arena_t *arena); 3984e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_postfork_parent(arena_t *arena); 3994e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_postfork_child(arena_t *arena); 400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_EXTERNS */ 402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 403e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_INLINES 404e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifndef JEMALLOC_ENABLE_INLINE 406d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_size2bin_compute(size_t size); 407d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_size2bin_lookup(size_t size); 4083541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evanssize_t small_size2bin(size_t size); 409d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_bin2size_compute(size_t binind); 410d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_bin2size_lookup(size_t binind); 4113541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evanssize_t small_bin2size(size_t binind); 412d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_s2u_compute(size_t size); 413d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_s2u_lookup(size_t size); 414d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssize_t small_s2u(size_t size); 415ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_chunk_map_bits_t *arena_bitselm_get(arena_chunk_t *chunk, 416ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind); 417ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_chunk_map_misc_t *arena_miscelm_get(arena_chunk_t *chunk, 418ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind); 4190c5dd03e889d0269170b5db9fa872738d906eb78Jason Evanssize_t arena_miscelm_to_pageind(arena_chunk_map_misc_t *miscelm); 4200c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansvoid *arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm); 4210c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_chunk_map_misc_t *arena_run_to_miscelm(arena_run_t *run); 422203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); 42387a02d2bb18dbcb2955541b849bc95862e864803Jason Evanssize_t arena_mapbitsp_read(size_t *mapbitsp); 424203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); 425203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, 426203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind); 427203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); 428203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); 42980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evanssize_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); 430203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); 431203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); 432203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); 433203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind); 43487a02d2bb18dbcb2955541b849bc95862e864803Jason Evansvoid arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits); 435203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, 436203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size, size_t flags); 437203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 438203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size); 439203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, 440203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size, size_t flags); 441203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 442203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind); 443203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, 444203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t runind, size_t binind, size_t flags); 445203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, 446203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t unzeroed); 44788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); 44888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); 44988c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum(arena_t *arena, uint64_t accumbytes); 45080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evanssize_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); 45149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evanssize_t arena_bin_index(arena_t *arena, arena_bin_t *bin); 45249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansunsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, 453b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans const void *ptr); 454602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansprof_tctx_t *arena_prof_tctx_get(const void *ptr); 455602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansvoid arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); 4565460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, 4575460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans bool try_tcache); 458f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evanssize_t arena_salloc(const void *ptr, bool demote); 4595460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, 4605460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans bool try_tcache); 4615460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, 4629c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans bool try_tcache); 463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) 466203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans# ifdef JEMALLOC_ARENA_INLINE_A 467d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_INLINE size_t 468d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_size2bin_compute(size_t size) 469d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 470d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#if (NTBINS != 0) 471d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { 472d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; 473d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_ceil = lg_floor(pow2_ceil(size)); 474d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); 475d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } else 476d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#endif 477d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans { 478d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t x = lg_floor((size<<1)-1); 479d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : 480d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); 481d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t grp = shift << LG_SIZE_CLASS_GROUP; 482d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 483d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) 484d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; 485d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 486d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t delta_inverse_mask = ZI(-1) << lg_delta; 487d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & 488d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); 489d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 490d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t bin = NTBINS + grp + mod; 491d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (bin); 492d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } 493d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 494d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 495d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_ALWAYS_INLINE size_t 496d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_size2bin_lookup(size_t size) 497d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 498d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 499d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(size <= LOOKUP_MAXCLASS); 500d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans { 501d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t ret = ((size_t)(small_size2bin_tab[(size-1) >> 502d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans LG_TINY_MIN])); 503d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(ret == small_size2bin_compute(size)); 504d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (ret); 505d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } 506d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 507d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 5083541a904d6fb949f3f0aea05418ccce7cbd4b705Jason EvansJEMALLOC_ALWAYS_INLINE size_t 5093541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evanssmall_size2bin(size_t size) 5103541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans{ 5113541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans 512d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(size > 0); 5139c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(size <= LOOKUP_MAXCLASS)) 514d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (small_size2bin_lookup(size)); 515d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans else 516d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (small_size2bin_compute(size)); 517d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 518d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 519d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_INLINE size_t 520d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_bin2size_compute(size_t binind) 521d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 522d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#if (NTBINS > 0) 523d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans if (binind < NTBINS) 524d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + binind)); 525d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans else 526d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#endif 527d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans { 528d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t reduced_binind = binind - NTBINS; 529d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t grp = reduced_binind >> LG_SIZE_CLASS_GROUP; 530d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t mod = reduced_binind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 531d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 1); 532d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 533d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t grp_size_mask = ~((!!grp)-1); 534d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t grp_size = ((ZU(1) << (LG_QUANTUM + 535d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; 536d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 537d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t shift = (grp == 0) ? 1 : grp; 538d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_delta = shift + (LG_QUANTUM-1); 539d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t mod_size = (mod+1) << lg_delta; 540d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 541d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t usize = grp_size + mod_size; 542d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (usize); 543d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } 544d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 545d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 546d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_ALWAYS_INLINE size_t 547d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_bin2size_lookup(size_t binind) 548d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 549d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 550d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(binind < NBINS); 551d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans { 552b718cf77e9917f6ae1995c2e2b219ff4219c9f46Jason Evans size_t ret = (size_t)small_bin2size_tab[binind]; 553d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(ret == small_bin2size_compute(binind)); 554d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (ret); 555d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } 5563541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans} 5573541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans 5583541a904d6fb949f3f0aea05418ccce7cbd4b705Jason EvansJEMALLOC_ALWAYS_INLINE size_t 5593541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evanssmall_bin2size(size_t binind) 5603541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans{ 5613541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans 562d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (small_bin2size_lookup(binind)); 563d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 564d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 565d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_ALWAYS_INLINE size_t 566d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_s2u_compute(size_t size) 567d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 568d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#if (NTBINS > 0) 569d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { 570d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; 571d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_ceil = lg_floor(pow2_ceil(size)); 572d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : 573d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans (ZU(1) << lg_ceil)); 574d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } else 575d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans#endif 576d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans { 577d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t x = lg_floor((size<<1)-1); 578d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) 579d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; 580d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t delta = ZU(1) << lg_delta; 581d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t delta_mask = delta - 1; 582d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans size_t usize = (size + delta_mask) & ~delta_mask; 583d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (usize); 584d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans } 585d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 586d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 587d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_ALWAYS_INLINE size_t 588d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_s2u_lookup(size_t size) 589d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 590b718cf77e9917f6ae1995c2e2b219ff4219c9f46Jason Evans size_t ret = small_bin2size(small_size2bin(size)); 591d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 592d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(ret == small_s2u_compute(size)); 593d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (ret); 594d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans} 595d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 596d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason EvansJEMALLOC_ALWAYS_INLINE size_t 597d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evanssmall_s2u(size_t size) 598d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans{ 599d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans 600d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans assert(size > 0); 6019c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(size <= LOOKUP_MAXCLASS)) 602d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (small_s2u_lookup(size)); 603d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans else 604d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0Jason Evans return (small_s2u_compute(size)); 6053541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans} 6063541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans# endif /* JEMALLOC_ARENA_INLINE_A */ 6073541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans 6083541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans# ifdef JEMALLOC_ARENA_INLINE_B 609ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan WuJEMALLOC_ALWAYS_INLINE arena_chunk_map_bits_t * 610ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_bitselm_get(arena_chunk_t *chunk, size_t pageind) 611ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu{ 612ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 613ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu assert(pageind >= map_bias); 614ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu assert(pageind < chunk_npages); 615ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 616ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return (&chunk->map_bits[pageind-map_bias]); 617ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu} 618ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 619ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan WuJEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 620ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_miscelm_get(arena_chunk_t *chunk, size_t pageind) 621203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 622203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 623203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind >= map_bias); 624203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind < chunk_npages); 625203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 626ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return ((arena_chunk_map_misc_t *)((uintptr_t)chunk + 627ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu (uintptr_t)map_misc_offset) + pageind-map_bias); 628203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 629203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 6300c5dd03e889d0269170b5db9fa872738d906eb78Jason EvansJEMALLOC_ALWAYS_INLINE size_t 6310c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_miscelm_to_pageind(arena_chunk_map_misc_t *miscelm) 6320c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans{ 6330c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 6340c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans size_t pageind = ((uintptr_t)miscelm - ((uintptr_t)chunk + 6350c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans map_misc_offset)) / sizeof(arena_chunk_map_misc_t) + map_bias; 6360c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6370c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(pageind >= map_bias); 6380c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(pageind < chunk_npages); 6390c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6400c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans return (pageind); 6410c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans} 6420c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6430c5dd03e889d0269170b5db9fa872738d906eb78Jason EvansJEMALLOC_ALWAYS_INLINE void * 6440c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm) 6450c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans{ 6460c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 6470c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans size_t pageind = arena_miscelm_to_pageind(miscelm); 6480c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6490c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE))); 6500c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans} 6510c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6520c5dd03e889d0269170b5db9fa872738d906eb78Jason EvansJEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 6530c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_run_to_miscelm(arena_run_t *run) 6540c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans{ 6550c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t 6560c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans *)((uintptr_t)run - offsetof(arena_chunk_map_misc_t, run)); 6570c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6580c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(arena_miscelm_to_pageind(miscelm) >= map_bias); 6590c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); 6600c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6610c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans return (miscelm); 6620c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans} 6630c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 66488393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t * 665203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) 666203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 667203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 668ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return (&arena_bitselm_get(chunk, pageind)->bits); 669203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 670203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 67188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 67287a02d2bb18dbcb2955541b849bc95862e864803Jason Evansarena_mapbitsp_read(size_t *mapbitsp) 67387a02d2bb18dbcb2955541b849bc95862e864803Jason Evans{ 67487a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 67587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans return (*mapbitsp); 67687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans} 67787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 67887a02d2bb18dbcb2955541b849bc95862e864803Jason EvansJEMALLOC_ALWAYS_INLINE size_t 679203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_get(arena_chunk_t *chunk, size_t pageind) 680203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 681203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 68287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans return (arena_mapbitsp_read(arena_mapbitsp_get(chunk, pageind))); 683203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 684203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 68588393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 686203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) 687203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 688203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 689203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 690203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 691203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 692203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & ~PAGE_MASK); 693203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 694203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 69588393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 696203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) 697203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 698203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 699203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 700203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 701203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 702203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); 703203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & ~PAGE_MASK); 704203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 705203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 70688393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 707203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) 708203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 709203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 710203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 711203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 712203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 713203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans CHUNK_MAP_ALLOCATED); 714203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits >> LG_PAGE); 715203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 716203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 71788393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 71880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evansarena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) 71980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans{ 72080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t mapbits; 72180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t binind; 72280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 72380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans mapbits = arena_mapbits_get(chunk, pageind); 72480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 72580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind < NBINS || binind == BININD_INVALID); 72680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans return (binind); 72780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans} 72880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 72988393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 730203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) 731203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 732203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 733203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 734203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 735203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_DIRTY); 736203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 737203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 73888393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 739203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind) 740203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 741203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 742203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 743203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 744203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_UNZEROED); 745203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 746203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 74788393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 748203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind) 749203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 750203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 751203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 752203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 753203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_LARGE); 754203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 755203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 75688393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 757203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind) 758203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 759203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 760203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 761203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 762203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_ALLOCATED); 763203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 764203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 76588393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 76687a02d2bb18dbcb2955541b849bc95862e864803Jason Evansarena_mapbitsp_write(size_t *mapbitsp, size_t mapbits) 76787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans{ 76887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 76987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans *mapbitsp = mapbits; 77087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans} 77187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 77287a02d2bb18dbcb2955541b849bc95862e864803Jason EvansJEMALLOC_ALWAYS_INLINE void 773203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, 774203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flags) 775203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 77687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 777203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 778203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((size & PAGE_MASK) == 0); 779203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0); 780d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans assert((flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == flags); 78187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags); 782203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 783203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 78488393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 785203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 786203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size) 787203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 78887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 78987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 790203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 791203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((size & PAGE_MASK) == 0); 79287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 79387a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, size | (mapbits & PAGE_MASK)); 794203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 795203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 79688393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 797203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, 798203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flags) 799203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 80087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 80187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 802d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans size_t unzeroed; 803203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 804203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((size & PAGE_MASK) == 0); 805d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans assert((flags & CHUNK_MAP_DIRTY) == flags); 80687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ 80787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags 80887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans | unzeroed | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED); 809203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 810203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 81188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 812203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 813203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind) 814203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 81587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 81687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 817203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 818203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind <= BININD_INVALID); 819203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_large_size_get(chunk, pageind) == PAGE); 82087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) | 82187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans (binind << CHUNK_MAP_BININD_SHIFT)); 822203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 823203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 82488393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 825203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, 826203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind, size_t flags) 827203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 82887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 82987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 830d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans size_t unzeroed; 831203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 832203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind < BININD_INVALID); 833203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind - runind >= map_bias); 834d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans assert((flags & CHUNK_MAP_DIRTY) == flags); 83587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ 83687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, (runind << LG_PAGE) | (binind << 83787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans CHUNK_MAP_BININD_SHIFT) | flags | unzeroed | CHUNK_MAP_ALLOCATED); 838203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 839203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 84088393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 841203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, 842203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t unzeroed) 843203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 84487a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 84587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 846203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 84787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_UNZEROED) | 84887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans unzeroed); 849203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 850203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 85188c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 852a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) 853a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 854a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 855a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 856a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans assert(prof_interval != 0); 857a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 858a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena->prof_accumbytes += accumbytes; 859a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans if (arena->prof_accumbytes >= prof_interval) { 860a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena->prof_accumbytes -= prof_interval; 86188c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (true); 862a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans } 86388c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 864a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 865a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 86688c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 867a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum_locked(arena_t *arena, uint64_t accumbytes) 868a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 869a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 870a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 871a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 8729c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(prof_interval == 0)) 87388c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 87488c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (arena_prof_accum_impl(arena, accumbytes)); 875a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 876a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 87788c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 878a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum(arena_t *arena, uint64_t accumbytes) 879a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 880a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 881a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 882a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 8839c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(prof_interval == 0)) 88488c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 88588c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans 88688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans { 88788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans bool ret; 88888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans 88988c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans malloc_mutex_lock(&arena->lock); 89088c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans ret = arena_prof_accum_impl(arena, accumbytes); 89188c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans malloc_mutex_unlock(&arena->lock); 89288c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (ret); 89388c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans } 894a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 895a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 89688393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 89780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evansarena_ptr_small_binind_get(const void *ptr, size_t mapbits) 898203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 899203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t binind; 900203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 901203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 902203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 903203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (config_debug) { 90480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_chunk_t *chunk; 90580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_t *arena; 90680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t pageind; 90780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t actual_mapbits; 9080c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans size_t rpages_ind; 90980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_run_t *run; 91080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_bin_t *bin; 91180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t actual_binind; 91280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_bin_info_t *bin_info; 9130c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_map_misc_t *miscelm; 9140c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans void *rpages; 91580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 91680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind != BININD_INVALID); 91780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind < NBINS); 91880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 91980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena = chunk->arena; 92080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 92180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans actual_mapbits = arena_mapbits_get(chunk, pageind); 922203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(mapbits == actual_mapbits); 92380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_large_get(chunk, pageind) == 0); 92480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 9250c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, 9260c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans pageind); 9270c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans miscelm = arena_miscelm_get(chunk, rpages_ind); 9280c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans run = &miscelm->run; 92980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans bin = run->bin; 93080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans actual_binind = bin - arena->bins; 931203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind == actual_binind); 93280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans bin_info = &arena_bin_info[actual_binind]; 9330c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans rpages = arena_miscelm_to_rpages(miscelm); 9340c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(((uintptr_t)ptr - ((uintptr_t)rpages + 935203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval 936203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans == 0); 937203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 938203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 939203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (binind); 940203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 9413541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans# endif /* JEMALLOC_ARENA_INLINE_B */ 942203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 9433541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans# ifdef JEMALLOC_ARENA_INLINE_C 94449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason EvansJEMALLOC_INLINE size_t 94549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_bin_index(arena_t *arena, arena_bin_t *bin) 94649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 94749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = bin - arena->bins; 948b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 94949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (binind); 95049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 95149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 95281b4e6eb6f06cac048e3743787a70676f1534269Jason EvansJEMALLOC_INLINE unsigned 953b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evansarena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) 95481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 95581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans unsigned shift, diff, regind; 956122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t interval; 9570c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 9580c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans void *rpages = arena_miscelm_to_rpages(miscelm); 95981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 96084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 96184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Freeing a pointer lower than region zero can cause assertion 96284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * failure. 96384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 9640c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert((uintptr_t)ptr >= (uintptr_t)rpages + 96584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset); 96681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 96781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 96881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * Avoid doing division with a variable divisor if possible. Using 96981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * actual division here can reduce allocator throughput by over 20%! 97081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 9710c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans diff = (unsigned)((uintptr_t)ptr - (uintptr_t)rpages - 97249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg0_offset); 97381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 97481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* Rescale (factor powers of 2 out of the numerator and denominator). */ 975122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans interval = bin_info->reg_interval; 9769c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond shift = jemalloc_ffs(interval) - 1; 97781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans diff >>= shift; 978122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans interval >>= shift; 97981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 980122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (interval == 1) { 98181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* The divisor was a power of 2. */ 98281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans regind = diff; 98381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } else { 98481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 98581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * To divide by a number D that is not a power of two we 98681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * multiply by (2^21 / D) and then right shift by 21 positions. 98781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 98881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * X / D 98981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 99081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * becomes 99181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 992122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT 99381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 99481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * We can omit the first three elements, because we never 99581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * divide by 0, and 1 and 2 are both powers of two, which are 99681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * handled above. 99781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 99847e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) 99947e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) 1000122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans static const unsigned interval_invs[] = { 100181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(3), 100281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 100381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 100481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 100581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 100681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 100781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 100881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 100981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans }; 101081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 10119c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(interval <= ((sizeof(interval_invs) / 10129c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans sizeof(unsigned)) + 2))) { 1013122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans regind = (diff * interval_invs[interval - 3]) >> 1014122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans SIZE_INV_SHIFT; 1015122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else 1016122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans regind = diff / interval; 101781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV 101881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV_SHIFT 101981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } 1020122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(diff == regind * interval); 102149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(regind < bin_info->nregs); 102281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 102381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (regind); 102481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 102581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 1026602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason EvansJEMALLOC_INLINE prof_tctx_t * 1027602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansarena_prof_tctx_get(const void *ptr) 102881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 1029602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *ret; 103081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans arena_chunk_t *chunk; 103181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans size_t pageind, mapbits; 103281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 10337372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 103481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert(ptr != NULL); 103581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 103681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 103781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1038ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1039203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 104081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 10419c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) 1042602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans ret = (prof_tctx_t *)(uintptr_t)1U; 10439b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evans else 1044ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu ret = arena_miscelm_get(chunk, pageind)->prof_tctx; 104581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 104681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (ret); 104781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 1048e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 1049e4f7846f1fd279a039ffa2a41707348187219de4Jason EvansJEMALLOC_INLINE void 1050602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansarena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) 1051e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans{ 1052e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans arena_chunk_t *chunk; 10535fbad0902b845b1a6b311994468d0b9962e4fd30Jason Evans size_t pageind; 1054e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 10557372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 1056e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert(ptr != NULL); 1057e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1058e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 1059e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1060ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 10615fbad0902b845b1a6b311994468d0b9962e4fd30Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1062665769357cd77b74e00a146f196fff19243b33c4Jason Evans 10639c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (unlikely(arena_mapbits_large_get(chunk, pageind) != 0)) 1064ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu arena_miscelm_get(chunk, pageind)->prof_tctx = tctx; 1065e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans} 106681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 106788393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void * 10685460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansarena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, 10695460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans bool try_tcache) 1070962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans{ 1071ef8897b4b938111fcc9b54725067f1dbb33a4c20Jason Evans tcache_t *tcache; 1072962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1073962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans assert(size != 0); 1074cd9a1346e96f71bdecdc654ea50fc62d76371e74Jason Evans assert(size <= arena_maxclass); 1075962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 10769c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(size <= SMALL_MAXCLASS)) { 10775460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans if (likely(try_tcache) && likely((tcache = tcache_get(tsd, 10785460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans true)) != NULL)) 1079962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans return (tcache_alloc_small(tcache, size, zero)); 108001b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans else { 10815460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans return (arena_malloc_small(choose_arena(tsd, arena), 10825460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans size, zero)); 108301b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans } 1084962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans } else { 1085746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans /* 1086746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans * Initialize tcache after checking size in order to avoid 1087746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans * infinite recursion during tcache initialization. 1088746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans */ 10899c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (try_tcache && size <= tcache_maxclass && likely((tcache = 10905460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans tcache_get(tsd, true)) != NULL)) 1091962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans return (tcache_alloc_large(tcache, size, zero)); 109201b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans else { 10935460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans return (arena_malloc_large(choose_arena(tsd, arena), 10945460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans size, zero)); 109501b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans } 1096962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans } 1097962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans} 1098962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1099f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans/* Return the size of the allocation pointed to by ptr. */ 110088393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 1101f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evansarena_salloc(const void *ptr, bool demote) 1102f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans{ 1103f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans size_t ret; 1104f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans arena_chunk_t *chunk; 110580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t pageind, binind; 1106f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1107f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(ptr != NULL); 1108f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1109f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1110f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1111f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1112203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 111380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = arena_mapbits_binind_get(chunk, pageind); 11149c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (unlikely(binind == BININD_INVALID || (config_prof && demote == false 11159c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans && arena_mapbits_large_get(chunk, pageind) != 0))) { 111680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans /* 111780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * Large allocation. In the common case (demote == true), and 111880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * as this is an inline function, most callers will only end up 111980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * looking at binind to determine that ptr is a small 112080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans * allocation. 112180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans */ 1122f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 1123203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans ret = arena_mapbits_large_size_get(chunk, pageind); 1124f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(ret != 0); 112580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(pageind + (ret>>LG_PAGE) <= chunk_npages); 112680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(ret == PAGE || arena_mapbits_large_size_get(chunk, 112780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind+(ret>>LG_PAGE)-1) == 0); 112880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind == arena_mapbits_binind_get(chunk, 112980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind+(ret>>LG_PAGE)-1)); 113080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_dirty_get(chunk, pageind) == 113180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1)); 113280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans } else { 11339b0cbf0850b130a9b0a8c58bd10b2926b2083510Jason Evans /* Small allocation (possibly promoted to a large object). */ 113480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_large_get(chunk, pageind) != 0 || 113580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 113680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind)) == binind); 11373541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans ret = small_bin2size(binind); 1138f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans } 1139f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1140f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans return (ret); 1141f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans} 1142f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 114388393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 11445460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansarena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache) 1145e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1146203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind, mapbits; 114701b3fe55ff3ac8e4aa689f09fcb0729da8037638Jason Evans tcache_t *tcache; 1148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1150e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 1151e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1152ae4c7b4b4092906c641d69b4bf9fcb4a7d50790dJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1153203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 1154203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 11559c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { 1156e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Small allocation. */ 11575460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans if (likely(try_tcache) && likely((tcache = tcache_get(tsd, 11585460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans false)) != NULL)) { 11599c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans size_t binind = arena_ptr_small_binind_get(ptr, 11609c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans mapbits); 1161203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans tcache_dalloc_small(tcache, ptr, binind); 1162203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } else 1163be8e59f5a64ef775c9694aee0d6a87d92336d303Ben Maurer arena_dalloc_small(chunk->arena, chunk, ptr, pageind); 1164f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans } else { 1165203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size = arena_mapbits_large_size_get(chunk, pageind); 1166962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1167962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 1168962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 11699c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (try_tcache && size <= tcache_maxclass && likely((tcache = 11705460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans tcache_get(tsd, false)) != NULL)) { 1171962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans tcache_dalloc_large(tcache, ptr, size); 1172203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } else 1173be8e59f5a64ef775c9694aee0d6a87d92336d303Ben Maurer arena_dalloc_large(chunk->arena, chunk, ptr); 1174f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans } 1175e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 11764cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 11774cfe55166e0173be745c53adb0fecf50d11d1227Daniel MicayJEMALLOC_ALWAYS_INLINE void 11785460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansarena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, 11795460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans bool try_tcache) 11804cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay{ 11814cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay tcache_t *tcache; 11824cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 11834cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay assert(ptr != NULL); 11844cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay assert(CHUNK_ADDR2BASE(ptr) != ptr); 11854cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 11869c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(size <= SMALL_MAXCLASS)) { 11874cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay /* Small allocation. */ 11885460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans if (likely(try_tcache) && likely((tcache = tcache_get(tsd, 11895460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans false)) != NULL)) { 11904cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay size_t binind = small_size2bin(size); 11914cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay tcache_dalloc_small(tcache, ptr, binind); 11924cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay } else { 11939c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 11949c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans LG_PAGE; 11954cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay arena_dalloc_small(chunk->arena, chunk, ptr, pageind); 11964cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay } 11974cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay } else { 11984cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay assert(((uintptr_t)ptr & PAGE_MASK) == 0); 11994cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 12004cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay if (try_tcache && size <= tcache_maxclass && (tcache = 12015460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evans tcache_get(tsd, false)) != NULL) { 12024cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay tcache_dalloc_large(tcache, ptr, size); 12034cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay } else 12044cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay arena_dalloc_large(chunk->arena, chunk, ptr); 12054cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay } 12064cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay} 12073541a904d6fb949f3f0aea05418ccce7cbd4b705Jason Evans# endif /* JEMALLOC_ARENA_INLINE_C */ 1208e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1209e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_INLINES */ 1211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 1212