arena.h revision 8a03cf039cd06f9fa6972711195055d865673966
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 2e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_TYPES 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans#define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) 5155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans 647e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans/* Maximum number of regions in one run. */ 70c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans#define LG_RUN_MAXREGS (LG_PAGE - LG_TINY_MIN) 847e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) 947e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 11122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Minimum redzone size. Redzones may be larger than this if necessary to 12122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * preserve region alignment. 13122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans */ 14122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans#define REDZONE_MINSIZE 16 15122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 16122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans/* 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The minimum ratio of active:dirty pages per arena is computed as: 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 198d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans * (nactive >> lg_dirty_mult) >= ndirty 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 218d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans * So, supposing that lg_dirty_mult is 3, there can be no less than 8 times as 228d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans * many active pages as dirty pages. 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 24e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans#define LG_DIRTY_MULT_DEFAULT 3 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2638e42d311c1844a66e8ced84551621de41e42b85Jason Evanstypedef struct arena_runs_dirty_link_s arena_runs_dirty_link_t; 270c5dd03e889d0269170b5db9fa872738d906eb78Jason Evanstypedef struct arena_run_s arena_run_t; 28ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; 29ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_chunk_s arena_chunk_t; 3149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evanstypedef struct arena_bin_info_s arena_bin_info_t; 32e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_bin_s arena_bin_t; 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_s arena_t; 34e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 35e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_TYPES */ 36e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 37e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_STRUCTS 38e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 39ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans#ifdef JEMALLOC_ARENA_STRUCTS_A 400c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansstruct arena_run_s { 41381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans /* Index of bin this run is associated with. */ 42381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans index_t binind; 430c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 440c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Number of free regions in run. */ 450c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans unsigned nfree; 460c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 470c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Per region allocated/deallocated bitmap. */ 480c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans bitmap_t bitmap[BITMAP_GROUPS_MAX]; 490c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans}; 500c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Each element of the chunk map corresponds to one page within the chunk. */ 52ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wustruct arena_chunk_map_bits_s { 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Run address (or size) and various flags are stored together. The bit 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * layout looks like (assuming 32-bit system): 56e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 5753bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ???????? ???????? ????nnnn nnnndula 58e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 59e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * ? : Unallocated: Run address for first/last pages, unset for internal 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * pages. 6119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Small: Run page offset. 62e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: Run size for first page, unset for trailing pages. 6353bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * n : binind for small size class, BININD_INVALID for large size class. 64e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * d : dirty? 658ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * u : unzeroed? 66e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * l : large? 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a : allocated? 68e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Following are example bit patterns for the three types of runs. 70e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 71e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * p : run page offset 72e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * s : run size 73203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * n : binind for size class; large objects set these to BININD_INVALID 74e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * x : don't care 75e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * - : 0 760b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * + : 1 773377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * [DULA] : bit set 783377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * [dula] : bit unset 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 8019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (clean): 8153bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++du-a 82203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxx-Uxx 8353bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++dU-a 8419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 8519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (dirty): 8653bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D--a 87203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 8853bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D--a 89e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 90dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Small: 91203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * pppppppp pppppppp ppppnnnn nnnnd--A 92203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * pppppppp pppppppp ppppnnnn nnnn---A 93203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * pppppppp pppppppp ppppnnnn nnnnd--A 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 95e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: 9653bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D-LA 97203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 9853bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * -------- -------- ----++++ ++++D-LA 990b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 100155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * Large (sampled, size <= LARGE_MINCLASS): 101203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * ssssssss ssssssss ssssnnnn nnnnD-LA 102155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 103155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * -------- -------- ----++++ ++++D-LA 1040b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 105155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * Large (not sampled, size == LARGE_MINCLASS): 10653bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * ssssssss ssssssss ssss++++ ++++D-LA 107155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 108155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * -------- -------- ----++++ ++++D-LA 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t bits; 111203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_SHIFT 4 112203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define BININD_INVALID ((size_t)0xffU) 113203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans/* CHUNK_MAP_BININD_MASK == (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) */ 114203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_MASK ((size_t)0xff0U) 115203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK 116203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_FLAGS_MASK ((size_t)0xcU) 1170b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_DIRTY ((size_t)0x8U) 1183377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans#define CHUNK_MAP_UNZEROED ((size_t)0x4U) 1190b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_LARGE ((size_t)0x2U) 1200b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_ALLOCATED ((size_t)0x1U) 1210b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED 122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 123ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 12438e42d311c1844a66e8ced84551621de41e42b85Jason Evansstruct arena_runs_dirty_link_s { 12538e42d311c1844a66e8ced84551621de41e42b85Jason Evans qr(arena_runs_dirty_link_t) rd_link; 12638e42d311c1844a66e8ced84551621de41e42b85Jason Evans}; 12738e42d311c1844a66e8ced84551621de41e42b85Jason Evans 128ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu/* 129ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just 130ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * like arena_chunk_map_bits_t. Two separate arrays are stored within each 131ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * chunk header in order to improve cache locality. 132ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu */ 133ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wustruct arena_chunk_map_misc_s { 134ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu /* 135ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * Linkage for run trees. There are two disjoint uses: 136ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 137ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 1) arena_t's runs_avail tree. 138ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 2) arena_run_t conceptually uses this linkage for in-use non-full 13938e42d311c1844a66e8ced84551621de41e42b85Jason Evans * runs, rather than directly embedding linkage. 140ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu */ 1410c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans rb_node(arena_chunk_map_misc_t) rb_link; 142ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 1430c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans union { 1440c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Linkage for list of dirty runs. */ 14538e42d311c1844a66e8ced84551621de41e42b85Jason Evans arena_runs_dirty_link_t rd; 146ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 1470c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Profile counters, used for large object runs. */ 1480c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans prof_tctx_t *prof_tctx; 1490c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 1500c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Small region run metadata. */ 1510c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_run_t run; 1520c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans }; 153ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu}; 154ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef rb_tree(arena_chunk_map_misc_t) arena_avail_tree_t; 155ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef rb_tree(arena_chunk_map_misc_t) arena_run_tree_t; 156ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans#endif /* JEMALLOC_ARENA_STRUCTS_A */ 157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 158ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans#ifdef JEMALLOC_ARENA_STRUCTS_B 159e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Arena chunk header. */ 160e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_chunk_s { 161cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* 162ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * A pointer to the arena that owns the chunk is stored within the node. 163ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * This field as a whole is used by chunks_rtree to support both 164ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * ivsalloc() and core-based debugging. 165cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans */ 166cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans extent_node_t node; 167e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1687393f44ff025ca67716fc53b68003fd65122fd97Jason Evans /* 1697393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * Map of pages within chunk that keeps track of free/large/small. The 1707393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * first map_bias entries are omitted, since the chunk header does not 1717393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * need to be tracked in the map. This omission saves a header page 1727393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * for common chunk sizes (e.g. 4 MiB). 1737393f44ff025ca67716fc53b68003fd65122fd97Jason Evans */ 174ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu arena_chunk_map_bits_t map_bits[1]; /* Dynamically sized. */ 175e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 17749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 17884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Read-only information associated with each element of arena_t's bins array 17949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * is stored separately, partly to reduce memory usage (only one copy, rather 18049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * than one per arena), but mainly to avoid false cacheline sharing. 181122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 182122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Each run has the following layout: 183122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 184122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * /--------------------\ 1850c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans * | pad? | 186122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 187122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 188122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * reg0_offset | region 0 | 189122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 190122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| \ 191122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | | 192122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | region 1 | > reg_interval 193122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | / 194122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 195122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 196122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 197122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 198122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 199122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 200122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | region nregs-1 | 201122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 202122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 203122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | alignment pad? | 204122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * \--------------------/ 205122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 206122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * reg_interval has at least the same minimum alignment as reg_size; this 207122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * preserves the alignment constraint that sa2u() depends on. Alignment pad is 208122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * either 0 or redzone_size; it is present only if needed to align reg0_offset. 20949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 21049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstruct arena_bin_info_s { 21149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Size of regions in a run for this bin's size class. */ 21249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t reg_size; 21349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 214122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Redzone size. */ 215122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t redzone_size; 216122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 217122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Interval between regions (reg_size + (redzone_size << 1)). */ 218122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t reg_interval; 219122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 22049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total size of a run for this bin's size class. */ 22149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t run_size; 22249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 22349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total number of regions in a run for this bin's size class. */ 22449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t nregs; 22549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 22684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 22784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Metadata used to manipulate bitmaps for runs associated with this 22884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin. 22984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 23084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_info_t bitmap_info; 23184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 23249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Offset of first region in a run for this bin's size class. */ 23349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t reg0_offset; 23449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans}; 23549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 236e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_bin_s { 237e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 23886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * All operations on runcur, runs, and stats require that lock be 23986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. Run allocation/deallocation are protected by the arena lock, 24086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * which may be acquired while holding one or more bin locks, but not 24186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * vise versa. 24286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 24386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_t lock; 24486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 24586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current run being used to service allocations of this bin's size 247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * class. 248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *runcur; 250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 251e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 252e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Tree of non-full runs. This tree is used when looking for an 253e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * existing run when runcur is no longer usable. We choose the 254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * non-full run that is lowest in memory; this policy tends to keep 255e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * objects packed well, and it can also help reduce the number of 256e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * almost-empty chunks. 257e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 258e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_t runs; 259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 260e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Bin statistics. */ 261e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_bin_stats_t stats; 262e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 263e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 264e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_s { 2656109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* This arena's index within the arenas array. */ 2666109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned ind; 2676109fe07a14b7a619365977d9523db9f8b333792Jason Evans 26886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 269597632be188d2bcc135dad2145cc46ef44897aadJason Evans * Number of threads currently assigned to this arena. This field is 270597632be188d2bcc135dad2145cc46ef44897aadJason Evans * protected by arenas_lock. 271597632be188d2bcc135dad2145cc46ef44897aadJason Evans */ 272597632be188d2bcc135dad2145cc46ef44897aadJason Evans unsigned nthreads; 273597632be188d2bcc135dad2145cc46ef44897aadJason Evans 274597632be188d2bcc135dad2145cc46ef44897aadJason Evans /* 275597632be188d2bcc135dad2145cc46ef44897aadJason Evans * There are three classes of arena operations from a locking 276597632be188d2bcc135dad2145cc46ef44897aadJason Evans * perspective: 277e12eaf93dca308a426c182956197b0eeb5f2cff3Jason Evans * 1) Thread assignment (modifies nthreads) is protected by arenas_lock. 278597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 2) Bin-related operations are protected by bin locks. 279597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 3) Chunk- and run-related operations are protected by this mutex. 28086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_t lock; 282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_stats_t stats; 284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * List of tcaches for extant threads associated with this arena. 2861cb181ed632e7573fb4eab194e4d216867222d27Jason Evans * Stats from these are merged incrementally, and at exit if 2871cb181ed632e7573fb4eab194e4d216867222d27Jason Evans * opt_stats_print is enabled. 288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_head(tcache_t) tcache_ql; 290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 291d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans uint64_t prof_accumbytes; 292d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 2938a03cf039cd06f9fa6972711195055d865673966Jason Evans /* 2948a03cf039cd06f9fa6972711195055d865673966Jason Evans * PRNG state for cache index randomization of large allocation base 2958a03cf039cd06f9fa6972711195055d865673966Jason Evans * pointers. 2968a03cf039cd06f9fa6972711195055d865673966Jason Evans */ 2978a03cf039cd06f9fa6972711195055d865673966Jason Evans uint64_t offset_state; 2988a03cf039cd06f9fa6972711195055d865673966Jason Evans 299609ae595f0358157b19311b0f9f9591db7cee705Jason Evans dss_prec_t dss_prec; 300609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 302e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * In order to avoid rapid chunk allocation/deallocation when an arena 303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * oscillates right on the cusp of needing a new chunk, cache the most 304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * recently freed chunk. The spare is left in the arena's chunk trees 305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * until it is deleted. 306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * There is one spare chunk per arena, rather than one spare total, in 308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * order to avoid interactions between multiple threads that could make 309e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a single spare inadequate. 310e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *spare; 312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 3138d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans /* Minimum ratio (log base 2) of nactive:ndirty. */ 3148d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans ssize_t lg_dirty_mult; 3158d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans 316e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evans /* Number of pages in active runs and huge regions. */ 317bc25a47ee0e2ac8e10a94d5fa070f0dbbdeb7e7eJason Evans size_t nactive; 318e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current count of pages within unused runs that are potentially 321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * dirty, and for which madvise(... MADV_DONTNEED) has not been called. 322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * By tracking this, we can institute a limit on how much dirty unused 323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * memory is mapped for each arena. 324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty; 326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 328ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * Size/address-ordered tree of this arena's available runs. The tree 32997c04a93838c4001688fe31bf018972b4696efe2Jason Evans * is used for first-fit run allocation. 330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 331e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans arena_avail_tree_t runs_avail; 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 333ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans /* 334ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * Unused dirty memory this arena manages. Dirty memory is conceptually 335738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans * tracked as an arbitrarily interleaved LRU of dirty runs and cached 336738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans * chunks, but the list linkage is actually semi-duplicated in order to 337738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans * avoid extra arena_chunk_map_misc_t space overhead. 338ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * 339ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * LRU-----------------------------------------------------------MRU 340ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * 341f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * /-- arena ---\ 342f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | 343f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | 344f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * |------------| /- chunk -\ 345f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * ...->|chunks_cache|<--------------------------->| /----\ |<--... 346f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * |------------| | |node| | 347f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | 348f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | /- run -\ /- run -\ | | | | 349f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | | | | | 350f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | | | | | 351f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * |------------| |-------| |-------| | |----| | 352f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * ...->|runs_dirty |<-->|rd |<-->|rd |<---->|rd |<----... 353f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * |------------| |-------| |-------| | |----| | 354f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | | | | | 355f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | | \----/ | 356f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | \-------/ \-------/ | | 357f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | 358f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | 359f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * \------------/ \---------/ 360ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans */ 36138e42d311c1844a66e8ced84551621de41e42b85Jason Evans arena_runs_dirty_link_t runs_dirty; 362738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans extent_node_t chunks_cache; 363070b3c3fbd90296610005c111ec6060e8bb23d31Jason Evans 364cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* Extant huge allocations. */ 365cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans ql_head(extent_node_t) huge; 366cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* Synchronizes all huge allocation/update/deallocation. */ 367cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans malloc_mutex_t huge_mtx; 368cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans 369cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* 370cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * Trees of chunks that were previously allocated (trees differ only in 371cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * node ordering). These are used when allocating chunks, in an attempt 372cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * to re-use address space. Depending on function, different tree 373cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * orderings are needed, which is why there are two trees with the same 374cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * contents. 375cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans */ 376738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans extent_tree_t chunks_szad_cache; 377738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans extent_tree_t chunks_ad_cache; 378cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans extent_tree_t chunks_szad_mmap; 379cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans extent_tree_t chunks_ad_mmap; 380cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans extent_tree_t chunks_szad_dss; 381cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans extent_tree_t chunks_ad_dss; 382cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans malloc_mutex_t chunks_mtx; 383cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* Cache of nodes that were allocated via base_alloc(). */ 384cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans ql_head(extent_node_t) node_cache; 385cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans malloc_mutex_t node_cache_mtx; 386cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans 387fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind /* 3888d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans * User-configurable chunk allocation/deallocation/purge functions. 389fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind */ 390fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind chunk_alloc_t *chunk_alloc; 391e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evans chunk_dalloc_t *chunk_dalloc; 3928d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans chunk_purge_t *chunk_purge; 393fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind 394b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans /* bins is used to store trees of free regions. */ 395b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans arena_bin_t bins[NBINS]; 396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 397ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans#endif /* JEMALLOC_ARENA_STRUCTS_B */ 398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_STRUCTS */ 400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_EXTERNS 402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4038a03cf039cd06f9fa6972711195055d865673966Jason Evansstatic const size_t large_pad = 4048a03cf039cd06f9fa6972711195055d865673966Jason Evans#ifdef JEMALLOC_CACHE_OBLIVIOUS 4058a03cf039cd06f9fa6972711195055d865673966Jason Evans PAGE 4068a03cf039cd06f9fa6972711195055d865673966Jason Evans#else 4078a03cf039cd06f9fa6972711195055d865673966Jason Evans 0 4088a03cf039cd06f9fa6972711195055d865673966Jason Evans#endif 4098a03cf039cd06f9fa6972711195055d865673966Jason Evans ; 4108a03cf039cd06f9fa6972711195055d865673966Jason Evans 4118a03cf039cd06f9fa6972711195055d865673966Jason Evansextern ssize_t opt_lg_dirty_mult; 412e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 413b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansextern arena_bin_info_t arena_bin_info[NBINS]; 414e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 415155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansextern size_t map_bias; /* Number of arena chunk header pages. */ 416155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansextern size_t map_misc_offset; 417155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansextern size_t arena_maxrun; /* Max run size for arenas. */ 418155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansextern size_t arena_maxclass; /* Max size class for arenas. */ 4193c4d92e82a31f652a7c77ca937a02d0185085b06Jason Evansextern unsigned nlclasses; /* Number of large size classes. */ 4203c4d92e82a31f652a7c77ca937a02d0185085b06Jason Evansextern unsigned nhclasses; /* Number of huge size classes. */ 4213c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans 422738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evansvoid arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, 423738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans bool cache); 424738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evansvoid arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, 425738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans bool cache); 426cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evansextent_node_t *arena_node_alloc(arena_t *arena); 427cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evansvoid arena_node_dalloc(arena_t *arena, extent_node_t *node); 4289b41ac909facf4f09bb1b637b78ba647348e572eJason Evansvoid *arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, 4299b41ac909facf4f09bb1b637b78ba647348e572eJason Evans bool *zero); 4303c4d92e82a31f652a7c77ca937a02d0185085b06Jason Evansvoid arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize); 4319b41ac909facf4f09bb1b637b78ba647348e572eJason Evansvoid arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk, 4329b41ac909facf4f09bb1b637b78ba647348e572eJason Evans size_t oldsize, size_t usize); 4339b41ac909facf4f09bb1b637b78ba647348e572eJason Evansvoid arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, 4349b41ac909facf4f09bb1b637b78ba647348e572eJason Evans size_t oldsize, size_t usize); 4359b41ac909facf4f09bb1b637b78ba647348e572eJason Evansbool arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, 4369b41ac909facf4f09bb1b637b78ba647348e572eJason Evans size_t oldsize, size_t usize, bool *zero); 4378d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evansssize_t arena_lg_dirty_mult_get(arena_t *arena); 4388d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evansbool arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult); 43999bd94fb65a0b6423c4efcc3e3e501179b92a4dbJason Evansvoid arena_maybe_purge(arena_t *arena); 4406005f0710cf07d60659d91b20b7ff5592d310027Jason Evansvoid arena_purge_all(arena_t *arena); 441dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, 442155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans index_t binind, uint64_t prof_accumbytes); 443122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, 444122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bool zero); 4450d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#ifdef JEMALLOC_JET 4460d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evanstypedef void (arena_redzone_corruption_t)(void *, size_t, bool, size_t, 4470d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans uint8_t); 4486b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_redzone_corruption_t *arena_redzone_corruption; 4496b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_dalloc_junk_small_t)(void *, arena_bin_info_t *); 4506b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_dalloc_junk_small_t *arena_dalloc_junk_small; 4516b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#else 452122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); 4536b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 4540d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansvoid arena_quarantine_junk_small(void *ptr, size_t usize); 455e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid *arena_malloc_small(arena_t *arena, size_t size, bool zero); 456dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid *arena_malloc_large(arena_t *arena, size_t size, bool zero); 45788fef7ceda6269598cef0cee8b984c8765673c27Jason Evansvoid *arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, 45888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t alignment, bool zero, tcache_t *tcache); 4590b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansvoid arena_prof_promoted(const void *ptr, size_t size); 460fc0b3b7383373d66cfed2cd4e2faa272a6868d32Jason Evansvoid arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, 461fc0b3b7383373d66cfed2cd4e2faa272a6868d32Jason Evans void *ptr, arena_chunk_map_bits_t *bitselm); 462203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 463ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind, arena_chunk_map_bits_t *bitselm); 464203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 465203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind); 4666b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 4676b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_dalloc_junk_large_t)(void *, size_t); 4686b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_dalloc_junk_large_t *arena_dalloc_junk_large; 469fc0b3b7383373d66cfed2cd4e2faa272a6868d32Jason Evans#else 470fc0b3b7383373d66cfed2cd4e2faa272a6868d32Jason Evansvoid arena_dalloc_junk_large(void *ptr, size_t usize); 4716b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 472fc0b3b7383373d66cfed2cd4e2faa272a6868d32Jason Evansvoid arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk, 473203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans void *ptr); 474e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); 4756b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 4766b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t); 4776b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_ralloc_junk_large_t *arena_ralloc_junk_large; 4786b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 479b2c31660be917ea6d59cd54e6f650b06b5e812edJason Evansbool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, 4808e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t extra, bool zero); 4815460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, 4821cb181ed632e7573fb4eab194e4d216867222d27Jason Evans size_t size, size_t extra, size_t alignment, bool zero, tcache_t *tcache); 483609ae595f0358157b19311b0f9f9591db7cee705Jason Evansdss_prec_t arena_dss_prec_get(arena_t *arena); 4844d434adb146375ad17f0d5e994ed5728d2942e3fJason Evansbool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); 4858d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evansssize_t arena_lg_dirty_mult_default_get(void); 4868d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evansbool arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult); 487562d266511053a51406e91c78eba640cb46ad9c8Jason Evansvoid arena_stats_merge(arena_t *arena, const char **dss, 488562d266511053a51406e91c78eba640cb46ad9c8Jason Evans ssize_t *lg_dirty_mult, size_t *nactive, size_t *ndirty, 489562d266511053a51406e91c78eba640cb46ad9c8Jason Evans arena_stats_t *astats, malloc_bin_stats_t *bstats, 4903c4d92e82a31f652a7c77ca937a02d0185085b06Jason Evans malloc_large_stats_t *lstats, malloc_huge_stats_t *hstats); 4918bb3198f72fc7587dc93527f9f19fb5be52fa553Jason Evansarena_t *arena_new(unsigned ind); 4928a03cf039cd06f9fa6972711195055d865673966Jason Evansbool arena_boot(void); 4934e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_prefork(arena_t *arena); 4944e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_postfork_parent(arena_t *arena); 4954e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_postfork_child(arena_t *arena); 496e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 497e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_EXTERNS */ 498e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 499e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_INLINES 500e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 501e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifndef JEMALLOC_ENABLE_INLINE 502ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_chunk_map_bits_t *arena_bitselm_get(arena_chunk_t *chunk, 503ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind); 504ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_chunk_map_misc_t *arena_miscelm_get(arena_chunk_t *chunk, 505ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind); 5060c5dd03e889d0269170b5db9fa872738d906eb78Jason Evanssize_t arena_miscelm_to_pageind(arena_chunk_map_misc_t *miscelm); 5070c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansvoid *arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm); 50838e42d311c1844a66e8ced84551621de41e42b85Jason Evansarena_chunk_map_misc_t *arena_rd_to_miscelm(arena_runs_dirty_link_t *rd); 5090c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_chunk_map_misc_t *arena_run_to_miscelm(arena_run_t *run); 510203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); 51187a02d2bb18dbcb2955541b849bc95862e864803Jason Evanssize_t arena_mapbitsp_read(size_t *mapbitsp); 512203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); 513203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, 514203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind); 515203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); 516203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); 517155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansindex_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); 518203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); 519203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); 520203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); 521203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind); 52287a02d2bb18dbcb2955541b849bc95862e864803Jason Evansvoid arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits); 523203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, 524203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size, size_t flags); 525203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 526203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size); 527203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, 528203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size, size_t flags); 529203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 530155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans index_t binind); 531203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, 532155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans size_t runind, index_t binind, size_t flags); 533203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, 534203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t unzeroed); 5354581b97809e7e545c38b996870a4e7284a620bc5Jason Evansvoid arena_metadata_allocated_add(arena_t *arena, size_t size); 5364581b97809e7e545c38b996870a4e7284a620bc5Jason Evansvoid arena_metadata_allocated_sub(arena_t *arena, size_t size); 5374581b97809e7e545c38b996870a4e7284a620bc5Jason Evanssize_t arena_metadata_allocated_get(arena_t *arena); 53888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); 53988c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); 54088c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum(arena_t *arena, uint64_t accumbytes); 541155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansindex_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); 542155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansindex_t arena_bin_index(arena_t *arena, arena_bin_t *bin); 54349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansunsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, 544b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans const void *ptr); 545602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansprof_tctx_t *arena_prof_tctx_get(const void *ptr); 546602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansvoid arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); 5475460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, 5481cb181ed632e7573fb4eab194e4d216867222d27Jason Evans tcache_t *tcache); 5494581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_t *arena_aalloc(const void *ptr); 550f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evanssize_t arena_salloc(const void *ptr, bool demote); 551cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evansvoid arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache); 55288fef7ceda6269598cef0cee8b984c8765673c27Jason Evansvoid arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache); 553e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 554e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 555e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) 556203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans# ifdef JEMALLOC_ARENA_INLINE_A 557ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan WuJEMALLOC_ALWAYS_INLINE arena_chunk_map_bits_t * 558ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_bitselm_get(arena_chunk_t *chunk, size_t pageind) 559ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu{ 560ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 561ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu assert(pageind >= map_bias); 562ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu assert(pageind < chunk_npages); 563ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 564ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return (&chunk->map_bits[pageind-map_bias]); 565ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu} 566ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 567ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan WuJEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 568ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_miscelm_get(arena_chunk_t *chunk, size_t pageind) 569203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 570203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 571203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind >= map_bias); 572203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind < chunk_npages); 573203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 574ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return ((arena_chunk_map_misc_t *)((uintptr_t)chunk + 575ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu (uintptr_t)map_misc_offset) + pageind-map_bias); 576203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 577203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 5780c5dd03e889d0269170b5db9fa872738d906eb78Jason EvansJEMALLOC_ALWAYS_INLINE size_t 5790c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_miscelm_to_pageind(arena_chunk_map_misc_t *miscelm) 5800c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans{ 5810c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 5820c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans size_t pageind = ((uintptr_t)miscelm - ((uintptr_t)chunk + 5830c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans map_misc_offset)) / sizeof(arena_chunk_map_misc_t) + map_bias; 5840c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 5850c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(pageind >= map_bias); 5860c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(pageind < chunk_npages); 5870c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 5880c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans return (pageind); 5890c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans} 5900c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 5910c5dd03e889d0269170b5db9fa872738d906eb78Jason EvansJEMALLOC_ALWAYS_INLINE void * 5920c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm) 5930c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans{ 5940c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 5950c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans size_t pageind = arena_miscelm_to_pageind(miscelm); 5960c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 5970c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE))); 5980c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans} 5990c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6000c5dd03e889d0269170b5db9fa872738d906eb78Jason EvansJEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 60138e42d311c1844a66e8ced84551621de41e42b85Jason Evansarena_rd_to_miscelm(arena_runs_dirty_link_t *rd) 60238e42d311c1844a66e8ced84551621de41e42b85Jason Evans{ 60338e42d311c1844a66e8ced84551621de41e42b85Jason Evans arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t 60438e42d311c1844a66e8ced84551621de41e42b85Jason Evans *)((uintptr_t)rd - offsetof(arena_chunk_map_misc_t, rd)); 60538e42d311c1844a66e8ced84551621de41e42b85Jason Evans 60638e42d311c1844a66e8ced84551621de41e42b85Jason Evans assert(arena_miscelm_to_pageind(miscelm) >= map_bias); 60738e42d311c1844a66e8ced84551621de41e42b85Jason Evans assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); 60838e42d311c1844a66e8ced84551621de41e42b85Jason Evans 60938e42d311c1844a66e8ced84551621de41e42b85Jason Evans return (miscelm); 61038e42d311c1844a66e8ced84551621de41e42b85Jason Evans} 61138e42d311c1844a66e8ced84551621de41e42b85Jason Evans 61238e42d311c1844a66e8ced84551621de41e42b85Jason EvansJEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 6130c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_run_to_miscelm(arena_run_t *run) 6140c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans{ 6150c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t 6160c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans *)((uintptr_t)run - offsetof(arena_chunk_map_misc_t, run)); 6170c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6180c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(arena_miscelm_to_pageind(miscelm) >= map_bias); 6190c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); 6200c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6210c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans return (miscelm); 6220c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans} 6230c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 62488393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t * 625203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) 626203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 627203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 628ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return (&arena_bitselm_get(chunk, pageind)->bits); 629203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 630203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 63188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 63287a02d2bb18dbcb2955541b849bc95862e864803Jason Evansarena_mapbitsp_read(size_t *mapbitsp) 63387a02d2bb18dbcb2955541b849bc95862e864803Jason Evans{ 63487a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 63587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans return (*mapbitsp); 63687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans} 63787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 63887a02d2bb18dbcb2955541b849bc95862e864803Jason EvansJEMALLOC_ALWAYS_INLINE size_t 639203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_get(arena_chunk_t *chunk, size_t pageind) 640203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 641203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 64287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans return (arena_mapbitsp_read(arena_mapbitsp_get(chunk, pageind))); 643203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 644203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 64588393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 646203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) 647203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 648203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 649203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 650203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 651203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 652203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & ~PAGE_MASK); 653203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 654203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 65588393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 656203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) 657203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 658203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 659203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 660203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 661203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 662203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); 663203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & ~PAGE_MASK); 664203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 665203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 66688393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 667203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) 668203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 669203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 670203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 671203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 672203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 673203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans CHUNK_MAP_ALLOCATED); 674203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits >> LG_PAGE); 675203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 676203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 677155bfa7da18cab0d21d87aa2dce4554166836f5dJason EvansJEMALLOC_ALWAYS_INLINE index_t 67880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evansarena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) 67980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans{ 68080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t mapbits; 681155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans index_t binind; 68280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 68380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans mapbits = arena_mapbits_get(chunk, pageind); 68480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 68580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind < NBINS || binind == BININD_INVALID); 68680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans return (binind); 68780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans} 68880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 68988393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 690203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) 691203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 692203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 693203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 694203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 695203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_DIRTY); 696203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 697203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 69888393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 699203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind) 700203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 701203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 702203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 703203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 704203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_UNZEROED); 705203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 706203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 70788393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 708203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind) 709203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 710203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 711203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 712203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 713203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_LARGE); 714203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 715203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 71688393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 717203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind) 718203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 719203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 720203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 721203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 722203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_ALLOCATED); 723203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 724203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 72588393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 72687a02d2bb18dbcb2955541b849bc95862e864803Jason Evansarena_mapbitsp_write(size_t *mapbitsp, size_t mapbits) 72787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans{ 72887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 72987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans *mapbitsp = mapbits; 73087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans} 73187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 73287a02d2bb18dbcb2955541b849bc95862e864803Jason EvansJEMALLOC_ALWAYS_INLINE void 733203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, 734203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flags) 735203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 73687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 737203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 7388a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(size == PAGE_CEILING(size)); 739203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0); 740d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans assert((flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == flags); 74187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags); 742203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 743203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 74488393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 745203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 746203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size) 747203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 74887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 74987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 750203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 7518a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(size == PAGE_CEILING(size)); 75287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 75387a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, size | (mapbits & PAGE_MASK)); 754203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 755203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 75688393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 757203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, 758203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flags) 759203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 76087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 76187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 762d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans size_t unzeroed; 763203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 7648a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(size == PAGE_CEILING(size)); 765d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans assert((flags & CHUNK_MAP_DIRTY) == flags); 76687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ 76787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags 76887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans | unzeroed | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED); 769203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 770203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 77188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 772203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 773155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans index_t binind) 774203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 77587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 77687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 777203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 778203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind <= BININD_INVALID); 7798a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(arena_mapbits_large_size_get(chunk, pageind) == LARGE_MINCLASS + 7808a03cf039cd06f9fa6972711195055d865673966Jason Evans large_pad); 78187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) | 78287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans (binind << CHUNK_MAP_BININD_SHIFT)); 783203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 784203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 78588393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 786203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, 787155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans index_t binind, size_t flags) 788203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 78987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 79087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 791d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans size_t unzeroed; 792203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 793203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind < BININD_INVALID); 794203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind - runind >= map_bias); 795d8ceef6c5558fdab8f9448376ae065a9e5ffcbddJason Evans assert((flags & CHUNK_MAP_DIRTY) == flags); 79687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ 79787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, (runind << LG_PAGE) | (binind << 79887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans CHUNK_MAP_BININD_SHIFT) | flags | unzeroed | CHUNK_MAP_ALLOCATED); 799203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 800203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 80188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 802203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, 803203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t unzeroed) 804203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 80587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 80687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 807203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 80887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_UNZEROED) | 80987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans unzeroed); 810203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 811203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 8124581b97809e7e545c38b996870a4e7284a620bc5Jason EvansJEMALLOC_INLINE void 8134581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_metadata_allocated_add(arena_t *arena, size_t size) 8144581b97809e7e545c38b996870a4e7284a620bc5Jason Evans{ 8154581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 8164581b97809e7e545c38b996870a4e7284a620bc5Jason Evans atomic_add_z(&arena->stats.metadata_allocated, size); 8174581b97809e7e545c38b996870a4e7284a620bc5Jason Evans} 8184581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 8194581b97809e7e545c38b996870a4e7284a620bc5Jason EvansJEMALLOC_INLINE void 8204581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_metadata_allocated_sub(arena_t *arena, size_t size) 8214581b97809e7e545c38b996870a4e7284a620bc5Jason Evans{ 8224581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 8234581b97809e7e545c38b996870a4e7284a620bc5Jason Evans atomic_sub_z(&arena->stats.metadata_allocated, size); 8244581b97809e7e545c38b996870a4e7284a620bc5Jason Evans} 8254581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 8264581b97809e7e545c38b996870a4e7284a620bc5Jason EvansJEMALLOC_INLINE size_t 8274581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_metadata_allocated_get(arena_t *arena) 8284581b97809e7e545c38b996870a4e7284a620bc5Jason Evans{ 8294581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 8304581b97809e7e545c38b996870a4e7284a620bc5Jason Evans return (atomic_read_z(&arena->stats.metadata_allocated)); 8314581b97809e7e545c38b996870a4e7284a620bc5Jason Evans} 8324581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 83388c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 834a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) 835a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 836a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 837a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 838a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans assert(prof_interval != 0); 839a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 840a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena->prof_accumbytes += accumbytes; 841a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans if (arena->prof_accumbytes >= prof_interval) { 842a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena->prof_accumbytes -= prof_interval; 84388c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (true); 844a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans } 84588c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 846a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 847a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 84888c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 849a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum_locked(arena_t *arena, uint64_t accumbytes) 850a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 851a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 852a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 853a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 8549c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(prof_interval == 0)) 85588c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 85688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (arena_prof_accum_impl(arena, accumbytes)); 857a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 858a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 85988c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 860a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum(arena_t *arena, uint64_t accumbytes) 861a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 862a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 863a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 864a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 8659c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(prof_interval == 0)) 86688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 86788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans 86888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans { 86988c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans bool ret; 87088c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans 87188c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans malloc_mutex_lock(&arena->lock); 87288c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans ret = arena_prof_accum_impl(arena, accumbytes); 87388c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans malloc_mutex_unlock(&arena->lock); 87488c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (ret); 87588c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans } 876a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 877a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 878155bfa7da18cab0d21d87aa2dce4554166836f5dJason EvansJEMALLOC_ALWAYS_INLINE index_t 87980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evansarena_ptr_small_binind_get(const void *ptr, size_t mapbits) 880203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 881155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans index_t binind; 882203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 883203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 884203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 885203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (config_debug) { 88680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_chunk_t *chunk; 88780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_t *arena; 88880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t pageind; 88980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t actual_mapbits; 8900c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans size_t rpages_ind; 89180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_run_t *run; 89280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_bin_t *bin; 893381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans index_t run_binind, actual_binind; 89480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_bin_info_t *bin_info; 8950c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_map_misc_t *miscelm; 8960c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans void *rpages; 89780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 89880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind != BININD_INVALID); 89980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind < NBINS); 90080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 901ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans arena = extent_node_arena_get(&chunk->node); 90280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 90380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans actual_mapbits = arena_mapbits_get(chunk, pageind); 904203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(mapbits == actual_mapbits); 90580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_large_get(chunk, pageind) == 0); 90680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 9070c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, 9080c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans pageind); 9090c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans miscelm = arena_miscelm_get(chunk, rpages_ind); 9100c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans run = &miscelm->run; 911381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans run_binind = run->binind; 912381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans bin = &arena->bins[run_binind]; 91380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans actual_binind = bin - arena->bins; 914381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans assert(run_binind == actual_binind); 91580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans bin_info = &arena_bin_info[actual_binind]; 9160c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans rpages = arena_miscelm_to_rpages(miscelm); 9170c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(((uintptr_t)ptr - ((uintptr_t)rpages + 918203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval 919203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans == 0); 920203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 921203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 922203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (binind); 923203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 924155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans# endif /* JEMALLOC_ARENA_INLINE_A */ 925203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 926155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans# ifdef JEMALLOC_ARENA_INLINE_B 927155bfa7da18cab0d21d87aa2dce4554166836f5dJason EvansJEMALLOC_INLINE index_t 92849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_bin_index(arena_t *arena, arena_bin_t *bin) 92949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 930155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans index_t binind = bin - arena->bins; 931b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 93249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (binind); 93349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 93449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 93581b4e6eb6f06cac048e3743787a70676f1534269Jason EvansJEMALLOC_INLINE unsigned 936b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evansarena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) 93781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 93881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans unsigned shift, diff, regind; 939122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t interval; 9400c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 9410c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans void *rpages = arena_miscelm_to_rpages(miscelm); 94281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 94384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 94484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Freeing a pointer lower than region zero can cause assertion 94584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * failure. 94684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 9470c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert((uintptr_t)ptr >= (uintptr_t)rpages + 94884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset); 94981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 95081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 95181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * Avoid doing division with a variable divisor if possible. Using 95281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * actual division here can reduce allocator throughput by over 20%! 95381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 9540c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans diff = (unsigned)((uintptr_t)ptr - (uintptr_t)rpages - 95549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg0_offset); 95681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 95781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* Rescale (factor powers of 2 out of the numerator and denominator). */ 958122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans interval = bin_info->reg_interval; 9599c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3Richard Diamond shift = jemalloc_ffs(interval) - 1; 96081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans diff >>= shift; 961122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans interval >>= shift; 96281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 963122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (interval == 1) { 96481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* The divisor was a power of 2. */ 96581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans regind = diff; 96681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } else { 96781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 96881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * To divide by a number D that is not a power of two we 96981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * multiply by (2^21 / D) and then right shift by 21 positions. 97081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 97181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * X / D 97281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 97381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * becomes 97481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 975122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT 97681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 97781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * We can omit the first three elements, because we never 97881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * divide by 0, and 1 and 2 are both powers of two, which are 97981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * handled above. 98081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 98147e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) 98247e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) 983122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans static const unsigned interval_invs[] = { 98481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(3), 98581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 98681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 98781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 98881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 98981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 99081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 99181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 99281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans }; 99381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 9949c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(interval <= ((sizeof(interval_invs) / 9959c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans sizeof(unsigned)) + 2))) { 996122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans regind = (diff * interval_invs[interval - 3]) >> 997122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans SIZE_INV_SHIFT; 998122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else 999122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans regind = diff / interval; 100081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV 100181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV_SHIFT 100281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } 1003122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(diff == regind * interval); 100449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(regind < bin_info->nregs); 100581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 100681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (regind); 100781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 100881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 1009602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason EvansJEMALLOC_INLINE prof_tctx_t * 1010602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansarena_prof_tctx_get(const void *ptr) 101181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 1012602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *ret; 101381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans arena_chunk_t *chunk; 101481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 10157372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 101681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert(ptr != NULL); 101781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 101881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 101988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 102088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 102188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t mapbits = arena_mapbits_get(chunk, pageind); 102288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 102388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) 102488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans ret = (prof_tctx_t *)(uintptr_t)1U; 10255f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans else { 10265f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans arena_chunk_map_misc_t *elm = arena_miscelm_get(chunk, 10275f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans pageind); 10285f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans ret = atomic_read_p((void **)&elm->prof_tctx); 10295f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans } 103088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 103188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans ret = huge_prof_tctx_get(ptr); 103281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 103381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (ret); 103481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 1035e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 1036e4f7846f1fd279a039ffa2a41707348187219de4Jason EvansJEMALLOC_INLINE void 1037602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansarena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) 1038e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans{ 1039e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans arena_chunk_t *chunk; 1040e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 10417372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 1042e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert(ptr != NULL); 1043e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 1044e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 104588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 104688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 104788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1048665769357cd77b74e00a146f196fff19243b33c4Jason Evans 10495f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans if (unlikely(arena_mapbits_large_get(chunk, pageind) != 0)) { 10505f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans arena_chunk_map_misc_t *elm = arena_miscelm_get(chunk, 10515f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans pageind); 10525f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans atomic_write_p((void **)&elm->prof_tctx, tctx); 10535f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans } 105488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 105588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans huge_prof_tctx_set(ptr, tctx); 1056e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans} 105781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 105888393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void * 10595460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansarena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, 10601cb181ed632e7573fb4eab194e4d216867222d27Jason Evans tcache_t *tcache) 1061962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans{ 1062962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1063962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans assert(size != 0); 1064962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 106541cfe03f39740fe61cf46d86982f66c24168de32Jason Evans arena = arena_choose(tsd, arena); 106641cfe03f39740fe61cf46d86982f66c24168de32Jason Evans if (unlikely(arena == NULL)) 106741cfe03f39740fe61cf46d86982f66c24168de32Jason Evans return (NULL); 106841cfe03f39740fe61cf46d86982f66c24168de32Jason Evans 10699c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(size <= SMALL_MAXCLASS)) { 107041cfe03f39740fe61cf46d86982f66c24168de32Jason Evans if (likely(tcache != NULL)) { 107141cfe03f39740fe61cf46d86982f66c24168de32Jason Evans return (tcache_alloc_small(tsd, arena, tcache, size, 107241cfe03f39740fe61cf46d86982f66c24168de32Jason Evans zero)); 107341cfe03f39740fe61cf46d86982f66c24168de32Jason Evans } else 10748bb3198f72fc7587dc93527f9f19fb5be52fa553Jason Evans return (arena_malloc_small(arena, size, zero)); 107588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else if (likely(size <= arena_maxclass)) { 1076746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans /* 1077746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans * Initialize tcache after checking size in order to avoid 1078746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans * infinite recursion during tcache initialization. 1079746868929afae3e346b47d0fa8a78d7fb131d5a4Jason Evans */ 108041cfe03f39740fe61cf46d86982f66c24168de32Jason Evans if (likely(tcache != NULL) && size <= tcache_maxclass) { 108141cfe03f39740fe61cf46d86982f66c24168de32Jason Evans return (tcache_alloc_large(tsd, arena, tcache, size, 108241cfe03f39740fe61cf46d86982f66c24168de32Jason Evans zero)); 108341cfe03f39740fe61cf46d86982f66c24168de32Jason Evans } else 10848bb3198f72fc7587dc93527f9f19fb5be52fa553Jason Evans return (arena_malloc_large(arena, size, zero)); 108588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 108688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans return (huge_malloc(tsd, arena, size, zero, tcache)); 1087962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans} 1088962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 10894581b97809e7e545c38b996870a4e7284a620bc5Jason EvansJEMALLOC_ALWAYS_INLINE arena_t * 10904581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_aalloc(const void *ptr) 10914581b97809e7e545c38b996870a4e7284a620bc5Jason Evans{ 10924581b97809e7e545c38b996870a4e7284a620bc5Jason Evans arena_chunk_t *chunk; 10934581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 10944581b97809e7e545c38b996870a4e7284a620bc5Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 109588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) 1096ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans return (extent_node_arena_get(&chunk->node)); 109788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans else 109888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans return (huge_aalloc(ptr)); 10994581b97809e7e545c38b996870a4e7284a620bc5Jason Evans} 11004581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 1101f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans/* Return the size of the allocation pointed to by ptr. */ 110288393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 1103f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evansarena_salloc(const void *ptr, bool demote) 1104f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans{ 1105f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans size_t ret; 1106f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans arena_chunk_t *chunk; 1107155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans size_t pageind; 1108155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans index_t binind; 1109f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1110f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(ptr != NULL); 1111f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1112f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 111388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 111488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 111588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 111688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans binind = arena_mapbits_binind_get(chunk, pageind); 111788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (unlikely(binind == BININD_INVALID || (config_prof && !demote 111888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans && arena_mapbits_large_get(chunk, pageind) != 0))) { 111988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* 112088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * Large allocation. In the common case (demote), and 112188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * as this is an inline function, most callers will only 112288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * end up looking at binind to determine that ptr is a 112388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * small allocation. 112488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans */ 11258a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(config_cache_oblivious || ((uintptr_t)ptr & 11268a03cf039cd06f9fa6972711195055d865673966Jason Evans PAGE_MASK) == 0); 11278a03cf039cd06f9fa6972711195055d865673966Jason Evans ret = arena_mapbits_large_size_get(chunk, pageind) - 11288a03cf039cd06f9fa6972711195055d865673966Jason Evans large_pad; 112988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(ret != 0); 11308a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(pageind + ((ret+large_pad)>>LG_PAGE) <= 11318a03cf039cd06f9fa6972711195055d865673966Jason Evans chunk_npages); 113288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_dirty_get(chunk, pageind) == 113388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans arena_mapbits_dirty_get(chunk, 11348a03cf039cd06f9fa6972711195055d865673966Jason Evans pageind+((ret+large_pad)>>LG_PAGE)-1)); 113588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else { 113688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* 113788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * Small allocation (possibly promoted to a large 113888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * object). 113988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans */ 114088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_large_get(chunk, pageind) != 0 || 114188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans arena_ptr_small_binind_get(ptr, 114288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans arena_mapbits_get(chunk, pageind)) == binind); 114388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans ret = index2size(binind); 114488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 114588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 114688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans ret = huge_salloc(ptr); 1147f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1148f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans return (ret); 1149f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans} 1150f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 115188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 1152cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evansarena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) 1153e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1154cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans arena_chunk_t *chunk; 1155203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind, mapbits; 1156e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1157e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1158cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans 1159cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 116088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 116188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 116288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans mapbits = arena_mapbits_get(chunk, pageind); 116388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 116488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { 116588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* Small allocation. */ 116688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(tcache != NULL)) { 116788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans index_t binind = arena_ptr_small_binind_get(ptr, 116888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans mapbits); 116988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans tcache_dalloc_small(tsd, tcache, ptr, binind); 117088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else { 1171ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans arena_dalloc_small(extent_node_arena_get( 1172ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans &chunk->node), chunk, ptr, pageind); 117388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 1174cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans } else { 117588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t size = arena_mapbits_large_size_get(chunk, 1176cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans pageind); 1177962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 11788a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(config_cache_oblivious || ((uintptr_t)ptr & 11798a03cf039cd06f9fa6972711195055d865673966Jason Evans PAGE_MASK) == 0); 1180962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 11818a03cf039cd06f9fa6972711195055d865673966Jason Evans if (likely(tcache != NULL) && size <= tcache_maxclass) { 11828a03cf039cd06f9fa6972711195055d865673966Jason Evans tcache_dalloc_large(tsd, tcache, ptr, size - 11838a03cf039cd06f9fa6972711195055d865673966Jason Evans large_pad); 11848a03cf039cd06f9fa6972711195055d865673966Jason Evans } else { 1185ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans arena_dalloc_large(extent_node_arena_get( 1186ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans &chunk->node), chunk, ptr); 118788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 118888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 118988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 119088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans huge_dalloc(tsd, ptr, tcache); 1191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 11924cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 11934cfe55166e0173be745c53adb0fecf50d11d1227Daniel MicayJEMALLOC_ALWAYS_INLINE void 119488fef7ceda6269598cef0cee8b984c8765673c27Jason Evansarena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache) 11954cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay{ 119688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans arena_chunk_t *chunk; 11974cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 119888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 119988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 120088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (config_prof && opt_prof) { 12019c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 12029c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans LG_PAGE; 120388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 120488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (arena_mapbits_large_get(chunk, pageind) != 0) { 120588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* 120688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * Make sure to use promoted size, not request 120788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * size. 120888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans */ 120988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 121088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size = arena_mapbits_large_size_get(chunk, 12118a03cf039cd06f9fa6972711195055d865673966Jason Evans pageind) - large_pad; 121288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 12134cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay } 121488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(s2u(size) == s2u(arena_salloc(ptr, false))); 121588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans 121688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(size <= SMALL_MAXCLASS)) { 121788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* Small allocation. */ 121888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(tcache != NULL)) { 121988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans index_t binind = size2index(size); 122088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans tcache_dalloc_small(tsd, tcache, ptr, binind); 122188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else { 122288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t pageind = ((uintptr_t)ptr - 122388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans (uintptr_t)chunk) >> LG_PAGE; 1224ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans arena_dalloc_small(extent_node_arena_get( 1225ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans &chunk->node), chunk, ptr, pageind); 122688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 122788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else { 12288a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(config_cache_oblivious || ((uintptr_t)ptr & 12298a03cf039cd06f9fa6972711195055d865673966Jason Evans PAGE_MASK) == 0); 12304cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 123188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(tcache != NULL) && size <= tcache_maxclass) 123288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans tcache_dalloc_large(tsd, tcache, ptr, size); 123388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans else { 1234ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans arena_dalloc_large(extent_node_arena_get( 1235ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans &chunk->node), chunk, ptr); 123688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 123788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 123888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 123988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans huge_dalloc(tsd, ptr, tcache); 12404cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay} 1241155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans# endif /* JEMALLOC_ARENA_INLINE_B */ 1242e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_INLINES */ 1245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 1246