arena.h revision 9e1810ca9dc4a5f5f0841b9a6c1abb4337753552
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 2e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_TYPES 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans#define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) 5155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans 647e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans/* Maximum number of regions in one run. */ 70c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans#define LG_RUN_MAXREGS (LG_PAGE - LG_TINY_MIN) 847e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) 947e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 11122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Minimum redzone size. Redzones may be larger than this if necessary to 12122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * preserve region alignment. 13122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans */ 14122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans#define REDZONE_MINSIZE 16 15122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 16122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans/* 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The minimum ratio of active:dirty pages per arena is computed as: 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 198d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans * (nactive >> lg_dirty_mult) >= ndirty 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 218d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans * So, supposing that lg_dirty_mult is 3, there can be no less than 8 times as 228d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans * many active pages as dirty pages. 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 24e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans#define LG_DIRTY_MULT_DEFAULT 3 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 26243f7a0508bb014c2a7bf592c466a923911db234Jason Evanstypedef enum { 27243f7a0508bb014c2a7bf592c466a923911db234Jason Evans purge_mode_ratio = 0, 28243f7a0508bb014c2a7bf592c466a923911db234Jason Evans purge_mode_decay = 1, 29243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 30243f7a0508bb014c2a7bf592c466a923911db234Jason Evans purge_mode_limit = 2 31243f7a0508bb014c2a7bf592c466a923911db234Jason Evans} purge_mode_t; 32243f7a0508bb014c2a7bf592c466a923911db234Jason Evans#define PURGE_DEFAULT purge_mode_ratio 33243f7a0508bb014c2a7bf592c466a923911db234Jason Evans/* Default decay time in seconds. */ 34243f7a0508bb014c2a7bf592c466a923911db234Jason Evans#define DECAY_TIME_DEFAULT 10 35243f7a0508bb014c2a7bf592c466a923911db234Jason Evans/* Number of event ticks between time checks. */ 36243f7a0508bb014c2a7bf592c466a923911db234Jason Evans#define DECAY_NTICKS_PER_UPDATE 1000 37243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 3838e42d311c1844a66e8ced84551621de41e42b85Jason Evanstypedef struct arena_runs_dirty_link_s arena_runs_dirty_link_t; 390c5dd03e889d0269170b5db9fa872738d906eb78Jason Evanstypedef struct arena_run_s arena_run_t; 40ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; 41ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; 42e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_chunk_s arena_chunk_t; 4349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evanstypedef struct arena_bin_info_s arena_bin_info_t; 44e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_bin_s arena_bin_t; 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_s arena_t; 46db927b672748994bef0df6b5f9e94fe6c1d40d02Jason Evanstypedef struct arena_tdata_s arena_tdata_t; 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_TYPES */ 49e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_STRUCTS 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 52ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans#ifdef JEMALLOC_ARENA_STRUCTS_A 530c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansstruct arena_run_s { 54381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans /* Index of bin this run is associated with. */ 55d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind; 560c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 570c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Number of free regions in run. */ 580c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans unsigned nfree; 590c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 600c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Per region allocated/deallocated bitmap. */ 610c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans bitmap_t bitmap[BITMAP_GROUPS_MAX]; 620c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans}; 630c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 64e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Each element of the chunk map corresponds to one page within the chunk. */ 65ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wustruct arena_chunk_map_bits_s { 66e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Run address (or size) and various flags are stored together. The bit 68e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * layout looks like (assuming 32-bit system): 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 7085ae064e96387347915973dcc1ac15e553259bc9Jason Evans * ???????? ???????? ???nnnnn nnndumla 71e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 72e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * ? : Unallocated: Run address for first/last pages, unset for internal 73e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * pages. 7419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Small: Run page offset. 758fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * Large: Run page count for first page, unset for trailing pages. 7653bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * n : binind for small size class, BININD_INVALID for large size class. 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * d : dirty? 788ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * u : unzeroed? 798fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * m : decommitted? 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * l : large? 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a : allocated? 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 83e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Following are example bit patterns for the three types of runs. 84e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 85e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * p : run page offset 86e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * s : run size 87203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * n : binind for size class; large objects set these to BININD_INVALID 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * x : don't care 89e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * - : 0 900b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * + : 1 918fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * [DUMLA] : bit set 928fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * [dumla] : bit unset 93e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 9419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (clean): 958fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sss+++++ +++dum-a 968fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxx-Uxxx 978fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sss+++++ +++dUm-a 9819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 9919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (dirty): 1008fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sss+++++ +++D-m-a 101203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 1028fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sss+++++ +++D-m-a 103e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 104dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Small: 1058fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * pppppppp pppppppp pppnnnnn nnnd---A 1068fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * pppppppp pppppppp pppnnnnn nnn----A 1078fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * pppppppp pppppppp pppnnnnn nnnd---A 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: 1108fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sss+++++ +++D--LA 111203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 1128fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * -------- -------- ---+++++ +++D--LA 1130b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 114155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * Large (sampled, size <= LARGE_MINCLASS): 1158fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sssnnnnn nnnD--LA 116155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 1178fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * -------- -------- ---+++++ +++D--LA 1180b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 119155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * Large (not sampled, size == LARGE_MINCLASS): 1208fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sss+++++ +++D--LA 121155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 1228fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * -------- -------- ---+++++ +++D--LA 123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t bits; 1258fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_ALLOCATED ((size_t)0x01U) 1268fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_LARGE ((size_t)0x02U) 1278fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_STATE_MASK ((size_t)0x3U) 1288fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans 1298fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_DECOMMITTED ((size_t)0x04U) 1308fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_UNZEROED ((size_t)0x08U) 1318fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_DIRTY ((size_t)0x10U) 1328fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_FLAGS_MASK ((size_t)0x1cU) 1338fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans 1348fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_BININD_SHIFT 5 135203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define BININD_INVALID ((size_t)0xffU) 1368fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_BININD_MASK (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) 137203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK 1388fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans 1398fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_RUNIND_SHIFT (CHUNK_MAP_BININD_SHIFT + 8) 1408fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_SIZE_SHIFT (CHUNK_MAP_RUNIND_SHIFT - LG_PAGE) 1418fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_SIZE_MASK \ 1428fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans (~(CHUNK_MAP_BININD_MASK | CHUNK_MAP_FLAGS_MASK | CHUNK_MAP_STATE_MASK)) 143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 144ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 14538e42d311c1844a66e8ced84551621de41e42b85Jason Evansstruct arena_runs_dirty_link_s { 14638e42d311c1844a66e8ced84551621de41e42b85Jason Evans qr(arena_runs_dirty_link_t) rd_link; 14738e42d311c1844a66e8ced84551621de41e42b85Jason Evans}; 14838e42d311c1844a66e8ced84551621de41e42b85Jason Evans 149ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu/* 150ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just 151ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * like arena_chunk_map_bits_t. Two separate arrays are stored within each 152ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * chunk header in order to improve cache locality. 153ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu */ 154ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wustruct arena_chunk_map_misc_s { 155ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu /* 156ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * Linkage for run trees. There are two disjoint uses: 157ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 158ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 1) arena_t's runs_avail tree. 159ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 2) arena_run_t conceptually uses this linkage for in-use non-full 16038e42d311c1844a66e8ced84551621de41e42b85Jason Evans * runs, rather than directly embedding linkage. 161ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu */ 1620c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans rb_node(arena_chunk_map_misc_t) rb_link; 163ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 1640c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans union { 1650c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Linkage for list of dirty runs. */ 16638e42d311c1844a66e8ced84551621de41e42b85Jason Evans arena_runs_dirty_link_t rd; 167ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 1680c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Profile counters, used for large object runs. */ 169c451831264885b84f54a05e0894ad88bb30bd5dfJason Evans union { 170817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans void *prof_tctx_pun; 171817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans prof_tctx_t *prof_tctx; 172c451831264885b84f54a05e0894ad88bb30bd5dfJason Evans }; 1730c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 1740c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Small region run metadata. */ 1750c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_run_t run; 1760c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans }; 177ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu}; 178ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef rb_tree(arena_chunk_map_misc_t) arena_run_tree_t; 179ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans#endif /* JEMALLOC_ARENA_STRUCTS_A */ 180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 181ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans#ifdef JEMALLOC_ARENA_STRUCTS_B 182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Arena chunk header. */ 183e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_chunk_s { 184cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* 185ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * A pointer to the arena that owns the chunk is stored within the node. 186ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * This field as a whole is used by chunks_rtree to support both 187ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * ivsalloc() and core-based debugging. 188cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans */ 189cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans extent_node_t node; 190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1917393f44ff025ca67716fc53b68003fd65122fd97Jason Evans /* 1927393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * Map of pages within chunk that keeps track of free/large/small. The 1937393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * first map_bias entries are omitted, since the chunk header does not 1947393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * need to be tracked in the map. This omission saves a header page 1957393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * for common chunk sizes (e.g. 4 MiB). 1967393f44ff025ca67716fc53b68003fd65122fd97Jason Evans */ 197ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu arena_chunk_map_bits_t map_bits[1]; /* Dynamically sized. */ 198e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 20049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 20184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Read-only information associated with each element of arena_t's bins array 20249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * is stored separately, partly to reduce memory usage (only one copy, rather 20349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * than one per arena), but mainly to avoid false cacheline sharing. 204122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 205122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Each run has the following layout: 206122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 207122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * /--------------------\ 2080c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans * | pad? | 209122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 210122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 211122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * reg0_offset | region 0 | 212122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 213122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| \ 214122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | | 215122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | region 1 | > reg_interval 216122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | / 217122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 218122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 219122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 220122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 221122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 222122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 223122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | region nregs-1 | 224122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 225122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 226122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | alignment pad? | 227122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * \--------------------/ 228122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 229122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * reg_interval has at least the same minimum alignment as reg_size; this 230122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * preserves the alignment constraint that sa2u() depends on. Alignment pad is 231122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * either 0 or redzone_size; it is present only if needed to align reg0_offset. 23249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 23349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstruct arena_bin_info_s { 23449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Size of regions in a run for this bin's size class. */ 235817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans size_t reg_size; 23649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 237122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Redzone size. */ 238817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans size_t redzone_size; 239122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 240122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Interval between regions (reg_size + (redzone_size << 1)). */ 241817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans size_t reg_interval; 242122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 24349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total size of a run for this bin's size class. */ 244817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans size_t run_size; 24549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 24649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total number of regions in a run for this bin's size class. */ 247817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans uint32_t nregs; 24849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 24984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 25084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Metadata used to manipulate bitmaps for runs associated with this 25184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin. 25284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 253817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans bitmap_info_t bitmap_info; 25484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 25549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Offset of first region in a run for this bin's size class. */ 256817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans uint32_t reg0_offset; 25749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans}; 25849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 259e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_bin_s { 260e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 26186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * All operations on runcur, runs, and stats require that lock be 26286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. Run allocation/deallocation are protected by the arena lock, 26386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * which may be acquired while holding one or more bin locks, but not 26486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * vise versa. 26586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 266817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans malloc_mutex_t lock; 26786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 26886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current run being used to service allocations of this bin's size 270e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * class. 271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 272817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans arena_run_t *runcur; 273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Tree of non-full runs. This tree is used when looking for an 276e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * existing run when runcur is no longer usable. We choose the 277e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * non-full run that is lowest in memory; this policy tends to keep 278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * objects packed well, and it can also help reduce the number of 279e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * almost-empty chunks. 280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 281817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans arena_run_tree_t runs; 282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Bin statistics. */ 284817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans malloc_bin_stats_t stats; 285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 287e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_s { 2886109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* This arena's index within the arenas array. */ 2896109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned ind; 2906109fe07a14b7a619365977d9523db9f8b333792Jason Evans 29186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 292597632be188d2bcc135dad2145cc46ef44897aadJason Evans * Number of threads currently assigned to this arena. This field is 293597632be188d2bcc135dad2145cc46ef44897aadJason Evans * protected by arenas_lock. 294597632be188d2bcc135dad2145cc46ef44897aadJason Evans */ 295597632be188d2bcc135dad2145cc46ef44897aadJason Evans unsigned nthreads; 296597632be188d2bcc135dad2145cc46ef44897aadJason Evans 297597632be188d2bcc135dad2145cc46ef44897aadJason Evans /* 298597632be188d2bcc135dad2145cc46ef44897aadJason Evans * There are three classes of arena operations from a locking 299597632be188d2bcc135dad2145cc46ef44897aadJason Evans * perspective: 300e12eaf93dca308a426c182956197b0eeb5f2cff3Jason Evans * 1) Thread assignment (modifies nthreads) is protected by arenas_lock. 301597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 2) Bin-related operations are protected by bin locks. 302597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 3) Chunk- and run-related operations are protected by this mutex. 30386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_t lock; 305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_stats_t stats; 307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * List of tcaches for extant threads associated with this arena. 3091cb181ed632e7573fb4eab194e4d216867222d27Jason Evans * Stats from these are merged incrementally, and at exit if 3101cb181ed632e7573fb4eab194e4d216867222d27Jason Evans * opt_stats_print is enabled. 311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_head(tcache_t) tcache_ql; 313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 314d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans uint64_t prof_accumbytes; 315d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 3168a03cf039cd06f9fa6972711195055d865673966Jason Evans /* 3178a03cf039cd06f9fa6972711195055d865673966Jason Evans * PRNG state for cache index randomization of large allocation base 3188a03cf039cd06f9fa6972711195055d865673966Jason Evans * pointers. 3198a03cf039cd06f9fa6972711195055d865673966Jason Evans */ 3208a03cf039cd06f9fa6972711195055d865673966Jason Evans uint64_t offset_state; 3218a03cf039cd06f9fa6972711195055d865673966Jason Evans 322609ae595f0358157b19311b0f9f9591db7cee705Jason Evans dss_prec_t dss_prec; 323609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 324e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 325e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * In order to avoid rapid chunk allocation/deallocation when an arena 326e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * oscillates right on the cusp of needing a new chunk, cache the most 327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * recently freed chunk. The spare is left in the arena's chunk trees 328e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * until it is deleted. 329e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * There is one spare chunk per arena, rather than one spare total, in 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * order to avoid interactions between multiple threads that could make 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a single spare inadequate. 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *spare; 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 3368d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans /* Minimum ratio (log base 2) of nactive:ndirty. */ 3378d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans ssize_t lg_dirty_mult; 3388d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans 339243f7a0508bb014c2a7bf592c466a923911db234Jason Evans /* True if a thread is currently executing arena_purge_to_limit(). */ 3400a9f9a4d511e0c3343ff26e04d9592fefd96c2bcJason Evans bool purging; 3410a9f9a4d511e0c3343ff26e04d9592fefd96c2bcJason Evans 342e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evans /* Number of pages in active runs and huge regions. */ 343bc25a47ee0e2ac8e10a94d5fa070f0dbbdeb7e7eJason Evans size_t nactive; 344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current count of pages within unused runs that are potentially 347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * dirty, and for which madvise(... MADV_DONTNEED) has not been called. 348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * By tracking this, we can institute a limit on how much dirty unused 349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * memory is mapped for each arena. 350e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty; 352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 354ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * Unused dirty memory this arena manages. Dirty memory is conceptually 355738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans * tracked as an arbitrarily interleaved LRU of dirty runs and cached 356738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans * chunks, but the list linkage is actually semi-duplicated in order to 357738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans * avoid extra arena_chunk_map_misc_t space overhead. 358ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * 359ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * LRU-----------------------------------------------------------MRU 360ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * 361f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * /-- arena ---\ 362f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | 363f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | 364f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * |------------| /- chunk -\ 365f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * ...->|chunks_cache|<--------------------------->| /----\ |<--... 366f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * |------------| | |node| | 367f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | 368f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | /- run -\ /- run -\ | | | | 369f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | | | | | 370f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | | | | | 371f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * |------------| |-------| |-------| | |----| | 372f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * ...->|runs_dirty |<-->|rd |<-->|rd |<---->|rd |<----... 373f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * |------------| |-------| |-------| | |----| | 374f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | | | | | 375f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | | \----/ | 376f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | \-------/ \-------/ | | 377f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | 378f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | 379f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * \------------/ \---------/ 380ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans */ 38138e42d311c1844a66e8ced84551621de41e42b85Jason Evans arena_runs_dirty_link_t runs_dirty; 382738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans extent_node_t chunks_cache; 383070b3c3fbd90296610005c111ec6060e8bb23d31Jason Evans 384243f7a0508bb014c2a7bf592c466a923911db234Jason Evans /* 385243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * Approximate time in seconds from the creation of a set of unused 386243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * dirty pages until an equivalent set of unused dirty pages is purged 387243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * and/or reused. 388243f7a0508bb014c2a7bf592c466a923911db234Jason Evans */ 389243f7a0508bb014c2a7bf592c466a923911db234Jason Evans ssize_t decay_time; 390243f7a0508bb014c2a7bf592c466a923911db234Jason Evans /* decay_time / SMOOTHSTEP_NSTEPS. */ 3919bad07903962962de9f656d281b9b1e7e9501c87Jason Evans nstime_t decay_interval; 392243f7a0508bb014c2a7bf592c466a923911db234Jason Evans /* 393243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * Time at which the current decay interval logically started. We do 394243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * not actually advance to a new epoch until sometime after it starts 395243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * because of scheduling and computation delays, and it is even possible 396243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * to completely skip epochs. In all cases, during epoch advancement we 397243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * merge all relevant activity into the most recently recorded epoch. 398243f7a0508bb014c2a7bf592c466a923911db234Jason Evans */ 3999bad07903962962de9f656d281b9b1e7e9501c87Jason Evans nstime_t decay_epoch; 400243f7a0508bb014c2a7bf592c466a923911db234Jason Evans /* decay_deadline randomness generator. */ 401243f7a0508bb014c2a7bf592c466a923911db234Jason Evans uint64_t decay_jitter_state; 402243f7a0508bb014c2a7bf592c466a923911db234Jason Evans /* 403243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * Deadline for current epoch. This is the sum of decay_interval and 404243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * per epoch jitter which is a uniform random variable in 405243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * [0..decay_interval). Epochs always advance by precise multiples of 406243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * decay_interval, but we randomize the deadline to reduce the 407243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * likelihood of arenas purging in lockstep. 408243f7a0508bb014c2a7bf592c466a923911db234Jason Evans */ 4099bad07903962962de9f656d281b9b1e7e9501c87Jason Evans nstime_t decay_deadline; 410243f7a0508bb014c2a7bf592c466a923911db234Jason Evans /* 411243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * Number of dirty pages at beginning of current epoch. During epoch 412243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * advancement we use the delta between decay_ndirty and ndirty to 413243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * determine how many dirty pages, if any, were generated, and record 414243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * the result in decay_backlog. 415243f7a0508bb014c2a7bf592c466a923911db234Jason Evans */ 416243f7a0508bb014c2a7bf592c466a923911db234Jason Evans size_t decay_ndirty; 417243f7a0508bb014c2a7bf592c466a923911db234Jason Evans /* 418243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * Memoized result of arena_decay_backlog_npages_limit() corresponding 419243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * to the current contents of decay_backlog, i.e. the limit on how many 420243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * pages are allowed to exist for the decay epochs. 421243f7a0508bb014c2a7bf592c466a923911db234Jason Evans */ 422243f7a0508bb014c2a7bf592c466a923911db234Jason Evans size_t decay_backlog_npages_limit; 423243f7a0508bb014c2a7bf592c466a923911db234Jason Evans /* 424243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * Trailing log of how many unused dirty pages were generated during 425243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * each of the past SMOOTHSTEP_NSTEPS decay epochs, where the last 426243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * element is the most recent epoch. Corresponding epoch times are 427243f7a0508bb014c2a7bf592c466a923911db234Jason Evans * relative to decay_epoch. 428243f7a0508bb014c2a7bf592c466a923911db234Jason Evans */ 429243f7a0508bb014c2a7bf592c466a923911db234Jason Evans size_t decay_backlog[SMOOTHSTEP_NSTEPS]; 430243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 431cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* Extant huge allocations. */ 432cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans ql_head(extent_node_t) huge; 433cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* Synchronizes all huge allocation/update/deallocation. */ 434cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans malloc_mutex_t huge_mtx; 435cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans 436cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* 437cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * Trees of chunks that were previously allocated (trees differ only in 438cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * node ordering). These are used when allocating chunks, in an attempt 439cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * to re-use address space. Depending on function, different tree 440cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * orderings are needed, which is why there are two trees with the same 441cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * contents. 442cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans */ 443b49a334a645b854dbb1649f15c38d646fee66738Jason Evans extent_tree_t chunks_szad_cached; 444b49a334a645b854dbb1649f15c38d646fee66738Jason Evans extent_tree_t chunks_ad_cached; 445b49a334a645b854dbb1649f15c38d646fee66738Jason Evans extent_tree_t chunks_szad_retained; 446b49a334a645b854dbb1649f15c38d646fee66738Jason Evans extent_tree_t chunks_ad_retained; 447b49a334a645b854dbb1649f15c38d646fee66738Jason Evans 448cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans malloc_mutex_t chunks_mtx; 449cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* Cache of nodes that were allocated via base_alloc(). */ 450cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans ql_head(extent_node_t) node_cache; 451cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans malloc_mutex_t node_cache_mtx; 452cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans 453b49a334a645b854dbb1649f15c38d646fee66738Jason Evans /* User-configurable chunk hook functions. */ 454b49a334a645b854dbb1649f15c38d646fee66738Jason Evans chunk_hooks_t chunk_hooks; 455fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind 456b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans /* bins is used to store trees of free regions. */ 457b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans arena_bin_t bins[NBINS]; 4583417a304ccde61ac1f68b436ec22c03f1d6824ecDave Watson 4593417a304ccde61ac1f68b436ec22c03f1d6824ecDave Watson /* 4603417a304ccde61ac1f68b436ec22c03f1d6824ecDave Watson * Quantized address-ordered trees of this arena's available runs. The 4613417a304ccde61ac1f68b436ec22c03f1d6824ecDave Watson * trees are used for first-best-fit run allocation. 4623417a304ccde61ac1f68b436ec22c03f1d6824ecDave Watson */ 463ae45142adc12d39793c45ecac4dafad5674a4591Jason Evans arena_run_tree_t runs_avail[1]; /* Dynamically sized. */ 464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 465db927b672748994bef0df6b5f9e94fe6c1d40d02Jason Evans 466db927b672748994bef0df6b5f9e94fe6c1d40d02Jason Evans/* Used in conjunction with tsd for fast arena-related context lookup. */ 467db927b672748994bef0df6b5f9e94fe6c1d40d02Jason Evansstruct arena_tdata_s { 468db927b672748994bef0df6b5f9e94fe6c1d40d02Jason Evans arena_t *arena; 469243f7a0508bb014c2a7bf592c466a923911db234Jason Evans ticker_t decay_ticker; 470db927b672748994bef0df6b5f9e94fe6c1d40d02Jason Evans}; 471ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans#endif /* JEMALLOC_ARENA_STRUCTS_B */ 472e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_STRUCTS */ 474e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 475e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_EXTERNS 476e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4778a03cf039cd06f9fa6972711195055d865673966Jason Evansstatic const size_t large_pad = 4788a03cf039cd06f9fa6972711195055d865673966Jason Evans#ifdef JEMALLOC_CACHE_OBLIVIOUS 4798a03cf039cd06f9fa6972711195055d865673966Jason Evans PAGE 4808a03cf039cd06f9fa6972711195055d865673966Jason Evans#else 4818a03cf039cd06f9fa6972711195055d865673966Jason Evans 0 4828a03cf039cd06f9fa6972711195055d865673966Jason Evans#endif 4838a03cf039cd06f9fa6972711195055d865673966Jason Evans ; 4848a03cf039cd06f9fa6972711195055d865673966Jason Evans 485243f7a0508bb014c2a7bf592c466a923911db234Jason Evansextern purge_mode_t opt_purge; 486243f7a0508bb014c2a7bf592c466a923911db234Jason Evansextern const char *purge_mode_names[]; 4878a03cf039cd06f9fa6972711195055d865673966Jason Evansextern ssize_t opt_lg_dirty_mult; 488243f7a0508bb014c2a7bf592c466a923911db234Jason Evansextern ssize_t opt_decay_time; 489e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 490b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansextern arena_bin_info_t arena_bin_info[NBINS]; 491e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 492155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansextern size_t map_bias; /* Number of arena chunk header pages. */ 493155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansextern size_t map_misc_offset; 494155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansextern size_t arena_maxrun; /* Max run size for arenas. */ 495676df88e48ae5ab77b05d78cb511cfa2e57d277fJason Evansextern size_t large_maxclass; /* Max large size class. */ 4960da8ce1e96bedff697f7133c8cfb328390b6d11dJason Evansextern size_t run_quantize_max; /* Max run_quantize_*() input. */ 4973c4d92e82a31f652a7c77ca937a02d0185085b06Jason Evansextern unsigned nlclasses; /* Number of large size classes. */ 4983c4d92e82a31f652a7c77ca937a02d0185085b06Jason Evansextern unsigned nhclasses; /* Number of huge size classes. */ 4993c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans 500a9a46847925e38373e6a5da250c0cecb11a8277bJason Evans#ifdef JEMALLOC_JET 501a9a46847925e38373e6a5da250c0cecb11a8277bJason Evanstypedef size_t (run_quantize_t)(size_t); 502a9a46847925e38373e6a5da250c0cecb11a8277bJason Evansextern run_quantize_t *run_quantize_floor; 503a9a46847925e38373e6a5da250c0cecb11a8277bJason Evansextern run_quantize_t *run_quantize_ceil; 504a9a46847925e38373e6a5da250c0cecb11a8277bJason Evans#endif 505738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evansvoid arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, 506738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans bool cache); 507738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evansvoid arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, 508738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans bool cache); 509cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evansextent_node_t *arena_node_alloc(arena_t *arena); 510cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evansvoid arena_node_dalloc(arena_t *arena, extent_node_t *node); 5119b41ac909facf4f09bb1b637b78ba647348e572eJason Evansvoid *arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, 5129b41ac909facf4f09bb1b637b78ba647348e572eJason Evans bool *zero); 5133c4d92e82a31f652a7c77ca937a02d0185085b06Jason Evansvoid arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize); 5149b41ac909facf4f09bb1b637b78ba647348e572eJason Evansvoid arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk, 5159b41ac909facf4f09bb1b637b78ba647348e572eJason Evans size_t oldsize, size_t usize); 5169b41ac909facf4f09bb1b637b78ba647348e572eJason Evansvoid arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, 5179b41ac909facf4f09bb1b637b78ba647348e572eJason Evans size_t oldsize, size_t usize); 5189b41ac909facf4f09bb1b637b78ba647348e572eJason Evansbool arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, 5199b41ac909facf4f09bb1b637b78ba647348e572eJason Evans size_t oldsize, size_t usize, bool *zero); 5208d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evansssize_t arena_lg_dirty_mult_get(arena_t *arena); 5218d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evansbool arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult); 522243f7a0508bb014c2a7bf592c466a923911db234Jason Evansssize_t arena_decay_time_get(arena_t *arena); 523243f7a0508bb014c2a7bf592c466a923911db234Jason Evansbool arena_decay_time_set(arena_t *arena, ssize_t decay_time); 52499bd94fb65a0b6423c4efcc3e3e501179b92a4dbJason Evansvoid arena_maybe_purge(arena_t *arena); 525243f7a0508bb014c2a7bf592c466a923911db234Jason Evansvoid arena_purge(arena_t *arena, bool all); 526243f7a0508bb014c2a7bf592c466a923911db234Jason Evansvoid arena_tcache_fill_small(tsd_t *tsd, arena_t *arena, tcache_bin_t *tbin, 527d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind, uint64_t prof_accumbytes); 528122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, 529122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bool zero); 5300d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#ifdef JEMALLOC_JET 5310d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evanstypedef void (arena_redzone_corruption_t)(void *, size_t, bool, size_t, 5320d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans uint8_t); 5336b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_redzone_corruption_t *arena_redzone_corruption; 5346b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_dalloc_junk_small_t)(void *, arena_bin_info_t *); 5356b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_dalloc_junk_small_t *arena_dalloc_junk_small; 5366b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#else 537122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); 5386b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 5390d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansvoid arena_quarantine_junk_small(void *ptr, size_t usize); 540243f7a0508bb014c2a7bf592c466a923911db234Jason Evansvoid *arena_malloc_large(tsd_t *tsd, arena_t *arena, size_t size, 541578cd165812a11cd7250bfe5051cddc30ffec6e5Jason Evans szind_t ind, bool zero); 542578cd165812a11cd7250bfe5051cddc30ffec6e5Jason Evansvoid *arena_malloc_hard(tsd_t *tsd, arena_t *arena, size_t size, szind_t ind, 543578cd165812a11cd7250bfe5051cddc30ffec6e5Jason Evans bool zero, tcache_t *tcache); 54488fef7ceda6269598cef0cee8b984c8765673c27Jason Evansvoid *arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, 54588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t alignment, bool zero, tcache_t *tcache); 5460b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansvoid arena_prof_promoted(const void *ptr, size_t size); 547fc0b3b7383373d66cfed2cd4e2faa272a6868d32Jason Evansvoid arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, 548fc0b3b7383373d66cfed2cd4e2faa272a6868d32Jason Evans void *ptr, arena_chunk_map_bits_t *bitselm); 549203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 550ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind, arena_chunk_map_bits_t *bitselm); 551243f7a0508bb014c2a7bf592c466a923911db234Jason Evansvoid arena_dalloc_small(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk, 552243f7a0508bb014c2a7bf592c466a923911db234Jason Evans void *ptr, size_t pageind); 5536b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 5546b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_dalloc_junk_large_t)(void *, size_t); 5556b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_dalloc_junk_large_t *arena_dalloc_junk_large; 556fc0b3b7383373d66cfed2cd4e2faa272a6868d32Jason Evans#else 557fc0b3b7383373d66cfed2cd4e2faa272a6868d32Jason Evansvoid arena_dalloc_junk_large(void *ptr, size_t usize); 5586b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 559fc0b3b7383373d66cfed2cd4e2faa272a6868d32Jason Evansvoid arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk, 560203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans void *ptr); 561243f7a0508bb014c2a7bf592c466a923911db234Jason Evansvoid arena_dalloc_large(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk, 562243f7a0508bb014c2a7bf592c466a923911db234Jason Evans void *ptr); 5636b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 5646b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t); 5656b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_ralloc_junk_large_t *arena_ralloc_junk_large; 5666b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 567243f7a0508bb014c2a7bf592c466a923911db234Jason Evansbool arena_ralloc_no_move(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, 5688e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t extra, bool zero); 5695460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, 570560a4e1e01d3733c2f107cdb3cc3580f3ed84442Jason Evans size_t size, size_t alignment, bool zero, tcache_t *tcache); 571609ae595f0358157b19311b0f9f9591db7cee705Jason Evansdss_prec_t arena_dss_prec_get(arena_t *arena); 5724d434adb146375ad17f0d5e994ed5728d2942e3fJason Evansbool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); 5738d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evansssize_t arena_lg_dirty_mult_default_get(void); 5748d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evansbool arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult); 575243f7a0508bb014c2a7bf592c466a923911db234Jason Evansssize_t arena_decay_time_default_get(void); 576243f7a0508bb014c2a7bf592c466a923911db234Jason Evansbool arena_decay_time_default_set(ssize_t decay_time); 577562d266511053a51406e91c78eba640cb46ad9c8Jason Evansvoid arena_stats_merge(arena_t *arena, const char **dss, 578243f7a0508bb014c2a7bf592c466a923911db234Jason Evans ssize_t *lg_dirty_mult, ssize_t *decay_time, size_t *nactive, 579243f7a0508bb014c2a7bf592c466a923911db234Jason Evans size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 5803c4d92e82a31f652a7c77ca937a02d0185085b06Jason Evans malloc_large_stats_t *lstats, malloc_huge_stats_t *hstats); 5818bb3198f72fc7587dc93527f9f19fb5be52fa553Jason Evansarena_t *arena_new(unsigned ind); 5828a03cf039cd06f9fa6972711195055d865673966Jason Evansbool arena_boot(void); 5834e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_prefork(arena_t *arena); 5844e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_postfork_parent(arena_t *arena); 5854e2e3dd9cf19ed5991938a708a8b50611aa5bbf8Jason Evansvoid arena_postfork_child(arena_t *arena); 586e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 587e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_EXTERNS */ 588e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 589e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_INLINES 590e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 591e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifndef JEMALLOC_ENABLE_INLINE 592ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_chunk_map_bits_t *arena_bitselm_get(arena_chunk_t *chunk, 593ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind); 594ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_chunk_map_misc_t *arena_miscelm_get(arena_chunk_t *chunk, 595ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind); 59613b401553172942c3cc1d89c70fd965be71c1540Joshua Kahnsize_t arena_miscelm_to_pageind(const arena_chunk_map_misc_t *miscelm); 5970c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansvoid *arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm); 59838e42d311c1844a66e8ced84551621de41e42b85Jason Evansarena_chunk_map_misc_t *arena_rd_to_miscelm(arena_runs_dirty_link_t *rd); 5990c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_chunk_map_misc_t *arena_run_to_miscelm(arena_run_t *run); 600203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); 60187a02d2bb18dbcb2955541b849bc95862e864803Jason Evanssize_t arena_mapbitsp_read(size_t *mapbitsp); 602203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); 6035ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evanssize_t arena_mapbits_size_decode(size_t mapbits); 604203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, 605203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind); 606203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); 607203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); 608d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evansszind_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); 609203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); 610203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); 6118fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evanssize_t arena_mapbits_decommitted_get(arena_chunk_t *chunk, size_t pageind); 612203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); 613203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evanssize_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind); 61487a02d2bb18dbcb2955541b849bc95862e864803Jason Evansvoid arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits); 6155ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evanssize_t arena_mapbits_size_encode(size_t size); 616203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, 617203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size, size_t flags); 618203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 619203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size); 62045186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evansvoid arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, 62145186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans size_t flags); 622203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, 623203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size, size_t flags); 624203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 625d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind); 626203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, 627d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans size_t runind, szind_t binind, size_t flags); 6284581b97809e7e545c38b996870a4e7284a620bc5Jason Evansvoid arena_metadata_allocated_add(arena_t *arena, size_t size); 6294581b97809e7e545c38b996870a4e7284a620bc5Jason Evansvoid arena_metadata_allocated_sub(arena_t *arena, size_t size); 6304581b97809e7e545c38b996870a4e7284a620bc5Jason Evanssize_t arena_metadata_allocated_get(arena_t *arena); 63188c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); 63288c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); 63388c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum(arena_t *arena, uint64_t accumbytes); 634d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evansszind_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); 635d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evansszind_t arena_bin_index(arena_t *arena, arena_bin_t *bin); 63649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansunsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, 637b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans const void *ptr); 638602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansprof_tctx_t *arena_prof_tctx_get(const void *ptr); 639594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evansvoid arena_prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx); 640708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evansvoid arena_prof_tctx_reset(const void *ptr, size_t usize, 641708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans const void *old_ptr, prof_tctx_t *old_tctx); 642243f7a0508bb014c2a7bf592c466a923911db234Jason Evansvoid arena_decay_ticks(tsd_t *tsd, arena_t *arena, unsigned nticks); 643243f7a0508bb014c2a7bf592c466a923911db234Jason Evansvoid arena_decay_tick(tsd_t *tsd, arena_t *arena); 644f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wangvoid *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, szind_t ind, 645f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang bool zero, tcache_t *tcache, bool slow_path); 6464581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_t *arena_aalloc(const void *ptr); 647f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evanssize_t arena_salloc(const void *ptr, bool demote); 648f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wangvoid arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path); 64988fef7ceda6269598cef0cee8b984c8765673c27Jason Evansvoid arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache); 650e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 651e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 652e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) 653203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans# ifdef JEMALLOC_ARENA_INLINE_A 654ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan WuJEMALLOC_ALWAYS_INLINE arena_chunk_map_bits_t * 655ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_bitselm_get(arena_chunk_t *chunk, size_t pageind) 656ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu{ 657ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 658ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu assert(pageind >= map_bias); 659ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu assert(pageind < chunk_npages); 660ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 661ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return (&chunk->map_bits[pageind-map_bias]); 662ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu} 663ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 664ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan WuJEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 665ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wuarena_miscelm_get(arena_chunk_t *chunk, size_t pageind) 666203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 667203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 668203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind >= map_bias); 669203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind < chunk_npages); 670203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 671ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return ((arena_chunk_map_misc_t *)((uintptr_t)chunk + 672ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu (uintptr_t)map_misc_offset) + pageind-map_bias); 673203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 674203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 6750c5dd03e889d0269170b5db9fa872738d906eb78Jason EvansJEMALLOC_ALWAYS_INLINE size_t 67613b401553172942c3cc1d89c70fd965be71c1540Joshua Kahnarena_miscelm_to_pageind(const arena_chunk_map_misc_t *miscelm) 6770c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans{ 6780c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 6790c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans size_t pageind = ((uintptr_t)miscelm - ((uintptr_t)chunk + 6800c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans map_misc_offset)) / sizeof(arena_chunk_map_misc_t) + map_bias; 6810c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6820c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(pageind >= map_bias); 6830c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(pageind < chunk_npages); 6840c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6850c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans return (pageind); 6860c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans} 6870c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6880c5dd03e889d0269170b5db9fa872738d906eb78Jason EvansJEMALLOC_ALWAYS_INLINE void * 6890c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm) 6900c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans{ 6910c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 6920c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans size_t pageind = arena_miscelm_to_pageind(miscelm); 6930c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6940c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE))); 6950c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans} 6960c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 6970c5dd03e889d0269170b5db9fa872738d906eb78Jason EvansJEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 69838e42d311c1844a66e8ced84551621de41e42b85Jason Evansarena_rd_to_miscelm(arena_runs_dirty_link_t *rd) 69938e42d311c1844a66e8ced84551621de41e42b85Jason Evans{ 70038e42d311c1844a66e8ced84551621de41e42b85Jason Evans arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t 70138e42d311c1844a66e8ced84551621de41e42b85Jason Evans *)((uintptr_t)rd - offsetof(arena_chunk_map_misc_t, rd)); 70238e42d311c1844a66e8ced84551621de41e42b85Jason Evans 70338e42d311c1844a66e8ced84551621de41e42b85Jason Evans assert(arena_miscelm_to_pageind(miscelm) >= map_bias); 70438e42d311c1844a66e8ced84551621de41e42b85Jason Evans assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); 70538e42d311c1844a66e8ced84551621de41e42b85Jason Evans 70638e42d311c1844a66e8ced84551621de41e42b85Jason Evans return (miscelm); 70738e42d311c1844a66e8ced84551621de41e42b85Jason Evans} 70838e42d311c1844a66e8ced84551621de41e42b85Jason Evans 70938e42d311c1844a66e8ced84551621de41e42b85Jason EvansJEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 7100c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_run_to_miscelm(arena_run_t *run) 7110c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans{ 7120c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t 7130c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans *)((uintptr_t)run - offsetof(arena_chunk_map_misc_t, run)); 7140c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 7150c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(arena_miscelm_to_pageind(miscelm) >= map_bias); 7160c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); 7170c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 7180c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans return (miscelm); 7190c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans} 7200c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 72188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t * 722203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) 723203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 724203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 725ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return (&arena_bitselm_get(chunk, pageind)->bits); 726203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 727203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 72888393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 72987a02d2bb18dbcb2955541b849bc95862e864803Jason Evansarena_mapbitsp_read(size_t *mapbitsp) 73087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans{ 73187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 73287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans return (*mapbitsp); 73387a02d2bb18dbcb2955541b849bc95862e864803Jason Evans} 73487a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 73587a02d2bb18dbcb2955541b849bc95862e864803Jason EvansJEMALLOC_ALWAYS_INLINE size_t 736203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_get(arena_chunk_t *chunk, size_t pageind) 737203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 738203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 73987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans return (arena_mapbitsp_read(arena_mapbitsp_get(chunk, pageind))); 740203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 741203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 74288393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 7435ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evansarena_mapbits_size_decode(size_t mapbits) 7445ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans{ 7455ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans size_t size; 7465ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans 747b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#if CHUNK_MAP_SIZE_SHIFT > 0 748b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans size = (mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT; 749b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#elif CHUNK_MAP_SIZE_SHIFT == 0 750b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans size = mapbits & CHUNK_MAP_SIZE_MASK; 751b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#else 752b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans size = (mapbits & CHUNK_MAP_SIZE_MASK) << -CHUNK_MAP_SIZE_SHIFT; 753b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#endif 7545ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans 7555ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans return (size); 7565ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans} 7575ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans 7585ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason EvansJEMALLOC_ALWAYS_INLINE size_t 759203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) 760203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 761203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 762203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 763203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 764203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 7655ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans return (arena_mapbits_size_decode(mapbits)); 766203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 767203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 76888393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 769203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) 770203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 771203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 772203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 773203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 774203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 775203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); 7765ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans return (arena_mapbits_size_decode(mapbits)); 777203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 778203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 77988393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 780203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) 781203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 782203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 783203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 784203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 785203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 786203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans CHUNK_MAP_ALLOCATED); 7878fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans return (mapbits >> CHUNK_MAP_RUNIND_SHIFT); 788203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 789203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 790d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason EvansJEMALLOC_ALWAYS_INLINE szind_t 79180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evansarena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) 79280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans{ 79380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t mapbits; 794d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind; 79580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 79680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans mapbits = arena_mapbits_get(chunk, pageind); 79780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 79880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind < NBINS || binind == BININD_INVALID); 79980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans return (binind); 80080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans} 80180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 80288393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 803203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) 804203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 805203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 806203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 807203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 8088fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & 8098fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 810203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_DIRTY); 811203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 812203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 81388393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 814203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind) 815203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 816203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 817203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 818203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 8198fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & 8208fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 821203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_UNZEROED); 822203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 823203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 82488393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 8258fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evansarena_mapbits_decommitted_get(arena_chunk_t *chunk, size_t pageind) 8268fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans{ 8278fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans size_t mapbits; 8288fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans 8298fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans mapbits = arena_mapbits_get(chunk, pageind); 8308fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & 8318fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 8328fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans return (mapbits & CHUNK_MAP_DECOMMITTED); 8338fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans} 8348fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans 8358fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason EvansJEMALLOC_ALWAYS_INLINE size_t 836203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind) 837203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 838203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 839203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 840203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 841203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_LARGE); 842203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 843203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 84488393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 845203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind) 846203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 847203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 848203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 849203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 850203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_ALLOCATED); 851203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 852203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 85388393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 85487a02d2bb18dbcb2955541b849bc95862e864803Jason Evansarena_mapbitsp_write(size_t *mapbitsp, size_t mapbits) 85587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans{ 85687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 85787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans *mapbitsp = mapbits; 85887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans} 85987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 8605ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason EvansJEMALLOC_ALWAYS_INLINE size_t 8615ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evansarena_mapbits_size_encode(size_t size) 8625ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans{ 8635ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans size_t mapbits; 8645ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans 865b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#if CHUNK_MAP_SIZE_SHIFT > 0 866b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans mapbits = size << CHUNK_MAP_SIZE_SHIFT; 867b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#elif CHUNK_MAP_SIZE_SHIFT == 0 868b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans mapbits = size; 869b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#else 870b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans mapbits = size >> -CHUNK_MAP_SIZE_SHIFT; 871b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#endif 8725ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans 8735ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans assert((mapbits & ~CHUNK_MAP_SIZE_MASK) == 0); 8745ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans return (mapbits); 8755ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans} 8765ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans 87787a02d2bb18dbcb2955541b849bc95862e864803Jason EvansJEMALLOC_ALWAYS_INLINE void 878203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, 879203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flags) 880203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 88187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 882203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 8838fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((size & PAGE_MASK) == 0); 8848fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((flags & CHUNK_MAP_FLAGS_MASK) == flags); 8858fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags & 8868fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 8875ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | 8888fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans CHUNK_MAP_BININD_INVALID | flags); 889203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 890203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 89188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 892203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 893203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size) 894203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 89587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 89687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 897203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 8988fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((size & PAGE_MASK) == 0); 89987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 9005ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | 9015ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans (mapbits & ~CHUNK_MAP_SIZE_MASK)); 902203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 903203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 90488393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 90545186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evansarena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, size_t flags) 90645186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans{ 90745186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 90845186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans 90945186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans assert((flags & CHUNK_MAP_UNZEROED) == flags); 91045186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans arena_mapbitsp_write(mapbitsp, flags); 91145186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans} 91245186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans 91345186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason EvansJEMALLOC_ALWAYS_INLINE void 914203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, 915203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flags) 916203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 91787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 918203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 9198fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((size & PAGE_MASK) == 0); 9201f27abc1b1f3583d9c6f999374613dc5319aeb12Jason Evans assert((flags & CHUNK_MAP_FLAGS_MASK) == flags); 9211f27abc1b1f3583d9c6f999374613dc5319aeb12Jason Evans assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags & 9221f27abc1b1f3583d9c6f999374613dc5319aeb12Jason Evans (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 9235ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | 9241f27abc1b1f3583d9c6f999374613dc5319aeb12Jason Evans CHUNK_MAP_BININD_INVALID | flags | CHUNK_MAP_LARGE | 9258fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans CHUNK_MAP_ALLOCATED); 926203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 927203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 92888393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 929203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 930d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind) 931203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 93287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 93387a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 934203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 935203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind <= BININD_INVALID); 9368a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(arena_mapbits_large_size_get(chunk, pageind) == LARGE_MINCLASS + 9378a03cf039cd06f9fa6972711195055d865673966Jason Evans large_pad); 93887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) | 93987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans (binind << CHUNK_MAP_BININD_SHIFT)); 940203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 941203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 94288393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 943203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, 944d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind, size_t flags) 945203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 94687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); 947203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 948203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind < BININD_INVALID); 949203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind - runind >= map_bias); 9501f27abc1b1f3583d9c6f999374613dc5319aeb12Jason Evans assert((flags & CHUNK_MAP_UNZEROED) == flags); 9518fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans arena_mapbitsp_write(mapbitsp, (runind << CHUNK_MAP_RUNIND_SHIFT) | 9521f27abc1b1f3583d9c6f999374613dc5319aeb12Jason Evans (binind << CHUNK_MAP_BININD_SHIFT) | flags | CHUNK_MAP_ALLOCATED); 953203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 954203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 9554581b97809e7e545c38b996870a4e7284a620bc5Jason EvansJEMALLOC_INLINE void 9564581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_metadata_allocated_add(arena_t *arena, size_t size) 9574581b97809e7e545c38b996870a4e7284a620bc5Jason Evans{ 9584581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 9594581b97809e7e545c38b996870a4e7284a620bc5Jason Evans atomic_add_z(&arena->stats.metadata_allocated, size); 9604581b97809e7e545c38b996870a4e7284a620bc5Jason Evans} 9614581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 9624581b97809e7e545c38b996870a4e7284a620bc5Jason EvansJEMALLOC_INLINE void 9634581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_metadata_allocated_sub(arena_t *arena, size_t size) 9644581b97809e7e545c38b996870a4e7284a620bc5Jason Evans{ 9654581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 9664581b97809e7e545c38b996870a4e7284a620bc5Jason Evans atomic_sub_z(&arena->stats.metadata_allocated, size); 9674581b97809e7e545c38b996870a4e7284a620bc5Jason Evans} 9684581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 9694581b97809e7e545c38b996870a4e7284a620bc5Jason EvansJEMALLOC_INLINE size_t 9704581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_metadata_allocated_get(arena_t *arena) 9714581b97809e7e545c38b996870a4e7284a620bc5Jason Evans{ 9724581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 9734581b97809e7e545c38b996870a4e7284a620bc5Jason Evans return (atomic_read_z(&arena->stats.metadata_allocated)); 9744581b97809e7e545c38b996870a4e7284a620bc5Jason Evans} 9754581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 97688c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 977a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) 978a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 979a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 980a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 981a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans assert(prof_interval != 0); 982a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 983a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena->prof_accumbytes += accumbytes; 984a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans if (arena->prof_accumbytes >= prof_interval) { 985a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena->prof_accumbytes -= prof_interval; 98688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (true); 987a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans } 98888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 989a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 990a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 99188c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 992a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum_locked(arena_t *arena, uint64_t accumbytes) 993a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 994a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 995a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 996a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 9979c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(prof_interval == 0)) 99888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 99988c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (arena_prof_accum_impl(arena, accumbytes)); 1000a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 1001a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 100288c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 1003a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum(arena_t *arena, uint64_t accumbytes) 1004a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 1005a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 1006a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 1007a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 10089c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(prof_interval == 0)) 100988c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 101088c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans 101188c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans { 101288c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans bool ret; 101388c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans 101488c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans malloc_mutex_lock(&arena->lock); 101588c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans ret = arena_prof_accum_impl(arena, accumbytes); 101688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans malloc_mutex_unlock(&arena->lock); 101788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (ret); 101888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans } 1019a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 1020a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 1021d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason EvansJEMALLOC_ALWAYS_INLINE szind_t 102280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evansarena_ptr_small_binind_get(const void *ptr, size_t mapbits) 1023203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 1024d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind; 1025203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1026203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 1027203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1028203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (config_debug) { 102980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_chunk_t *chunk; 103080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_t *arena; 103180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t pageind; 103280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t actual_mapbits; 10330c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans size_t rpages_ind; 103480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_run_t *run; 103580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_bin_t *bin; 1036d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t run_binind, actual_binind; 103780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_bin_info_t *bin_info; 10380c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_map_misc_t *miscelm; 10390c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans void *rpages; 104080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 104180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind != BININD_INVALID); 104280737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind < NBINS); 104380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1044ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans arena = extent_node_arena_get(&chunk->node); 104580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 104680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans actual_mapbits = arena_mapbits_get(chunk, pageind); 1047203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(mapbits == actual_mapbits); 104880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_large_get(chunk, pageind) == 0); 104980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 10500c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, 10510c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans pageind); 10520c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans miscelm = arena_miscelm_get(chunk, rpages_ind); 10530c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans run = &miscelm->run; 1054381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans run_binind = run->binind; 1055381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans bin = &arena->bins[run_binind]; 10569e1810ca9dc4a5f5f0841b9a6c1abb4337753552Jason Evans actual_binind = (szind_t)(bin - arena->bins); 1057381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans assert(run_binind == actual_binind); 105880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans bin_info = &arena_bin_info[actual_binind]; 10590c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans rpages = arena_miscelm_to_rpages(miscelm); 10600c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(((uintptr_t)ptr - ((uintptr_t)rpages + 1061203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval 1062203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans == 0); 1063203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 1064203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1065203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (binind); 1066203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 1067155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans# endif /* JEMALLOC_ARENA_INLINE_A */ 1068203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1069155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans# ifdef JEMALLOC_ARENA_INLINE_B 1070d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason EvansJEMALLOC_INLINE szind_t 107149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_bin_index(arena_t *arena, arena_bin_t *bin) 107249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 10739e1810ca9dc4a5f5f0841b9a6c1abb4337753552Jason Evans szind_t binind = (szind_t)(bin - arena->bins); 1074b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 107549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (binind); 107649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 107749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 107881b4e6eb6f06cac048e3743787a70676f1534269Jason EvansJEMALLOC_INLINE unsigned 1079b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evansarena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) 108081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 108181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans unsigned shift, diff, regind; 1082122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans size_t interval; 10830c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 10840c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans void *rpages = arena_miscelm_to_rpages(miscelm); 108581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 108684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 108784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Freeing a pointer lower than region zero can cause assertion 108884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * failure. 108984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 10900c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert((uintptr_t)ptr >= (uintptr_t)rpages + 109184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset); 109281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 109381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 109481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * Avoid doing division with a variable divisor if possible. Using 109581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * actual division here can reduce allocator throughput by over 20%! 109681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 10970c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans diff = (unsigned)((uintptr_t)ptr - (uintptr_t)rpages - 109849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg0_offset); 109981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 110081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* Rescale (factor powers of 2 out of the numerator and denominator). */ 1101122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans interval = bin_info->reg_interval; 11029f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans shift = ffs_zu(interval) - 1; 110381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans diff >>= shift; 1104122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans interval >>= shift; 110581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 1106122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (interval == 1) { 110781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* The divisor was a power of 2. */ 110881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans regind = diff; 110981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } else { 111081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 111181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * To divide by a number D that is not a power of two we 111281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * multiply by (2^21 / D) and then right shift by 21 positions. 111381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 111481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * X / D 111581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 111681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * becomes 111781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 1118122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT 111981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 112081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * We can omit the first three elements, because we never 112181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * divide by 0, and 1 and 2 are both powers of two, which are 112281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * handled above. 112381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 112447e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) 112547e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) 1126122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans static const unsigned interval_invs[] = { 112781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(3), 112881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 112981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 113081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 113181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 113281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 113381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 113481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 113581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans }; 113681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 11379c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(interval <= ((sizeof(interval_invs) / 11389c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans sizeof(unsigned)) + 2))) { 1139122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans regind = (diff * interval_invs[interval - 3]) >> 1140122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans SIZE_INV_SHIFT; 1141122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else 1142122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans regind = diff / interval; 114381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV 114481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV_SHIFT 114581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } 1146122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(diff == regind * interval); 114749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(regind < bin_info->nregs); 114881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 114981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (regind); 115081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 115181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 1152602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason EvansJEMALLOC_INLINE prof_tctx_t * 1153602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evansarena_prof_tctx_get(const void *ptr) 115481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 1155602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *ret; 115681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans arena_chunk_t *chunk; 115781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 11587372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 115981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert(ptr != NULL); 116081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 116181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 116288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 116388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 116488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t mapbits = arena_mapbits_get(chunk, pageind); 116588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 116688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) 116788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans ret = (prof_tctx_t *)(uintptr_t)1U; 11685f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans else { 11695f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans arena_chunk_map_misc_t *elm = arena_miscelm_get(chunk, 11705f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans pageind); 1171c451831264885b84f54a05e0894ad88bb30bd5dfJason Evans ret = atomic_read_p(&elm->prof_tctx_pun); 11725f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans } 117388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 117488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans ret = huge_prof_tctx_get(ptr); 117581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 117681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (ret); 117781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 1178e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 1179e4f7846f1fd279a039ffa2a41707348187219de4Jason EvansJEMALLOC_INLINE void 1180594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evansarena_prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx) 1181e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans{ 1182e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans arena_chunk_t *chunk; 1183e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 11847372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 1185e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert(ptr != NULL); 1186e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 1187e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 118888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 118988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1190594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans 119188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1192665769357cd77b74e00a146f196fff19243b33c4Jason Evans 1193b4330b02a8a909aed71c46d2c661d69545628fb4Jason Evans if (unlikely(usize > SMALL_MAXCLASS || (uintptr_t)tctx > 1194b4330b02a8a909aed71c46d2c661d69545628fb4Jason Evans (uintptr_t)1U)) { 1195594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans arena_chunk_map_misc_t *elm; 1196594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans 1197594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans assert(arena_mapbits_large_get(chunk, pageind) != 0); 1198594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans 1199594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans elm = arena_miscelm_get(chunk, pageind); 1200c451831264885b84f54a05e0894ad88bb30bd5dfJason Evans atomic_write_p(&elm->prof_tctx_pun, tctx); 1201594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans } else { 1202594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans /* 1203594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans * tctx must always be initialized for large runs. 1204594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans * Assert that the surrounding conditional logic is 1205594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans * equivalent to checking whether ptr refers to a large 1206594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans * run. 1207594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans */ 1208594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans assert(arena_mapbits_large_get(chunk, pageind) == 0); 12095f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans } 121088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 121188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans huge_prof_tctx_set(ptr, tctx); 1212e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans} 121381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 1214708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason EvansJEMALLOC_INLINE void 1215708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evansarena_prof_tctx_reset(const void *ptr, size_t usize, const void *old_ptr, 1216708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans prof_tctx_t *old_tctx) 1217708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans{ 1218708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 1219708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans cassert(config_prof); 1220708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans assert(ptr != NULL); 1221708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 1222708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans if (unlikely(usize > SMALL_MAXCLASS || (ptr == old_ptr && 1223708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans (uintptr_t)old_tctx > (uintptr_t)1U))) { 1224708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1225708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans if (likely(chunk != ptr)) { 1226708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans size_t pageind; 1227708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans arena_chunk_map_misc_t *elm; 1228708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 1229708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 1230708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans LG_PAGE; 1231708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 1232708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 0); 1233708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans assert(arena_mapbits_large_get(chunk, pageind) != 0); 1234708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 1235708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans elm = arena_miscelm_get(chunk, pageind); 1236708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans atomic_write_p(&elm->prof_tctx_pun, 1237708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans (prof_tctx_t *)(uintptr_t)1U); 1238708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans } else 1239708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans huge_prof_tctx_reset(ptr); 1240708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans } 1241708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans} 1242708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 1243243f7a0508bb014c2a7bf592c466a923911db234Jason EvansJEMALLOC_ALWAYS_INLINE void 1244243f7a0508bb014c2a7bf592c466a923911db234Jason Evansarena_decay_ticks(tsd_t *tsd, arena_t *arena, unsigned nticks) 1245243f7a0508bb014c2a7bf592c466a923911db234Jason Evans{ 1246243f7a0508bb014c2a7bf592c466a923911db234Jason Evans ticker_t *decay_ticker; 1247243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 1248243f7a0508bb014c2a7bf592c466a923911db234Jason Evans if (unlikely(tsd == NULL)) 1249243f7a0508bb014c2a7bf592c466a923911db234Jason Evans return; 1250243f7a0508bb014c2a7bf592c466a923911db234Jason Evans decay_ticker = decay_ticker_get(tsd, arena->ind); 1251243f7a0508bb014c2a7bf592c466a923911db234Jason Evans if (unlikely(decay_ticker == NULL)) 1252243f7a0508bb014c2a7bf592c466a923911db234Jason Evans return; 1253243f7a0508bb014c2a7bf592c466a923911db234Jason Evans if (unlikely(ticker_ticks(decay_ticker, nticks))) 1254243f7a0508bb014c2a7bf592c466a923911db234Jason Evans arena_purge(arena, false); 1255243f7a0508bb014c2a7bf592c466a923911db234Jason Evans} 1256243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 1257243f7a0508bb014c2a7bf592c466a923911db234Jason EvansJEMALLOC_ALWAYS_INLINE void 1258243f7a0508bb014c2a7bf592c466a923911db234Jason Evansarena_decay_tick(tsd_t *tsd, arena_t *arena) 1259243f7a0508bb014c2a7bf592c466a923911db234Jason Evans{ 1260243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 1261243f7a0508bb014c2a7bf592c466a923911db234Jason Evans arena_decay_ticks(tsd, arena, 1); 1262243f7a0508bb014c2a7bf592c466a923911db234Jason Evans} 1263243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 126488393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void * 1265578cd165812a11cd7250bfe5051cddc30ffec6e5Jason Evansarena_malloc(tsd_t *tsd, arena_t *arena, size_t size, szind_t ind, bool zero, 1266578cd165812a11cd7250bfe5051cddc30ffec6e5Jason Evans tcache_t *tcache, bool slow_path) 1267962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans{ 1268962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1269962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans assert(size != 0); 1270962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1271f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang if (likely(tcache != NULL)) { 1272f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang if (likely(size <= SMALL_MAXCLASS)) { 1273f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang return (tcache_alloc_small(tsd, arena, tcache, size, 1274f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang ind, zero, slow_path)); 1275f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang } 1276f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang if (likely(size <= tcache_maxclass)) { 1277f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang return (tcache_alloc_large(tsd, arena, tcache, size, 1278f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang ind, zero, slow_path)); 1279f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang } 1280f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang /* (size > tcache_maxclass) case falls through. */ 1281f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang assert(size > tcache_maxclass); 1282f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang } 1283f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang 1284578cd165812a11cd7250bfe5051cddc30ffec6e5Jason Evans return (arena_malloc_hard(tsd, arena, size, ind, zero, tcache)); 1285962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans} 1286962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 12874581b97809e7e545c38b996870a4e7284a620bc5Jason EvansJEMALLOC_ALWAYS_INLINE arena_t * 12884581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_aalloc(const void *ptr) 12894581b97809e7e545c38b996870a4e7284a620bc5Jason Evans{ 12904581b97809e7e545c38b996870a4e7284a620bc5Jason Evans arena_chunk_t *chunk; 12914581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 12924581b97809e7e545c38b996870a4e7284a620bc5Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 129388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) 1294ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans return (extent_node_arena_get(&chunk->node)); 129588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans else 129688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans return (huge_aalloc(ptr)); 12974581b97809e7e545c38b996870a4e7284a620bc5Jason Evans} 12984581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 1299f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans/* Return the size of the allocation pointed to by ptr. */ 130088393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 1301f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evansarena_salloc(const void *ptr, bool demote) 1302f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans{ 1303f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans size_t ret; 1304f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans arena_chunk_t *chunk; 1305155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans size_t pageind; 1306d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind; 1307f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1308f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(ptr != NULL); 1309f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1310f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 131188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 131288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 131388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 131488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans binind = arena_mapbits_binind_get(chunk, pageind); 131588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (unlikely(binind == BININD_INVALID || (config_prof && !demote 131688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans && arena_mapbits_large_get(chunk, pageind) != 0))) { 131788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* 131888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * Large allocation. In the common case (demote), and 131988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * as this is an inline function, most callers will only 132088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * end up looking at binind to determine that ptr is a 132188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * small allocation. 132288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans */ 13238a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(config_cache_oblivious || ((uintptr_t)ptr & 13248a03cf039cd06f9fa6972711195055d865673966Jason Evans PAGE_MASK) == 0); 13258a03cf039cd06f9fa6972711195055d865673966Jason Evans ret = arena_mapbits_large_size_get(chunk, pageind) - 13268a03cf039cd06f9fa6972711195055d865673966Jason Evans large_pad; 132788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(ret != 0); 13288a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(pageind + ((ret+large_pad)>>LG_PAGE) <= 13298a03cf039cd06f9fa6972711195055d865673966Jason Evans chunk_npages); 133088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_dirty_get(chunk, pageind) == 133188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans arena_mapbits_dirty_get(chunk, 13328a03cf039cd06f9fa6972711195055d865673966Jason Evans pageind+((ret+large_pad)>>LG_PAGE)-1)); 133388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else { 133488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* 133588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * Small allocation (possibly promoted to a large 133688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * object). 133788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans */ 133888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_large_get(chunk, pageind) != 0 || 133988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans arena_ptr_small_binind_get(ptr, 134088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans arena_mapbits_get(chunk, pageind)) == binind); 134188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans ret = index2size(binind); 134288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 134388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 134488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans ret = huge_salloc(ptr); 1345f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1346f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans return (ret); 1347f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans} 1348f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 134988393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 1350f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wangarena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) 1351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1352cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans arena_chunk_t *chunk; 1353203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind, mapbits; 1354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1355e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1356cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans 1357cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 135888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 135988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 136088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans mapbits = arena_mapbits_get(chunk, pageind); 136188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 136288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { 136388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* Small allocation. */ 136488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(tcache != NULL)) { 1365d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind = arena_ptr_small_binind_get(ptr, 136688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans mapbits); 1367f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang tcache_dalloc_small(tsd, tcache, ptr, binind, 1368f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang slow_path); 136988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else { 1370243f7a0508bb014c2a7bf592c466a923911db234Jason Evans arena_dalloc_small(tsd, extent_node_arena_get( 1371ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans &chunk->node), chunk, ptr, pageind); 137288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 1373cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans } else { 137488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t size = arena_mapbits_large_size_get(chunk, 1375cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans pageind); 1376962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 13778a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(config_cache_oblivious || ((uintptr_t)ptr & 13788a03cf039cd06f9fa6972711195055d865673966Jason Evans PAGE_MASK) == 0); 1379962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 13806591ff09d80e11f36603a75b32dc6a9b81fb3d47Jason Evans if (likely(tcache != NULL) && size - large_pad <= 13816591ff09d80e11f36603a75b32dc6a9b81fb3d47Jason Evans tcache_maxclass) { 13828a03cf039cd06f9fa6972711195055d865673966Jason Evans tcache_dalloc_large(tsd, tcache, ptr, size - 1383f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang large_pad, slow_path); 13848a03cf039cd06f9fa6972711195055d865673966Jason Evans } else { 1385243f7a0508bb014c2a7bf592c466a923911db234Jason Evans arena_dalloc_large(tsd, extent_node_arena_get( 1386ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans &chunk->node), chunk, ptr); 138788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 138888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 138988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 139088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans huge_dalloc(tsd, ptr, tcache); 1391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 13924cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 13934cfe55166e0173be745c53adb0fecf50d11d1227Daniel MicayJEMALLOC_ALWAYS_INLINE void 139488fef7ceda6269598cef0cee8b984c8765673c27Jason Evansarena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache) 13954cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay{ 139688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans arena_chunk_t *chunk; 13974cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 139888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 139988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 140088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (config_prof && opt_prof) { 14019c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 14029c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans LG_PAGE; 1403ef349f3f944b9b40bdeeff6cc322ef753f1ad4beJason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 1404ef349f3f944b9b40bdeeff6cc322ef753f1ad4beJason Evans 0); 140588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (arena_mapbits_large_get(chunk, pageind) != 0) { 140688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* 140788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * Make sure to use promoted size, not request 140888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * size. 140988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans */ 141088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size = arena_mapbits_large_size_get(chunk, 14118a03cf039cd06f9fa6972711195055d865673966Jason Evans pageind) - large_pad; 141288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 14134cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay } 141488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(s2u(size) == s2u(arena_salloc(ptr, false))); 141588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans 141688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(size <= SMALL_MAXCLASS)) { 141788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* Small allocation. */ 141888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(tcache != NULL)) { 1419d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind = size2index(size); 1420ef349f3f944b9b40bdeeff6cc322ef753f1ad4beJason Evans tcache_dalloc_small(tsd, tcache, ptr, binind, 1421ef349f3f944b9b40bdeeff6cc322ef753f1ad4beJason Evans true); 142288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else { 142388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t pageind = ((uintptr_t)ptr - 142488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans (uintptr_t)chunk) >> LG_PAGE; 1425243f7a0508bb014c2a7bf592c466a923911db234Jason Evans arena_dalloc_small(tsd, extent_node_arena_get( 1426ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans &chunk->node), chunk, ptr, pageind); 142788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 142888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else { 14298a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(config_cache_oblivious || ((uintptr_t)ptr & 14308a03cf039cd06f9fa6972711195055d865673966Jason Evans PAGE_MASK) == 0); 14314cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 1432ef349f3f944b9b40bdeeff6cc322ef753f1ad4beJason Evans if (likely(tcache != NULL) && size <= tcache_maxclass) { 1433ef349f3f944b9b40bdeeff6cc322ef753f1ad4beJason Evans tcache_dalloc_large(tsd, tcache, ptr, size, 1434ef349f3f944b9b40bdeeff6cc322ef753f1ad4beJason Evans true); 1435ef349f3f944b9b40bdeeff6cc322ef753f1ad4beJason Evans } else { 1436243f7a0508bb014c2a7bf592c466a923911db234Jason Evans arena_dalloc_large(tsd, extent_node_arena_get( 1437ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans &chunk->node), chunk, ptr); 143888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 143988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 144088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 144188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans huge_dalloc(tsd, ptr, tcache); 14424cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay} 1443155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans# endif /* JEMALLOC_ARENA_INLINE_B */ 1444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1445e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_INLINES */ 1447e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 1448