1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 2e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_TYPES 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans#define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) 5155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans 647e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans/* Maximum number of regions in one run. */ 70c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans#define LG_RUN_MAXREGS (LG_PAGE - LG_TINY_MIN) 847e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) 947e57f9bdadfaf999c9dea5d126edf3a4f1b2995Jason Evans 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 11122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Minimum redzone size. Redzones may be larger than this if necessary to 12122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * preserve region alignment. 13122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans */ 14122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans#define REDZONE_MINSIZE 16 15122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 16122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans/* 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The minimum ratio of active:dirty pages per arena is computed as: 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 198d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans * (nactive >> lg_dirty_mult) >= ndirty 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 218d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans * So, supposing that lg_dirty_mult is 3, there can be no less than 8 times as 228d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans * many active pages as dirty pages. 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 24e3d13060c8a04f08764b16b003169eb205fa09ebJason Evans#define LG_DIRTY_MULT_DEFAULT 3 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 26243f7a0508bb014c2a7bf592c466a923911db234Jason Evanstypedef enum { 27243f7a0508bb014c2a7bf592c466a923911db234Jason Evans purge_mode_ratio = 0, 28243f7a0508bb014c2a7bf592c466a923911db234Jason Evans purge_mode_decay = 1, 29243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 30243f7a0508bb014c2a7bf592c466a923911db234Jason Evans purge_mode_limit = 2 31243f7a0508bb014c2a7bf592c466a923911db234Jason Evans} purge_mode_t; 32473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris/* ANDROID change */ 337d7fbe660d1fe68fa412449d6033177319c7da3eChristopher Ferris/* Use the decay mode purge method. 347d7fbe660d1fe68fa412449d6033177319c7da3eChristopher Ferris * Setting this value to zero results in performance issues because it 357d7fbe660d1fe68fa412449d6033177319c7da3eChristopher Ferris * causes purges at every free. Leave the default at zero, but zygote 367d7fbe660d1fe68fa412449d6033177319c7da3eChristopher Ferris * processes will set this to one using mallopt. This allows apps which 377d7fbe660d1fe68fa412449d6033177319c7da3eChristopher Ferris * tend to be active to benefit from the extra performance, but allow system 387d7fbe660d1fe68fa412449d6033177319c7da3eChristopher Ferris * servers to free PSS while they are sitting idle. 39473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris */ 40473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris#define PURGE_DEFAULT purge_mode_decay 41243f7a0508bb014c2a7bf592c466a923911db234Jason Evans/* Default decay time in seconds. */ 427d7fbe660d1fe68fa412449d6033177319c7da3eChristopher Ferris#define DECAY_TIME_DEFAULT 0 43473a1853074261bb265060e04deeb8efb0fe6facChristopher Ferris/* End ANDROID change */ 44243f7a0508bb014c2a7bf592c466a923911db234Jason Evans/* Number of event ticks between time checks. */ 45243f7a0508bb014c2a7bf592c466a923911db234Jason Evans#define DECAY_NTICKS_PER_UPDATE 1000 46243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 4738e42d311c1844a66e8ced84551621de41e42b85Jason Evanstypedef struct arena_runs_dirty_link_s arena_runs_dirty_link_t; 484a0dbb5ac844830ebd7f89af20203a574ce1b3daDave Watsontypedef struct arena_avail_links_s arena_avail_links_t; 490c5dd03e889d0269170b5db9fa872738d906eb78Jason Evanstypedef struct arena_run_s arena_run_t; 50ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; 51ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wutypedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_chunk_s arena_chunk_t; 5349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evanstypedef struct arena_bin_info_s arena_bin_info_t; 5494e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evanstypedef struct arena_decay_s arena_decay_t; 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_bin_s arena_bin_t; 56e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_s arena_t; 57db927b672748994bef0df6b5f9e94fe6c1d40d02Jason Evanstypedef struct arena_tdata_s arena_tdata_t; 58e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 59e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_TYPES */ 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 61e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_STRUCTS 62e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 63ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans#ifdef JEMALLOC_ARENA_STRUCTS_A 640c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansstruct arena_run_s { 65381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans /* Index of bin this run is associated with. */ 66d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind; 670c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 680c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Number of free regions in run. */ 690c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans unsigned nfree; 700c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 710c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Per region allocated/deallocated bitmap. */ 720c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans bitmap_t bitmap[BITMAP_GROUPS_MAX]; 730c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans}; 740c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 75e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Each element of the chunk map corresponds to one page within the chunk. */ 76ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wustruct arena_chunk_map_bits_s { 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Run address (or size) and various flags are stored together. The bit 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * layout looks like (assuming 32-bit system): 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 8185ae064e96387347915973dcc1ac15e553259bc9Jason Evans * ???????? ???????? ???nnnnn nnndumla 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 83e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * ? : Unallocated: Run address for first/last pages, unset for internal 84e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * pages. 8519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Small: Run page offset. 868fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * Large: Run page count for first page, unset for trailing pages. 8753bd42c1fe35c25ea299b96d546a9d0089c6f78dJason Evans * n : binind for small size class, BININD_INVALID for large size class. 88e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * d : dirty? 898ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * u : unzeroed? 908fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * m : decommitted? 91e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * l : large? 92e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a : allocated? 93e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 94e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Following are example bit patterns for the three types of runs. 95e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 96e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * p : run page offset 97e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * s : run size 98203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * n : binind for size class; large objects set these to BININD_INVALID 99e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * x : don't care 100e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * - : 0 1010b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * + : 1 1028fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * [DUMLA] : bit set 1038fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * [dumla] : bit unset 104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 10519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (clean): 1068fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sss+++++ +++dum-a 1078fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxx-Uxxx 1088fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sss+++++ +++dUm-a 10919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 11019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (dirty): 1118fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sss+++++ +++D-m-a 112203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 1138fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sss+++++ +++D-m-a 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 115dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Small: 1168fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * pppppppp pppppppp pppnnnnn nnnd---A 1178fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * pppppppp pppppppp pppnnnnn nnn----A 1188fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * pppppppp pppppppp pppnnnnn nnnd---A 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: 1218fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sss+++++ +++D--LA 122203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 1238fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * -------- -------- ---+++++ +++D--LA 1240b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 125155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * Large (sampled, size <= LARGE_MINCLASS): 1268fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sssnnnnn nnnD--LA 127155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 1288fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * -------- -------- ---+++++ +++D--LA 1290b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 130155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * Large (not sampled, size == LARGE_MINCLASS): 1318fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * ssssssss ssssssss sss+++++ +++D--LA 132155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 1338fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans * -------- -------- ---+++++ +++D--LA 134e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 135e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t bits; 1368fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_ALLOCATED ((size_t)0x01U) 1378fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_LARGE ((size_t)0x02U) 1388fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_STATE_MASK ((size_t)0x3U) 1398fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans 1408fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_DECOMMITTED ((size_t)0x04U) 1418fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_UNZEROED ((size_t)0x08U) 1428fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_DIRTY ((size_t)0x10U) 1438fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_FLAGS_MASK ((size_t)0x1cU) 1448fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans 1458fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_BININD_SHIFT 5 146203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define BININD_INVALID ((size_t)0xffU) 1478fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_BININD_MASK (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) 148203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK 1498fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans 1508fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_RUNIND_SHIFT (CHUNK_MAP_BININD_SHIFT + 8) 1518fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_SIZE_SHIFT (CHUNK_MAP_RUNIND_SHIFT - LG_PAGE) 1528fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans#define CHUNK_MAP_SIZE_MASK \ 1538fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans (~(CHUNK_MAP_BININD_MASK | CHUNK_MAP_FLAGS_MASK | CHUNK_MAP_STATE_MASK)) 154e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 155ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 15638e42d311c1844a66e8ced84551621de41e42b85Jason Evansstruct arena_runs_dirty_link_s { 15738e42d311c1844a66e8ced84551621de41e42b85Jason Evans qr(arena_runs_dirty_link_t) rd_link; 15838e42d311c1844a66e8ced84551621de41e42b85Jason Evans}; 15938e42d311c1844a66e8ced84551621de41e42b85Jason Evans 160ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu/* 161ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just 162ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * like arena_chunk_map_bits_t. Two separate arrays are stored within each 163ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * chunk header in order to improve cache locality. 164ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu */ 165ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wustruct arena_chunk_map_misc_s { 166ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu /* 167613cdc80f6b61f698b3b0c3f2d22442044473f9bJason Evans * Linkage for run heaps. There are two disjoint uses: 168ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 169613cdc80f6b61f698b3b0c3f2d22442044473f9bJason Evans * 1) arena_t's runs_avail heaps. 170ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu * 2) arena_run_t conceptually uses this linkage for in-use non-full 17138e42d311c1844a66e8ced84551621de41e42b85Jason Evans * runs, rather than directly embedding linkage. 172ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu */ 173c6a2c39404df9a3fb27735b93cf4cb3a76a2d4a7Jason Evans phn(arena_chunk_map_misc_t) ph_link; 174ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 1750c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans union { 1760c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Linkage for list of dirty runs. */ 17738e42d311c1844a66e8ced84551621de41e42b85Jason Evans arena_runs_dirty_link_t rd; 178ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 1790c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Profile counters, used for large object runs. */ 180c451831264885b84f54a05e0894ad88bb30bd5dfJason Evans union { 181817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans void *prof_tctx_pun; 182817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans prof_tctx_t *prof_tctx; 183c451831264885b84f54a05e0894ad88bb30bd5dfJason Evans }; 1840c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 1850c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans /* Small region run metadata. */ 1860c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_run_t run; 1870c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans }; 188ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu}; 189c6a2c39404df9a3fb27735b93cf4cb3a76a2d4a7Jason Evanstypedef ph(arena_chunk_map_misc_t) arena_run_heap_t; 190ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans#endif /* JEMALLOC_ARENA_STRUCTS_A */ 191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 192ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans#ifdef JEMALLOC_ARENA_STRUCTS_B 193e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Arena chunk header. */ 194e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_chunk_s { 195cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* 196ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * A pointer to the arena that owns the chunk is stored within the node. 197ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * This field as a whole is used by chunks_rtree to support both 198ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * ivsalloc() and core-based debugging. 199cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans */ 200cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans extent_node_t node; 201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2027393f44ff025ca67716fc53b68003fd65122fd97Jason Evans /* 203e98a620c59ac20b13e2de796164cc67f050ed2bfJason Evans * True if memory could be backed by transparent huge pages. This is 204e98a620c59ac20b13e2de796164cc67f050ed2bfJason Evans * only directly relevant to Linux, since it is the only supported 205e98a620c59ac20b13e2de796164cc67f050ed2bfJason Evans * platform on which jemalloc interacts with explicit transparent huge 206e98a620c59ac20b13e2de796164cc67f050ed2bfJason Evans * page controls. 207e98a620c59ac20b13e2de796164cc67f050ed2bfJason Evans */ 208e98a620c59ac20b13e2de796164cc67f050ed2bfJason Evans bool hugepage; 209e98a620c59ac20b13e2de796164cc67f050ed2bfJason Evans 210e98a620c59ac20b13e2de796164cc67f050ed2bfJason Evans /* 2117393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * Map of pages within chunk that keeps track of free/large/small. The 2127393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * first map_bias entries are omitted, since the chunk header does not 2137393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * need to be tracked in the map. This omission saves a header page 2147393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * for common chunk sizes (e.g. 4 MiB). 2157393f44ff025ca67716fc53b68003fd65122fd97Jason Evans */ 216ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu arena_chunk_map_bits_t map_bits[1]; /* Dynamically sized. */ 217e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 21949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 22084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Read-only information associated with each element of arena_t's bins array 22149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * is stored separately, partly to reduce memory usage (only one copy, rather 22249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * than one per arena), but mainly to avoid false cacheline sharing. 223122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 224122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * Each run has the following layout: 225122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 226122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * /--------------------\ 2270c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans * | pad? | 228122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 229122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 230122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * reg0_offset | region 0 | 231122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 232122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| \ 233122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | | 234122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | region 1 | > reg_interval 235122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | / 236122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 237122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 238122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 239122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | ... | 240122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 241122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 242122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | region nregs-1 | 243122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | redzone | 244122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * |--------------------| 245122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * | alignment pad? | 246122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * \--------------------/ 247122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * 248122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * reg_interval has at least the same minimum alignment as reg_size; this 249122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * preserves the alignment constraint that sa2u() depends on. Alignment pad is 250122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * either 0 or redzone_size; it is present only if needed to align reg0_offset. 25149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 25249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstruct arena_bin_info_s { 25349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Size of regions in a run for this bin's size class. */ 254817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans size_t reg_size; 25549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 256122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Redzone size. */ 257817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans size_t redzone_size; 258122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 259122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans /* Interval between regions (reg_size + (redzone_size << 1)). */ 260817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans size_t reg_interval; 261122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans 26249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total size of a run for this bin's size class. */ 263817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans size_t run_size; 26449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 26549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total number of regions in a run for this bin's size class. */ 266817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans uint32_t nregs; 26749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 26884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 26984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Metadata used to manipulate bitmaps for runs associated with this 27084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin. 27184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 272817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans bitmap_info_t bitmap_info; 27384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 27449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Offset of first region in a run for this bin's size class. */ 275817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans uint32_t reg0_offset; 27649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans}; 27749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 27894e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evansstruct arena_decay_s { 27994e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans /* 28094e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans * Approximate time in seconds from the creation of a set of unused 28194e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans * dirty pages until an equivalent set of unused dirty pages is purged 28294e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans * and/or reused. 28394e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans */ 28494e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans ssize_t time; 285d419bb09ef6700dde95c74e1f1752f81e5d15d92Jason Evans /* time / SMOOTHSTEP_NSTEPS. */ 28694e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans nstime_t interval; 28794e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans /* 28894e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans * Time at which the current decay interval logically started. We do 28994e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans * not actually advance to a new epoch until sometime after it starts 29094e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans * because of scheduling and computation delays, and it is even possible 29194e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans * to completely skip epochs. In all cases, during epoch advancement we 29294e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans * merge all relevant activity into the most recently recorded epoch. 29394e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans */ 29494e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans nstime_t epoch; 295d419bb09ef6700dde95c74e1f1752f81e5d15d92Jason Evans /* Deadline randomness generator. */ 29694e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans uint64_t jitter_state; 29794e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans /* 298d419bb09ef6700dde95c74e1f1752f81e5d15d92Jason Evans * Deadline for current epoch. This is the sum of interval and per 299d419bb09ef6700dde95c74e1f1752f81e5d15d92Jason Evans * epoch jitter which is a uniform random variable in [0..interval). 300d419bb09ef6700dde95c74e1f1752f81e5d15d92Jason Evans * Epochs always advance by precise multiples of interval, but we 301d419bb09ef6700dde95c74e1f1752f81e5d15d92Jason Evans * randomize the deadline to reduce the likelihood of arenas purging in 302d419bb09ef6700dde95c74e1f1752f81e5d15d92Jason Evans * lockstep. 30394e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans */ 30494e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans nstime_t deadline; 30594e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans /* 30694e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans * Number of dirty pages at beginning of current epoch. During epoch 307d419bb09ef6700dde95c74e1f1752f81e5d15d92Jason Evans * advancement we use the delta between arena->decay.ndirty and 308d419bb09ef6700dde95c74e1f1752f81e5d15d92Jason Evans * arena->ndirty to determine how many dirty pages, if any, were 309d419bb09ef6700dde95c74e1f1752f81e5d15d92Jason Evans * generated. 31094e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans */ 31194e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans size_t ndirty; 31294e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans /* 31394e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans * Trailing log of how many unused dirty pages were generated during 31494e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans * each of the past SMOOTHSTEP_NSTEPS decay epochs, where the last 31594e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans * element is the most recent epoch. Corresponding epoch times are 316d419bb09ef6700dde95c74e1f1752f81e5d15d92Jason Evans * relative to epoch. 31794e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans */ 31894e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans size_t backlog[SMOOTHSTEP_NSTEPS]; 31994e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans}; 32094e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans 321e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_bin_s { 322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 32386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * All operations on runcur, runs, and stats require that lock be 32486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. Run allocation/deallocation are protected by the arena lock, 32586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * which may be acquired while holding one or more bin locks, but not 32686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * vise versa. 32786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 328817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans malloc_mutex_t lock; 32986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 33086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current run being used to service allocations of this bin's size 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * class. 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 334817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans arena_run_t *runcur; 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 337613cdc80f6b61f698b3b0c3f2d22442044473f9bJason Evans * Heap of non-full runs. This heap is used when looking for an 338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * existing run when runcur is no longer usable. We choose the 339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * non-full run that is lowest in memory; this policy tends to keep 340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * objects packed well, and it can also help reduce the number of 341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * almost-empty chunks. 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 343c6a2c39404df9a3fb27735b93cf4cb3a76a2d4a7Jason Evans arena_run_heap_t runs; 344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Bin statistics. */ 346817d9030a5811f98c43b10ac53b8f17180dbc44fJason Evans malloc_bin_stats_t stats; 347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 349e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_s { 3506109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* This arena's index within the arenas array. */ 3516109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned ind; 3526109fe07a14b7a619365977d9523db9f8b333792Jason Evans 35386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 35466cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evans * Number of threads currently assigned to this arena, synchronized via 35566cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evans * atomic operations. Each thread has two distinct assignments, one for 35666cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evans * application-serving allocation, and the other for internal metadata 35766cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evans * allocation. Internal metadata must not be allocated from arenas 35866cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evans * created via the arenas.extend mallctl, because the arena.<i>.reset 35966cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evans * mallctl indiscriminately discards all allocations for the affected 36066cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evans * arena. 36166cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evans * 36266cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evans * 0: Application allocation. 36366cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evans * 1: Internal metadata allocation. 364597632be188d2bcc135dad2145cc46ef44897aadJason Evans */ 36566cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evans unsigned nthreads[2]; 366597632be188d2bcc135dad2145cc46ef44897aadJason Evans 367597632be188d2bcc135dad2145cc46ef44897aadJason Evans /* 368597632be188d2bcc135dad2145cc46ef44897aadJason Evans * There are three classes of arena operations from a locking 369597632be188d2bcc135dad2145cc46ef44897aadJason Evans * perspective: 370767d85061a6fb88ec977bbcd9b429a43aff391e6Jason Evans * 1) Thread assignment (modifies nthreads) is synchronized via atomics. 371597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 2) Bin-related operations are protected by bin locks. 372597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 3) Chunk- and run-related operations are protected by this mutex. 37386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_t lock; 375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_stats_t stats; 377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * List of tcaches for extant threads associated with this arena. 3791cb181ed632e7573fb4eab194e4d216867222d27Jason Evans * Stats from these are merged incrementally, and at exit if 3801cb181ed632e7573fb4eab194e4d216867222d27Jason Evans * opt_stats_print is enabled. 381e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_head(tcache_t) tcache_ql; 383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 384d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans uint64_t prof_accumbytes; 385d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 3868a03cf039cd06f9fa6972711195055d865673966Jason Evans /* 3878a03cf039cd06f9fa6972711195055d865673966Jason Evans * PRNG state for cache index randomization of large allocation base 3888a03cf039cd06f9fa6972711195055d865673966Jason Evans * pointers. 3898a03cf039cd06f9fa6972711195055d865673966Jason Evans */ 3905d6cb6eb66b05261cccd2b416f50ad98d1735229Jason Evans size_t offset_state; 3918a03cf039cd06f9fa6972711195055d865673966Jason Evans 392609ae595f0358157b19311b0f9f9591db7cee705Jason Evans dss_prec_t dss_prec; 393609ae595f0358157b19311b0f9f9591db7cee705Jason Evans 39419ff2cefba48d1ddab8fb52e3d78f309ca2553cfJason Evans /* Extant arena chunks. */ 39519ff2cefba48d1ddab8fb52e3d78f309ca2553cfJason Evans ql_head(extent_node_t) achunks; 39619ff2cefba48d1ddab8fb52e3d78f309ca2553cfJason Evans 3975c77af98b16a0f5b15bc807f2b323a91fe2a048bJason Evans /* Extent serial number generator state. */ 3985c77af98b16a0f5b15bc807f2b323a91fe2a048bJason Evans size_t extent_sn_next; 3995c77af98b16a0f5b15bc807f2b323a91fe2a048bJason Evans 400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 401e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * In order to avoid rapid chunk allocation/deallocation when an arena 402e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * oscillates right on the cusp of needing a new chunk, cache the most 403e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * recently freed chunk. The spare is left in the arena's chunk trees 404e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * until it is deleted. 405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 406e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * There is one spare chunk per arena, rather than one spare total, in 407e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * order to avoid interactions between multiple threads that could make 408e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a single spare inadequate. 409e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 410e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *spare; 411e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4128d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans /* Minimum ratio (log base 2) of nactive:ndirty. */ 4138d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans ssize_t lg_dirty_mult; 4148d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evans 415243f7a0508bb014c2a7bf592c466a923911db234Jason Evans /* True if a thread is currently executing arena_purge_to_limit(). */ 4160a9f9a4d511e0c3343ff26e04d9592fefd96c2bcJason Evans bool purging; 4170a9f9a4d511e0c3343ff26e04d9592fefd96c2bcJason Evans 418e2deab7a751c8080c2b2cdcfd7b11887332be1bbJason Evans /* Number of pages in active runs and huge regions. */ 419bc25a47ee0e2ac8e10a94d5fa070f0dbbdeb7e7eJason Evans size_t nactive; 420e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 421e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 422e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current count of pages within unused runs that are potentially 423e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * dirty, and for which madvise(... MADV_DONTNEED) has not been called. 424e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * By tracking this, we can institute a limit on how much dirty unused 425e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * memory is mapped for each arena. 426e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 427e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty; 428e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 429e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 430ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * Unused dirty memory this arena manages. Dirty memory is conceptually 431738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans * tracked as an arbitrarily interleaved LRU of dirty runs and cached 432738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans * chunks, but the list linkage is actually semi-duplicated in order to 433738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans * avoid extra arena_chunk_map_misc_t space overhead. 434ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * 435ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * LRU-----------------------------------------------------------MRU 436ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans * 437f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * /-- arena ---\ 438f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | 439f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | 440f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * |------------| /- chunk -\ 441f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * ...->|chunks_cache|<--------------------------->| /----\ |<--... 442f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * |------------| | |node| | 443f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | 444f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | /- run -\ /- run -\ | | | | 445f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | | | | | 446f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | | | | | 447f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * |------------| |-------| |-------| | |----| | 448f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * ...->|runs_dirty |<-->|rd |<-->|rd |<---->|rd |<----... 449f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * |------------| |-------| |-------| | |----| | 450f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | | | | | 451f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | | | | \----/ | 452f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | \-------/ \-------/ | | 453f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | 454f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * | | | | 455f5c8f37259d7697c3f850ac1e5ef63b724cf7689Jason Evans * \------------/ \---------/ 456ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans */ 45738e42d311c1844a66e8ced84551621de41e42b85Jason Evans arena_runs_dirty_link_t runs_dirty; 458738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans extent_node_t chunks_cache; 459070b3c3fbd90296610005c111ec6060e8bb23d31Jason Evans 46094e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans /* Decay-based purging state. */ 46194e7ffa9794792d2ec70269a0ab9c282a32aa2ecJason Evans arena_decay_t decay; 462243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 463cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* Extant huge allocations. */ 464cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans ql_head(extent_node_t) huge; 465cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* Synchronizes all huge allocation/update/deallocation. */ 466cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans malloc_mutex_t huge_mtx; 467cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans 468cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* 469cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * Trees of chunks that were previously allocated (trees differ only in 470cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * node ordering). These are used when allocating chunks, in an attempt 471cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * to re-use address space. Depending on function, different tree 472cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * orderings are needed, which is why there are two trees with the same 473cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans * contents. 474cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans */ 4755c77af98b16a0f5b15bc807f2b323a91fe2a048bJason Evans extent_tree_t chunks_szsnad_cached; 476b49a334a645b854dbb1649f15c38d646fee66738Jason Evans extent_tree_t chunks_ad_cached; 4775c77af98b16a0f5b15bc807f2b323a91fe2a048bJason Evans extent_tree_t chunks_szsnad_retained; 478b49a334a645b854dbb1649f15c38d646fee66738Jason Evans extent_tree_t chunks_ad_retained; 479b49a334a645b854dbb1649f15c38d646fee66738Jason Evans 480cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans malloc_mutex_t chunks_mtx; 481cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans /* Cache of nodes that were allocated via base_alloc(). */ 482cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans ql_head(extent_node_t) node_cache; 483cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans malloc_mutex_t node_cache_mtx; 484cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans 485b49a334a645b854dbb1649f15c38d646fee66738Jason Evans /* User-configurable chunk hook functions. */ 486b49a334a645b854dbb1649f15c38d646fee66738Jason Evans chunk_hooks_t chunk_hooks; 487fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28aravind 488b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans /* bins is used to store trees of free regions. */ 489b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans arena_bin_t bins[NBINS]; 4903417a304ccde61ac1f68b436ec22c03f1d6824ecDave Watson 4913417a304ccde61ac1f68b436ec22c03f1d6824ecDave Watson /* 492f193fd80cf1f99bce2bc9f5f4a8b149219965da2Jason Evans * Size-segregated address-ordered heaps of this arena's available runs, 493f193fd80cf1f99bce2bc9f5f4a8b149219965da2Jason Evans * used for first-best-fit run allocation. Runs are quantized, i.e. 494f193fd80cf1f99bce2bc9f5f4a8b149219965da2Jason Evans * they reside in the last heap which corresponds to a size class less 495f193fd80cf1f99bce2bc9f5f4a8b149219965da2Jason Evans * than or equal to the run size. 4963417a304ccde61ac1f68b436ec22c03f1d6824ecDave Watson */ 497f193fd80cf1f99bce2bc9f5f4a8b149219965da2Jason Evans arena_run_heap_t runs_avail[NPSIZES]; 498e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 499db927b672748994bef0df6b5f9e94fe6c1d40d02Jason Evans 500db927b672748994bef0df6b5f9e94fe6c1d40d02Jason Evans/* Used in conjunction with tsd for fast arena-related context lookup. */ 501db927b672748994bef0df6b5f9e94fe6c1d40d02Jason Evansstruct arena_tdata_s { 502243f7a0508bb014c2a7bf592c466a923911db234Jason Evans ticker_t decay_ticker; 503e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 504ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans#endif /* JEMALLOC_ARENA_STRUCTS_B */ 505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 506e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_STRUCTS */ 507e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 508e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_EXTERNS 509e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 5108a03cf039cd06f9fa6972711195055d865673966Jason Evansstatic const size_t large_pad = 5118a03cf039cd06f9fa6972711195055d865673966Jason Evans#ifdef JEMALLOC_CACHE_OBLIVIOUS 5128a03cf039cd06f9fa6972711195055d865673966Jason Evans PAGE 5138a03cf039cd06f9fa6972711195055d865673966Jason Evans#else 5148a03cf039cd06f9fa6972711195055d865673966Jason Evans 0 5158a03cf039cd06f9fa6972711195055d865673966Jason Evans#endif 5168a03cf039cd06f9fa6972711195055d865673966Jason Evans ; 5178a03cf039cd06f9fa6972711195055d865673966Jason Evans 518243f7a0508bb014c2a7bf592c466a923911db234Jason Evansextern purge_mode_t opt_purge; 519243f7a0508bb014c2a7bf592c466a923911db234Jason Evansextern const char *purge_mode_names[]; 5208a03cf039cd06f9fa6972711195055d865673966Jason Evansextern ssize_t opt_lg_dirty_mult; 521243f7a0508bb014c2a7bf592c466a923911db234Jason Evansextern ssize_t opt_decay_time; 522e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 523b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evansextern arena_bin_info_t arena_bin_info[NBINS]; 524e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 525155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansextern size_t map_bias; /* Number of arena chunk header pages. */ 526155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansextern size_t map_misc_offset; 527155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evansextern size_t arena_maxrun; /* Max run size for arenas. */ 528676df88e48ae5ab77b05d78cb511cfa2e57d277fJason Evansextern size_t large_maxclass; /* Max large size class. */ 5293c4d92e82a31f652a7c77ca937a02d0185085b06Jason Evansextern unsigned nlclasses; /* Number of large size classes. */ 5303c4d92e82a31f652a7c77ca937a02d0185085b06Jason Evansextern unsigned nhclasses; /* Number of huge size classes. */ 5313c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans 532a9a46847925e38373e6a5da250c0cecb11a8277bJason Evans#ifdef JEMALLOC_JET 533a9a46847925e38373e6a5da250c0cecb11a8277bJason Evanstypedef size_t (run_quantize_t)(size_t); 534a9a46847925e38373e6a5da250c0cecb11a8277bJason Evansextern run_quantize_t *run_quantize_floor; 535a9a46847925e38373e6a5da250c0cecb11a8277bJason Evansextern run_quantize_t *run_quantize_ceil; 536a9a46847925e38373e6a5da250c0cecb11a8277bJason Evans#endif 537738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evansvoid arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, 538738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans bool cache); 539738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evansvoid arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, 540738e089a2e707dbfc70286f7deeebc68e03d2347Jason Evans bool cache); 541c1e00ef2a6442d1d047950247c757821560db329Jason Evansextent_node_t *arena_node_alloc(tsdn_t *tsdn, arena_t *arena); 542c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_node_dalloc(tsdn_t *tsdn, arena_t *arena, extent_node_t *node); 543c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid *arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize, 5445c77af98b16a0f5b15bc807f2b323a91fe2a048bJason Evans size_t alignment, size_t *sn, bool *zero); 545c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, 5465c77af98b16a0f5b15bc807f2b323a91fe2a048bJason Evans size_t usize, size_t sn); 547c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, 548c1e00ef2a6442d1d047950247c757821560db329Jason Evans void *chunk, size_t oldsize, size_t usize); 549c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, 5505c77af98b16a0f5b15bc807f2b323a91fe2a048bJason Evans void *chunk, size_t oldsize, size_t usize, size_t sn); 551c1e00ef2a6442d1d047950247c757821560db329Jason Evansbool arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, 552c1e00ef2a6442d1d047950247c757821560db329Jason Evans void *chunk, size_t oldsize, size_t usize, bool *zero); 553c1e00ef2a6442d1d047950247c757821560db329Jason Evansssize_t arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena); 554c1e00ef2a6442d1d047950247c757821560db329Jason Evansbool arena_lg_dirty_mult_set(tsdn_t *tsdn, arena_t *arena, 555b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evans ssize_t lg_dirty_mult); 556c1e00ef2a6442d1d047950247c757821560db329Jason Evansssize_t arena_decay_time_get(tsdn_t *tsdn, arena_t *arena); 557c1e00ef2a6442d1d047950247c757821560db329Jason Evansbool arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time); 558c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_purge(tsdn_t *tsdn, arena_t *arena, bool all); 559c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_maybe_purge(tsdn_t *tsdn, arena_t *arena); 56019ff2cefba48d1ddab8fb52e3d78f309ca2553cfJason Evansvoid arena_reset(tsd_t *tsd, arena_t *arena); 561c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, 562c1e00ef2a6442d1d047950247c757821560db329Jason Evans tcache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes); 563122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, 564122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans bool zero); 5650d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans#ifdef JEMALLOC_JET 5660d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evanstypedef void (arena_redzone_corruption_t)(void *, size_t, bool, size_t, 5670d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evans uint8_t); 5686b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_redzone_corruption_t *arena_redzone_corruption; 5696b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_dalloc_junk_small_t)(void *, arena_bin_info_t *); 5706b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_dalloc_junk_small_t *arena_dalloc_junk_small; 5716b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#else 572122449b073bcbaa504c4f592ea2d733503c272d2Jason Evansvoid arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); 5736b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 5740d6c5d8bd0d866a0ce4ce321259cec65d6459821Jason Evansvoid arena_quarantine_junk_small(void *ptr, size_t usize); 575c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid *arena_malloc_large(tsdn_t *tsdn, arena_t *arena, szind_t ind, 57666cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evans bool zero); 577c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, 578c1e00ef2a6442d1d047950247c757821560db329Jason Evans szind_t ind, bool zero); 579c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, 58088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t alignment, bool zero, tcache_t *tcache); 581c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size); 582c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, 583b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evans arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm); 584c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 585b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evans void *ptr, size_t pageind, arena_chunk_map_bits_t *bitselm); 586c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 587243f7a0508bb014c2a7bf592c466a923911db234Jason Evans void *ptr, size_t pageind); 5886b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 5896b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_dalloc_junk_large_t)(void *, size_t); 5906b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_dalloc_junk_large_t *arena_dalloc_junk_large; 591fc0b3b7383373d66cfed2cd4e2faa272a6868d32Jason Evans#else 592fc0b3b7383373d66cfed2cd4e2faa272a6868d32Jason Evansvoid arena_dalloc_junk_large(void *ptr, size_t usize); 5936b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 594c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_dalloc_large_junked_locked(tsdn_t *tsdn, arena_t *arena, 595b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evans arena_chunk_t *chunk, void *ptr); 596c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, 597243f7a0508bb014c2a7bf592c466a923911db234Jason Evans void *ptr); 5986b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#ifdef JEMALLOC_JET 5996b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evanstypedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t); 6006b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evansextern arena_ralloc_junk_large_t *arena_ralloc_junk_large; 6016b694c4d47278cddfaaedeb7ee49fa5757e35ed5Jason Evans#endif 602c1e00ef2a6442d1d047950247c757821560db329Jason Evansbool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, 603c1e00ef2a6442d1d047950247c757821560db329Jason Evans size_t size, size_t extra, bool zero); 6045460aa6f6676c7f253bfcb75c028dfd38cae8aafJason Evansvoid *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, 605560a4e1e01d3733c2f107cdb3cc3580f3ed84442Jason Evans size_t size, size_t alignment, bool zero, tcache_t *tcache); 606c1e00ef2a6442d1d047950247c757821560db329Jason Evansdss_prec_t arena_dss_prec_get(tsdn_t *tsdn, arena_t *arena); 607c1e00ef2a6442d1d047950247c757821560db329Jason Evansbool arena_dss_prec_set(tsdn_t *tsdn, arena_t *arena, dss_prec_t dss_prec); 6088d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evansssize_t arena_lg_dirty_mult_default_get(void); 6098d6a3e8321a7767cb2ca0930b85d5d488a8cc659Jason Evansbool arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult); 610243f7a0508bb014c2a7bf592c466a923911db234Jason Evansssize_t arena_decay_time_default_get(void); 611243f7a0508bb014c2a7bf592c466a923911db234Jason Evansbool arena_decay_time_default_set(ssize_t decay_time); 612c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, 613c1e00ef2a6442d1d047950247c757821560db329Jason Evans unsigned *nthreads, const char **dss, ssize_t *lg_dirty_mult, 614c1e00ef2a6442d1d047950247c757821560db329Jason Evans ssize_t *decay_time, size_t *nactive, size_t *ndirty); 615c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, 6163c07f803aa282598451eb0664cc94717b769a5e6Jason Evans const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, 617b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evans size_t *nactive, size_t *ndirty, arena_stats_t *astats, 618b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evans malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats, 619b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evans malloc_huge_stats_t *hstats); 62066cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evansunsigned arena_nthreads_get(arena_t *arena, bool internal); 62166cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evansvoid arena_nthreads_inc(arena_t *arena, bool internal); 62266cd953514a18477eb49732e40d5c2ab5f1b12c5Jason Evansvoid arena_nthreads_dec(arena_t *arena, bool internal); 6235c77af98b16a0f5b15bc807f2b323a91fe2a048bJason Evanssize_t arena_extent_sn_next(arena_t *arena); 624c1e00ef2a6442d1d047950247c757821560db329Jason Evansarena_t *arena_new(tsdn_t *tsdn, unsigned ind); 6255d8db15db91c85d47b343cfc07fc6ea736f0de48Jason Evansvoid arena_boot(void); 626c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_prefork0(tsdn_t *tsdn, arena_t *arena); 627c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_prefork1(tsdn_t *tsdn, arena_t *arena); 628c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_prefork2(tsdn_t *tsdn, arena_t *arena); 629c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_prefork3(tsdn_t *tsdn, arena_t *arena); 630c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_postfork_parent(tsdn_t *tsdn, arena_t *arena); 631c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_postfork_child(tsdn_t *tsdn, arena_t *arena); 632e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 633e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_EXTERNS */ 634e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 635e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_INLINES 636e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 637e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifndef JEMALLOC_ENABLE_INLINE 63861a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_chunk_map_bits_t *arena_bitselm_get_mutable(arena_chunk_t *chunk, 639ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind); 64061a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansconst arena_chunk_map_bits_t *arena_bitselm_get_const( 64161a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans const arena_chunk_t *chunk, size_t pageind); 64261a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_chunk_map_misc_t *arena_miscelm_get_mutable(arena_chunk_t *chunk, 643ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu size_t pageind); 64461a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansconst arena_chunk_map_misc_t *arena_miscelm_get_const( 64561a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans const arena_chunk_t *chunk, size_t pageind); 64613b401553172942c3cc1d89c70fd965be71c1540Joshua Kahnsize_t arena_miscelm_to_pageind(const arena_chunk_map_misc_t *miscelm); 64761a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansvoid *arena_miscelm_to_rpages(const arena_chunk_map_misc_t *miscelm); 64838e42d311c1844a66e8ced84551621de41e42b85Jason Evansarena_chunk_map_misc_t *arena_rd_to_miscelm(arena_runs_dirty_link_t *rd); 6490c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_chunk_map_misc_t *arena_run_to_miscelm(arena_run_t *run); 65061a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evanssize_t *arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind); 65161a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansconst size_t *arena_mapbitsp_get_const(const arena_chunk_t *chunk, 65261a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans size_t pageind); 65361a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evanssize_t arena_mapbitsp_read(const size_t *mapbitsp); 65461a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evanssize_t arena_mapbits_get(const arena_chunk_t *chunk, size_t pageind); 6555ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evanssize_t arena_mapbits_size_decode(size_t mapbits); 65661a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evanssize_t arena_mapbits_unallocated_size_get(const arena_chunk_t *chunk, 657203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind); 65861a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evanssize_t arena_mapbits_large_size_get(const arena_chunk_t *chunk, 65961a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans size_t pageind); 66061a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evanssize_t arena_mapbits_small_runind_get(const arena_chunk_t *chunk, 66161a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans size_t pageind); 66261a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansszind_t arena_mapbits_binind_get(const arena_chunk_t *chunk, size_t pageind); 66361a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evanssize_t arena_mapbits_dirty_get(const arena_chunk_t *chunk, size_t pageind); 66461a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evanssize_t arena_mapbits_unzeroed_get(const arena_chunk_t *chunk, size_t pageind); 66561a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evanssize_t arena_mapbits_decommitted_get(const arena_chunk_t *chunk, 666203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind); 66761a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evanssize_t arena_mapbits_large_get(const arena_chunk_t *chunk, size_t pageind); 66861a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evanssize_t arena_mapbits_allocated_get(const arena_chunk_t *chunk, size_t pageind); 66987a02d2bb18dbcb2955541b849bc95862e864803Jason Evansvoid arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits); 6705ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evanssize_t arena_mapbits_size_encode(size_t size); 671203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, 672203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size, size_t flags); 673203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 674203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size); 67545186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evansvoid arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, 67645186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans size_t flags); 677203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, 678203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size, size_t flags); 679203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 680d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind); 681203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansvoid arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, 682d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans size_t runind, szind_t binind, size_t flags); 6834581b97809e7e545c38b996870a4e7284a620bc5Jason Evansvoid arena_metadata_allocated_add(arena_t *arena, size_t size); 6844581b97809e7e545c38b996870a4e7284a620bc5Jason Evansvoid arena_metadata_allocated_sub(arena_t *arena, size_t size); 6854581b97809e7e545c38b996870a4e7284a620bc5Jason Evanssize_t arena_metadata_allocated_get(arena_t *arena); 68688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); 68788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evansbool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); 688c1e00ef2a6442d1d047950247c757821560db329Jason Evansbool arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes); 689d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evansszind_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); 690d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evansszind_t arena_bin_index(arena_t *arena, arena_bin_t *bin); 69142ce80e15a5aa2ab6f2ec7e5f7c18164803f3076Jason Evanssize_t arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, 692b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans const void *ptr); 693c1e00ef2a6442d1d047950247c757821560db329Jason Evansprof_tctx_t *arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr); 694c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, 695b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evans prof_tctx_t *tctx); 696c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, 697708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans const void *old_ptr, prof_tctx_t *old_tctx); 698c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks); 699c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_decay_tick(tsdn_t *tsdn, arena_t *arena); 700c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid *arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, 701f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang bool zero, tcache_t *tcache, bool slow_path); 7024581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_t *arena_aalloc(const void *ptr); 703c1e00ef2a6442d1d047950247c757821560db329Jason Evanssize_t arena_salloc(tsdn_t *tsdn, const void *ptr, bool demote); 704c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path); 705c1e00ef2a6442d1d047950247c757821560db329Jason Evansvoid arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, 7063ef51d7f733ac6432e80fa902a779ab5b98d74f6Jason Evans bool slow_path); 707e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 708e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 709e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) 710203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans# ifdef JEMALLOC_ARENA_INLINE_A 711ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan WuJEMALLOC_ALWAYS_INLINE arena_chunk_map_bits_t * 71261a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_bitselm_get_mutable(arena_chunk_t *chunk, size_t pageind) 713ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu{ 714ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 715ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu assert(pageind >= map_bias); 716ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu assert(pageind < chunk_npages); 717ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 718ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return (&chunk->map_bits[pageind-map_bias]); 719ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu} 720ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu 72161a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason EvansJEMALLOC_ALWAYS_INLINE const arena_chunk_map_bits_t * 72261a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_bitselm_get_const(const arena_chunk_t *chunk, size_t pageind) 72361a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans{ 72461a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans 72561a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans return (arena_bitselm_get_mutable((arena_chunk_t *)chunk, pageind)); 72661a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans} 72761a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans 728ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan WuJEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 72961a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_miscelm_get_mutable(arena_chunk_t *chunk, size_t pageind) 730203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 731203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 732203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind >= map_bias); 733203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind < chunk_npages); 734203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 735ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu return ((arena_chunk_map_misc_t *)((uintptr_t)chunk + 736ff6a31d3b92b7c63446ce645341d2bbd77b67dc6Qinfan Wu (uintptr_t)map_misc_offset) + pageind-map_bias); 737203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 738203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 73961a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason EvansJEMALLOC_ALWAYS_INLINE const arena_chunk_map_misc_t * 74061a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_miscelm_get_const(const arena_chunk_t *chunk, size_t pageind) 74161a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans{ 74261a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans 74361a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans return (arena_miscelm_get_mutable((arena_chunk_t *)chunk, pageind)); 74461a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans} 74561a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans 7460c5dd03e889d0269170b5db9fa872738d906eb78Jason EvansJEMALLOC_ALWAYS_INLINE size_t 74713b401553172942c3cc1d89c70fd965be71c1540Joshua Kahnarena_miscelm_to_pageind(const arena_chunk_map_misc_t *miscelm) 7480c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans{ 7490c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 7500c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans size_t pageind = ((uintptr_t)miscelm - ((uintptr_t)chunk + 7510c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans map_misc_offset)) / sizeof(arena_chunk_map_misc_t) + map_bias; 7520c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 7530c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(pageind >= map_bias); 7540c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(pageind < chunk_npages); 7550c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 7560c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans return (pageind); 7570c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans} 7580c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 7590c5dd03e889d0269170b5db9fa872738d906eb78Jason EvansJEMALLOC_ALWAYS_INLINE void * 76061a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_miscelm_to_rpages(const arena_chunk_map_misc_t *miscelm) 7610c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans{ 7620c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); 7630c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans size_t pageind = arena_miscelm_to_pageind(miscelm); 7640c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 7650c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE))); 7660c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans} 7670c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 7680c5dd03e889d0269170b5db9fa872738d906eb78Jason EvansJEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 76938e42d311c1844a66e8ced84551621de41e42b85Jason Evansarena_rd_to_miscelm(arena_runs_dirty_link_t *rd) 77038e42d311c1844a66e8ced84551621de41e42b85Jason Evans{ 77138e42d311c1844a66e8ced84551621de41e42b85Jason Evans arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t 77238e42d311c1844a66e8ced84551621de41e42b85Jason Evans *)((uintptr_t)rd - offsetof(arena_chunk_map_misc_t, rd)); 77338e42d311c1844a66e8ced84551621de41e42b85Jason Evans 77438e42d311c1844a66e8ced84551621de41e42b85Jason Evans assert(arena_miscelm_to_pageind(miscelm) >= map_bias); 77538e42d311c1844a66e8ced84551621de41e42b85Jason Evans assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); 77638e42d311c1844a66e8ced84551621de41e42b85Jason Evans 77738e42d311c1844a66e8ced84551621de41e42b85Jason Evans return (miscelm); 77838e42d311c1844a66e8ced84551621de41e42b85Jason Evans} 77938e42d311c1844a66e8ced84551621de41e42b85Jason Evans 78038e42d311c1844a66e8ced84551621de41e42b85Jason EvansJEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * 7810c5dd03e889d0269170b5db9fa872738d906eb78Jason Evansarena_run_to_miscelm(arena_run_t *run) 7820c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans{ 7830c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t 7840c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans *)((uintptr_t)run - offsetof(arena_chunk_map_misc_t, run)); 7850c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 7860c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(arena_miscelm_to_pageind(miscelm) >= map_bias); 7870c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); 7880c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 7890c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans return (miscelm); 7900c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans} 7910c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans 79288393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t * 79361a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind) 794203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 795203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 79661a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans return (&arena_bitselm_get_mutable(chunk, pageind)->bits); 79761a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans} 79861a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans 79961a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason EvansJEMALLOC_ALWAYS_INLINE const size_t * 80061a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbitsp_get_const(const arena_chunk_t *chunk, size_t pageind) 801203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 802203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 80361a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans return (arena_mapbitsp_get_mutable((arena_chunk_t *)chunk, pageind)); 804203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 805203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 80688393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 80761a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbitsp_read(const size_t *mapbitsp) 80887a02d2bb18dbcb2955541b849bc95862e864803Jason Evans{ 80987a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 81087a02d2bb18dbcb2955541b849bc95862e864803Jason Evans return (*mapbitsp); 81187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans} 81287a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 81387a02d2bb18dbcb2955541b849bc95862e864803Jason EvansJEMALLOC_ALWAYS_INLINE size_t 81461a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbits_get(const arena_chunk_t *chunk, size_t pageind) 815203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 816203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 81761a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans return (arena_mapbitsp_read(arena_mapbitsp_get_const(chunk, pageind))); 818203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 819203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 82088393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 8215ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evansarena_mapbits_size_decode(size_t mapbits) 8225ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans{ 8235ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans size_t size; 8245ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans 825b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#if CHUNK_MAP_SIZE_SHIFT > 0 826b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans size = (mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT; 827b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#elif CHUNK_MAP_SIZE_SHIFT == 0 828b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans size = mapbits & CHUNK_MAP_SIZE_MASK; 829b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#else 830b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans size = (mapbits & CHUNK_MAP_SIZE_MASK) << -CHUNK_MAP_SIZE_SHIFT; 831b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#endif 8325ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans 8335ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans return (size); 8345ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans} 8355ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans 8365ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason EvansJEMALLOC_ALWAYS_INLINE size_t 83761a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbits_unallocated_size_get(const arena_chunk_t *chunk, size_t pageind) 838203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 839203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 840203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 841203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 842203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 8435ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans return (arena_mapbits_size_decode(mapbits)); 844203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 845203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 84688393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 84761a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbits_large_size_get(const arena_chunk_t *chunk, size_t pageind) 848203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 849203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 850203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 851203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 852203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 853203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); 8545ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans return (arena_mapbits_size_decode(mapbits)); 855203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 856203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 85788393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 85861a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbits_small_runind_get(const arena_chunk_t *chunk, size_t pageind) 859203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 860203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 861203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 862203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 863203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 864203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans CHUNK_MAP_ALLOCATED); 8658fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans return (mapbits >> CHUNK_MAP_RUNIND_SHIFT); 866203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 867203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 868d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason EvansJEMALLOC_ALWAYS_INLINE szind_t 86961a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbits_binind_get(const arena_chunk_t *chunk, size_t pageind) 87080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans{ 87180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t mapbits; 872d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind; 87380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 87480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans mapbits = arena_mapbits_get(chunk, pageind); 87580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 87680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind < NBINS || binind == BININD_INVALID); 87780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans return (binind); 87880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans} 87980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 88088393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 88161a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbits_dirty_get(const arena_chunk_t *chunk, size_t pageind) 882203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 883203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 884203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 885203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 8868fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & 8878fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 888203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_DIRTY); 889203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 890203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 89188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 89261a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbits_unzeroed_get(const arena_chunk_t *chunk, size_t pageind) 893203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 894203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 895203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 896203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 8978fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & 8988fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 899203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_UNZEROED); 900203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 901203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 90288393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 90361a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbits_decommitted_get(const arena_chunk_t *chunk, size_t pageind) 9048fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans{ 9058fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans size_t mapbits; 9068fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans 9078fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans mapbits = arena_mapbits_get(chunk, pageind); 9088fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & 9098fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 9108fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans return (mapbits & CHUNK_MAP_DECOMMITTED); 9118fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans} 9128fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans 9138fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason EvansJEMALLOC_ALWAYS_INLINE size_t 91461a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbits_large_get(const arena_chunk_t *chunk, size_t pageind) 915203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 916203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 917203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 918203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 919203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_LARGE); 920203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 921203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 92288393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 92361a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evansarena_mapbits_allocated_get(const arena_chunk_t *chunk, size_t pageind) 924203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 925203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t mapbits; 926203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 927203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans mapbits = arena_mapbits_get(chunk, pageind); 928203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (mapbits & CHUNK_MAP_ALLOCATED); 929203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 930203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 93188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 93287a02d2bb18dbcb2955541b849bc95862e864803Jason Evansarena_mapbitsp_write(size_t *mapbitsp, size_t mapbits) 93387a02d2bb18dbcb2955541b849bc95862e864803Jason Evans{ 93487a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 93587a02d2bb18dbcb2955541b849bc95862e864803Jason Evans *mapbitsp = mapbits; 93687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans} 93787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans 9385ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason EvansJEMALLOC_ALWAYS_INLINE size_t 9395ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evansarena_mapbits_size_encode(size_t size) 9405ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans{ 9415ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans size_t mapbits; 9425ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans 943b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#if CHUNK_MAP_SIZE_SHIFT > 0 944b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans mapbits = size << CHUNK_MAP_SIZE_SHIFT; 945b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#elif CHUNK_MAP_SIZE_SHIFT == 0 946b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans mapbits = size; 947b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#else 948b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans mapbits = size >> -CHUNK_MAP_SIZE_SHIFT; 949b5c2a347d7cbf1154181ccb3adc599c8bd2094c9Jason Evans#endif 9505ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans 9515ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans assert((mapbits & ~CHUNK_MAP_SIZE_MASK) == 0); 9525ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans return (mapbits); 9535ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans} 9545ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans 95587a02d2bb18dbcb2955541b849bc95862e864803Jason EvansJEMALLOC_ALWAYS_INLINE void 956203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, 957203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flags) 958203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 95961a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); 960203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 9618fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((size & PAGE_MASK) == 0); 9628fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((flags & CHUNK_MAP_FLAGS_MASK) == flags); 9638fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags & 9648fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 9655ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | 9668fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans CHUNK_MAP_BININD_INVALID | flags); 967203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 968203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 96988393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 970203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 971203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t size) 972203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 97361a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); 97487a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 975203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 9768fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((size & PAGE_MASK) == 0); 97787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 9785ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | 9795ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans (mapbits & ~CHUNK_MAP_SIZE_MASK)); 980203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 981203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 98288393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 98345186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evansarena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, size_t flags) 98445186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans{ 98561a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); 98645186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans 98745186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans assert((flags & CHUNK_MAP_UNZEROED) == flags); 98845186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans arena_mapbitsp_write(mapbitsp, flags); 98945186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans} 99045186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason Evans 99145186f0c074a5fba345d04ac1df1b77b60bb3eb6Jason EvansJEMALLOC_ALWAYS_INLINE void 992203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, 993203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t flags) 994203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 99561a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); 996203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 9978fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans assert((size & PAGE_MASK) == 0); 9981f27abc1b1f3583d9c6f999374613dc5319aeb12Jason Evans assert((flags & CHUNK_MAP_FLAGS_MASK) == flags); 9991f27abc1b1f3583d9c6f999374613dc5319aeb12Jason Evans assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags & 10001f27abc1b1f3583d9c6f999374613dc5319aeb12Jason Evans (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); 10015ef33a9f2b9f4fb56553529f7b31f4f5f57ce014Jason Evans arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | 10021f27abc1b1f3583d9c6f999374613dc5319aeb12Jason Evans CHUNK_MAP_BININD_INVALID | flags | CHUNK_MAP_LARGE | 10038fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans CHUNK_MAP_ALLOCATED); 1004203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 1005203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 100688393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 1007203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 1008d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind) 1009203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 101061a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); 101187a02d2bb18dbcb2955541b849bc95862e864803Jason Evans size_t mapbits = arena_mapbitsp_read(mapbitsp); 1012203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1013203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind <= BININD_INVALID); 10148a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(arena_mapbits_large_size_get(chunk, pageind) == LARGE_MINCLASS + 10158a03cf039cd06f9fa6972711195055d865673966Jason Evans large_pad); 101687a02d2bb18dbcb2955541b849bc95862e864803Jason Evans arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) | 101787a02d2bb18dbcb2955541b849bc95862e864803Jason Evans (binind << CHUNK_MAP_BININD_SHIFT)); 1018203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 1019203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 102088393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 1021203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evansarena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, 1022d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind, size_t flags) 1023203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 102461a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); 1025203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1026203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(binind < BININD_INVALID); 1027203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(pageind - runind >= map_bias); 10281f27abc1b1f3583d9c6f999374613dc5319aeb12Jason Evans assert((flags & CHUNK_MAP_UNZEROED) == flags); 10298fadb1a8c2d0219aded566bc5fac7d29cff9bb67Jason Evans arena_mapbitsp_write(mapbitsp, (runind << CHUNK_MAP_RUNIND_SHIFT) | 10301f27abc1b1f3583d9c6f999374613dc5319aeb12Jason Evans (binind << CHUNK_MAP_BININD_SHIFT) | flags | CHUNK_MAP_ALLOCATED); 1031203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 1032203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 10334581b97809e7e545c38b996870a4e7284a620bc5Jason EvansJEMALLOC_INLINE void 10344581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_metadata_allocated_add(arena_t *arena, size_t size) 10354581b97809e7e545c38b996870a4e7284a620bc5Jason Evans{ 10364581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 10374581b97809e7e545c38b996870a4e7284a620bc5Jason Evans atomic_add_z(&arena->stats.metadata_allocated, size); 10384581b97809e7e545c38b996870a4e7284a620bc5Jason Evans} 10394581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 10404581b97809e7e545c38b996870a4e7284a620bc5Jason EvansJEMALLOC_INLINE void 10414581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_metadata_allocated_sub(arena_t *arena, size_t size) 10424581b97809e7e545c38b996870a4e7284a620bc5Jason Evans{ 10434581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 10444581b97809e7e545c38b996870a4e7284a620bc5Jason Evans atomic_sub_z(&arena->stats.metadata_allocated, size); 10454581b97809e7e545c38b996870a4e7284a620bc5Jason Evans} 10464581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 10474581b97809e7e545c38b996870a4e7284a620bc5Jason EvansJEMALLOC_INLINE size_t 10484581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_metadata_allocated_get(arena_t *arena) 10494581b97809e7e545c38b996870a4e7284a620bc5Jason Evans{ 10504581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 10514581b97809e7e545c38b996870a4e7284a620bc5Jason Evans return (atomic_read_z(&arena->stats.metadata_allocated)); 10524581b97809e7e545c38b996870a4e7284a620bc5Jason Evans} 10534581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 105488c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 1055a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) 1056a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 1057a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 1058a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 1059a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans assert(prof_interval != 0); 1060a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 1061a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena->prof_accumbytes += accumbytes; 1062a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans if (arena->prof_accumbytes >= prof_interval) { 1063a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans arena->prof_accumbytes -= prof_interval; 106488c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (true); 1065a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans } 106688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 1067a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 1068a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 106988c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 1070a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evansarena_prof_accum_locked(arena_t *arena, uint64_t accumbytes) 1071a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 1072a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 1073a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 1074a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 10759c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(prof_interval == 0)) 107688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 107788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (arena_prof_accum_impl(arena, accumbytes)); 1078a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 1079a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 108088c222c8e91499bf5d3fba53b24222df0cda5771Jason EvansJEMALLOC_INLINE bool 1081c1e00ef2a6442d1d047950247c757821560db329Jason Evansarena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes) 1082a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans{ 1083a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 1084a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans cassert(config_prof); 1085a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 10869c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans if (likely(prof_interval == 0)) 108788c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (false); 108888c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans 108988c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans { 109088c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans bool ret; 109188c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans 1092c1e00ef2a6442d1d047950247c757821560db329Jason Evans malloc_mutex_lock(tsdn, &arena->lock); 109388c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans ret = arena_prof_accum_impl(arena, accumbytes); 1094c1e00ef2a6442d1d047950247c757821560db329Jason Evans malloc_mutex_unlock(tsdn, &arena->lock); 109588c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans return (ret); 109688c222c8e91499bf5d3fba53b24222df0cda5771Jason Evans } 1097a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans} 1098a3b3386ddde8048b9d6b54c397bb93da5e806cefJason Evans 1099d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason EvansJEMALLOC_ALWAYS_INLINE szind_t 110080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evansarena_ptr_small_binind_get(const void *ptr, size_t mapbits) 1101203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans{ 1102d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind; 1103203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1104203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 1105203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1106203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans if (config_debug) { 110780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_chunk_t *chunk; 110880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_t *arena; 110980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t pageind; 111080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans size_t actual_mapbits; 11110c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans size_t rpages_ind; 111261a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans const arena_run_t *run; 111380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_bin_t *bin; 1114d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t run_binind, actual_binind; 111580737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans arena_bin_info_t *bin_info; 111661a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans const arena_chunk_map_misc_t *miscelm; 111761a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans const void *rpages; 111880737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans 111980737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind != BININD_INVALID); 112080737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(binind < NBINS); 112180737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1122ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans arena = extent_node_arena_get(&chunk->node); 112380737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 112480737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans actual_mapbits = arena_mapbits_get(chunk, pageind); 1125203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans assert(mapbits == actual_mapbits); 112680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_large_get(chunk, pageind) == 0); 112780737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 11280c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, 11290c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans pageind); 113061a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans miscelm = arena_miscelm_get_const(chunk, rpages_ind); 11310c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans run = &miscelm->run; 1132381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans run_binind = run->binind; 1133381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans bin = &arena->bins[run_binind]; 11349e1810ca9dc4a5f5f0841b9a6c1abb4337753552Jason Evans actual_binind = (szind_t)(bin - arena->bins); 1135381c23dd9d3bf019cc4c7523a900be1e888802a7Jason Evans assert(run_binind == actual_binind); 113680737c3323dabc45232affcaeb99ac2bad6ea647Jason Evans bin_info = &arena_bin_info[actual_binind]; 11370c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans rpages = arena_miscelm_to_rpages(miscelm); 11380c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert(((uintptr_t)ptr - ((uintptr_t)rpages + 1139203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval 1140203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans == 0); 1141203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans } 1142203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1143203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans return (binind); 1144203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans} 1145155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans# endif /* JEMALLOC_ARENA_INLINE_A */ 1146203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans 1147155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans# ifdef JEMALLOC_ARENA_INLINE_B 1148d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason EvansJEMALLOC_INLINE szind_t 114949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_bin_index(arena_t *arena, arena_bin_t *bin) 115049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 11519e1810ca9dc4a5f5f0841b9a6c1abb4337753552Jason Evans szind_t binind = (szind_t)(bin - arena->bins); 1152b172610317babc7f365584ddd7fdaf4eb8d9d04cJason Evans assert(binind < NBINS); 115349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (binind); 115449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 115549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 115642ce80e15a5aa2ab6f2ec7e5f7c18164803f3076Jason EvansJEMALLOC_INLINE size_t 1157b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evansarena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) 115881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 115942ce80e15a5aa2ab6f2ec7e5f7c18164803f3076Jason Evans size_t diff, interval, shift, regind; 11600c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); 11610c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans void *rpages = arena_miscelm_to_rpages(miscelm); 116281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 116384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 116484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Freeing a pointer lower than region zero can cause assertion 116584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * failure. 116684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 11670c5dd03e889d0269170b5db9fa872738d906eb78Jason Evans assert((uintptr_t)ptr >= (uintptr_t)rpages + 116884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset); 116981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 117081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 117181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * Avoid doing division with a variable divisor if possible. Using 117281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * actual division here can reduce allocator throughput by over 20%! 117381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 117442ce80e15a5aa2ab6f2ec7e5f7c18164803f3076Jason Evans diff = (size_t)((uintptr_t)ptr - (uintptr_t)rpages - 117549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg0_offset); 117681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 117781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* Rescale (factor powers of 2 out of the numerator and denominator). */ 1178122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans interval = bin_info->reg_interval; 11799f4ee6034c3ac6a8c8b5f9a0d76822fb2fd90c41Jason Evans shift = ffs_zu(interval) - 1; 118081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans diff >>= shift; 1181122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans interval >>= shift; 118281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 1183122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans if (interval == 1) { 118481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* The divisor was a power of 2. */ 118581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans regind = diff; 118681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } else { 118781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 118881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * To divide by a number D that is not a power of two we 118981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * multiply by (2^21 / D) and then right shift by 21 positions. 119081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 119181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * X / D 119281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 119381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * becomes 119481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 1195122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT 119681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 119781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * We can omit the first three elements, because we never 119881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * divide by 0, and 1 and 2 are both powers of two, which are 119981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * handled above. 120081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 120142ce80e15a5aa2ab6f2ec7e5f7c18164803f3076Jason Evans#define SIZE_INV_SHIFT ((sizeof(size_t) << 3) - LG_RUN_MAXREGS) 120242ce80e15a5aa2ab6f2ec7e5f7c18164803f3076Jason Evans#define SIZE_INV(s) (((ZU(1) << SIZE_INV_SHIFT) / (s)) + 1) 120342ce80e15a5aa2ab6f2ec7e5f7c18164803f3076Jason Evans static const size_t interval_invs[] = { 120481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(3), 120581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 120681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 120781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 120881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 120981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 121081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 121181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 121281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans }; 121381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 121442ce80e15a5aa2ab6f2ec7e5f7c18164803f3076Jason Evans if (likely(interval <= ((sizeof(interval_invs) / sizeof(size_t)) 121542ce80e15a5aa2ab6f2ec7e5f7c18164803f3076Jason Evans + 2))) { 1216122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans regind = (diff * interval_invs[interval - 3]) >> 1217122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans SIZE_INV_SHIFT; 1218122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans } else 1219122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans regind = diff / interval; 122081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV 122181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV_SHIFT 122281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } 1223122449b073bcbaa504c4f592ea2d733503c272d2Jason Evans assert(diff == regind * interval); 122449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(regind < bin_info->nregs); 122581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 122681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (regind); 122781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 122881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 1229602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason EvansJEMALLOC_INLINE prof_tctx_t * 1230c1e00ef2a6442d1d047950247c757821560db329Jason Evansarena_prof_tctx_get(tsdn_t *tsdn, const void *ptr) 123181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 1232602c8e0971160e4b85b08b16cf8a2375aa24bc04Jason Evans prof_tctx_t *ret; 123381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans arena_chunk_t *chunk; 123481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 12357372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 123681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert(ptr != NULL); 123781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 123881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 123988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 124088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 124188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t mapbits = arena_mapbits_get(chunk, pageind); 124288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 124388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) 124488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans ret = (prof_tctx_t *)(uintptr_t)1U; 12455f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans else { 124661a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans arena_chunk_map_misc_t *elm = 124761a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans arena_miscelm_get_mutable(chunk, pageind); 1248c451831264885b84f54a05e0894ad88bb30bd5dfJason Evans ret = atomic_read_p(&elm->prof_tctx_pun); 12495f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans } 125088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 1251c1e00ef2a6442d1d047950247c757821560db329Jason Evans ret = huge_prof_tctx_get(tsdn, ptr); 125281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 125381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (ret); 125481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 1255e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 1256e4f7846f1fd279a039ffa2a41707348187219de4Jason EvansJEMALLOC_INLINE void 1257c1e00ef2a6442d1d047950247c757821560db329Jason Evansarena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, 1258b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evans prof_tctx_t *tctx) 1259e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans{ 1260e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans arena_chunk_t *chunk; 1261e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 12627372b15a31c63ac5cb9ed8aeabc2a0a3c005e8bfJason Evans cassert(config_prof); 1263e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert(ptr != NULL); 1264e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 1265e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 126688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 126788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 1268594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans 126988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 1270665769357cd77b74e00a146f196fff19243b33c4Jason Evans 1271b4330b02a8a909aed71c46d2c661d69545628fb4Jason Evans if (unlikely(usize > SMALL_MAXCLASS || (uintptr_t)tctx > 1272b4330b02a8a909aed71c46d2c661d69545628fb4Jason Evans (uintptr_t)1U)) { 1273594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans arena_chunk_map_misc_t *elm; 1274594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans 1275594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans assert(arena_mapbits_large_get(chunk, pageind) != 0); 1276594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans 127761a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans elm = arena_miscelm_get_mutable(chunk, pageind); 1278c451831264885b84f54a05e0894ad88bb30bd5dfJason Evans atomic_write_p(&elm->prof_tctx_pun, tctx); 1279594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans } else { 1280594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans /* 1281594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans * tctx must always be initialized for large runs. 1282594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans * Assert that the surrounding conditional logic is 1283594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans * equivalent to checking whether ptr refers to a large 1284594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans * run. 1285594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans */ 1286594c759f37c301d0245dc2accf4d4aaf9d202819Jason Evans assert(arena_mapbits_large_get(chunk, pageind) == 0); 12875f7140b045136232b1bbe66fcf2a7f63d08682a1Jason Evans } 128888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 1289c1e00ef2a6442d1d047950247c757821560db329Jason Evans huge_prof_tctx_set(tsdn, ptr, tctx); 1290e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans} 129181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 1292708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason EvansJEMALLOC_INLINE void 1293c1e00ef2a6442d1d047950247c757821560db329Jason Evansarena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, 1294b2c0d6322d2307458ae2b28545f8a5c9903d7ef5Jason Evans const void *old_ptr, prof_tctx_t *old_tctx) 1295708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans{ 1296708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 1297708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans cassert(config_prof); 1298708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans assert(ptr != NULL); 1299708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 1300708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans if (unlikely(usize > SMALL_MAXCLASS || (ptr == old_ptr && 1301708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans (uintptr_t)old_tctx > (uintptr_t)1U))) { 1302708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 1303708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans if (likely(chunk != ptr)) { 1304708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans size_t pageind; 1305708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans arena_chunk_map_misc_t *elm; 1306708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 1307708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 1308708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans LG_PAGE; 1309708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 1310708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 0); 1311708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans assert(arena_mapbits_large_get(chunk, pageind) != 0); 1312708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 131361a6dfcd5fd89d21f04c99fabaf7269d05f61adfJason Evans elm = arena_miscelm_get_mutable(chunk, pageind); 1314708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans atomic_write_p(&elm->prof_tctx_pun, 1315708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans (prof_tctx_t *)(uintptr_t)1U); 1316708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans } else 1317c1e00ef2a6442d1d047950247c757821560db329Jason Evans huge_prof_tctx_reset(tsdn, ptr); 1318708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans } 1319708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans} 1320708ed79834fc3b8e5b14dbb0128a0ebfce63a38fJason Evans 1321243f7a0508bb014c2a7bf592c466a923911db234Jason EvansJEMALLOC_ALWAYS_INLINE void 1322c1e00ef2a6442d1d047950247c757821560db329Jason Evansarena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) 1323243f7a0508bb014c2a7bf592c466a923911db234Jason Evans{ 1324c1e00ef2a6442d1d047950247c757821560db329Jason Evans tsd_t *tsd; 1325243f7a0508bb014c2a7bf592c466a923911db234Jason Evans ticker_t *decay_ticker; 1326243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 1327c1e00ef2a6442d1d047950247c757821560db329Jason Evans if (unlikely(tsdn_null(tsdn))) 1328243f7a0508bb014c2a7bf592c466a923911db234Jason Evans return; 1329c1e00ef2a6442d1d047950247c757821560db329Jason Evans tsd = tsdn_tsd(tsdn); 1330243f7a0508bb014c2a7bf592c466a923911db234Jason Evans decay_ticker = decay_ticker_get(tsd, arena->ind); 1331243f7a0508bb014c2a7bf592c466a923911db234Jason Evans if (unlikely(decay_ticker == NULL)) 1332243f7a0508bb014c2a7bf592c466a923911db234Jason Evans return; 1333243f7a0508bb014c2a7bf592c466a923911db234Jason Evans if (unlikely(ticker_ticks(decay_ticker, nticks))) 1334c1e00ef2a6442d1d047950247c757821560db329Jason Evans arena_purge(tsdn, arena, false); 1335243f7a0508bb014c2a7bf592c466a923911db234Jason Evans} 1336243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 1337243f7a0508bb014c2a7bf592c466a923911db234Jason EvansJEMALLOC_ALWAYS_INLINE void 1338c1e00ef2a6442d1d047950247c757821560db329Jason Evansarena_decay_tick(tsdn_t *tsdn, arena_t *arena) 1339243f7a0508bb014c2a7bf592c466a923911db234Jason Evans{ 1340243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 1341c1e00ef2a6442d1d047950247c757821560db329Jason Evans arena_decay_ticks(tsdn, arena, 1); 1342243f7a0508bb014c2a7bf592c466a923911db234Jason Evans} 1343243f7a0508bb014c2a7bf592c466a923911db234Jason Evans 134488393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void * 1345c1e00ef2a6442d1d047950247c757821560db329Jason Evansarena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero, 1346578cd165812a11cd7250bfe5051cddc30ffec6e5Jason Evans tcache_t *tcache, bool slow_path) 1347962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans{ 1348962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1349c1e00ef2a6442d1d047950247c757821560db329Jason Evans assert(!tsdn_null(tsdn) || tcache == NULL); 1350962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans assert(size != 0); 1351962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 1352f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang if (likely(tcache != NULL)) { 1353f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang if (likely(size <= SMALL_MAXCLASS)) { 1354c1e00ef2a6442d1d047950247c757821560db329Jason Evans return (tcache_alloc_small(tsdn_tsd(tsdn), arena, 1355c1e00ef2a6442d1d047950247c757821560db329Jason Evans tcache, size, ind, zero, slow_path)); 1356f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang } 1357f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang if (likely(size <= tcache_maxclass)) { 1358c1e00ef2a6442d1d047950247c757821560db329Jason Evans return (tcache_alloc_large(tsdn_tsd(tsdn), arena, 1359c1e00ef2a6442d1d047950247c757821560db329Jason Evans tcache, size, ind, zero, slow_path)); 1360f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang } 1361f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang /* (size > tcache_maxclass) case falls through. */ 1362f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang assert(size > tcache_maxclass); 1363f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang } 1364f4a0f32d340985de477bbe329ecdaecd69ed1055Qi Wang 1365c1e00ef2a6442d1d047950247c757821560db329Jason Evans return (arena_malloc_hard(tsdn, arena, size, ind, zero)); 1366962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans} 1367962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 13684581b97809e7e545c38b996870a4e7284a620bc5Jason EvansJEMALLOC_ALWAYS_INLINE arena_t * 13694581b97809e7e545c38b996870a4e7284a620bc5Jason Evansarena_aalloc(const void *ptr) 13704581b97809e7e545c38b996870a4e7284a620bc5Jason Evans{ 13714581b97809e7e545c38b996870a4e7284a620bc5Jason Evans arena_chunk_t *chunk; 13724581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 13734581b97809e7e545c38b996870a4e7284a620bc5Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 137488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) 1375ee41ad409a43d12900a5a3108f6c14f84e4eb0ebJason Evans return (extent_node_arena_get(&chunk->node)); 137688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans else 137788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans return (huge_aalloc(ptr)); 13784581b97809e7e545c38b996870a4e7284a620bc5Jason Evans} 13794581b97809e7e545c38b996870a4e7284a620bc5Jason Evans 1380f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans/* Return the size of the allocation pointed to by ptr. */ 138188393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE size_t 1382c1e00ef2a6442d1d047950247c757821560db329Jason Evansarena_salloc(tsdn_t *tsdn, const void *ptr, bool demote) 1383f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans{ 1384f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans size_t ret; 1385f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans arena_chunk_t *chunk; 1386155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans size_t pageind; 1387d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind; 1388f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1389f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans assert(ptr != NULL); 1390f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1391f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 139288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 139388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 139488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 139588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans binind = arena_mapbits_binind_get(chunk, pageind); 139688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (unlikely(binind == BININD_INVALID || (config_prof && !demote 139788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans && arena_mapbits_large_get(chunk, pageind) != 0))) { 139888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* 139988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * Large allocation. In the common case (demote), and 140088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * as this is an inline function, most callers will only 140188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * end up looking at binind to determine that ptr is a 140288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * small allocation. 140388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans */ 14048a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(config_cache_oblivious || ((uintptr_t)ptr & 14058a03cf039cd06f9fa6972711195055d865673966Jason Evans PAGE_MASK) == 0); 14068a03cf039cd06f9fa6972711195055d865673966Jason Evans ret = arena_mapbits_large_size_get(chunk, pageind) - 14078a03cf039cd06f9fa6972711195055d865673966Jason Evans large_pad; 140888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(ret != 0); 14098a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(pageind + ((ret+large_pad)>>LG_PAGE) <= 14108a03cf039cd06f9fa6972711195055d865673966Jason Evans chunk_npages); 141188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_dirty_get(chunk, pageind) == 141288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans arena_mapbits_dirty_get(chunk, 14138a03cf039cd06f9fa6972711195055d865673966Jason Evans pageind+((ret+large_pad)>>LG_PAGE)-1)); 141488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else { 141588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* 141688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * Small allocation (possibly promoted to a large 141788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * object). 141888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans */ 141988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_large_get(chunk, pageind) != 0 || 142088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans arena_ptr_small_binind_get(ptr, 142188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans arena_mapbits_get(chunk, pageind)) == binind); 142288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans ret = index2size(binind); 142388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 142488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 1425c1e00ef2a6442d1d047950247c757821560db329Jason Evans ret = huge_salloc(tsdn, ptr); 1426f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 1427f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans return (ret); 1428f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans} 1429f7088e6c992d079bc3162e0c48ed4dc5def6d263Jason Evans 143088393cb0eb9a046000d20809809d4adac11957abJason EvansJEMALLOC_ALWAYS_INLINE void 1431c1e00ef2a6442d1d047950247c757821560db329Jason Evansarena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) 1432e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 1433cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans arena_chunk_t *chunk; 1434203484e2ea267e068a68fd2922263f0ff1d5ac6fJason Evans size_t pageind, mapbits; 1435e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1436c1e00ef2a6442d1d047950247c757821560db329Jason Evans assert(!tsdn_null(tsdn) || tcache == NULL); 1437e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 1438cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans 1439cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 144088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 144188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 14426f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris#if defined(__ANDROID__) 14436f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris /* Verify the ptr is actually in the chunk. */ 14446f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris if (unlikely(pageind < map_bias || pageind >= chunk_npages)) { 14456f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris __libc_fatal("Invalid address %p passed to free: invalid page index", ptr); 14466f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris } 14476f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris#endif 144888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans mapbits = arena_mapbits_get(chunk, pageind); 144988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 14506f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris#if defined(__ANDROID__) 14516f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris /* Verify the ptr has been allocated. */ 14526f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris if (unlikely((mapbits & CHUNK_MAP_ALLOCATED) == 0)) { 14536f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris __libc_fatal("Invalid address %p passed to free: value not allocated", ptr); 14546f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris } 14556f50cbc975f782b7576ed699a7c2b123655d1b95Christopher Ferris#endif 145688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { 145788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* Small allocation. */ 145888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(tcache != NULL)) { 1459d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind = arena_ptr_small_binind_get(ptr, 146088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans mapbits); 1461c1e00ef2a6442d1d047950247c757821560db329Jason Evans tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, 1462c1e00ef2a6442d1d047950247c757821560db329Jason Evans binind, slow_path); 146388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else { 1464c1e00ef2a6442d1d047950247c757821560db329Jason Evans arena_dalloc_small(tsdn, 1465c1e00ef2a6442d1d047950247c757821560db329Jason Evans extent_node_arena_get(&chunk->node), chunk, 1466c1e00ef2a6442d1d047950247c757821560db329Jason Evans ptr, pageind); 146788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 1468cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans } else { 146988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t size = arena_mapbits_large_size_get(chunk, 1470cbf3a6d70371d2390b8b0e76814e04cc6088002cJason Evans pageind); 1471962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 14728a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(config_cache_oblivious || ((uintptr_t)ptr & 14738a03cf039cd06f9fa6972711195055d865673966Jason Evans PAGE_MASK) == 0); 1474962463d9b57bcc65de2fa108a691b4183b9b2fafJason Evans 14756591ff09d80e11f36603a75b32dc6a9b81fb3d47Jason Evans if (likely(tcache != NULL) && size - large_pad <= 14766591ff09d80e11f36603a75b32dc6a9b81fb3d47Jason Evans tcache_maxclass) { 1477c1e00ef2a6442d1d047950247c757821560db329Jason Evans tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, 1478c1e00ef2a6442d1d047950247c757821560db329Jason Evans size - large_pad, slow_path); 14798a03cf039cd06f9fa6972711195055d865673966Jason Evans } else { 1480c1e00ef2a6442d1d047950247c757821560db329Jason Evans arena_dalloc_large(tsdn, 1481c1e00ef2a6442d1d047950247c757821560db329Jason Evans extent_node_arena_get(&chunk->node), chunk, 1482c1e00ef2a6442d1d047950247c757821560db329Jason Evans ptr); 148388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 148488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 148588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 1486c1e00ef2a6442d1d047950247c757821560db329Jason Evans huge_dalloc(tsdn, ptr); 1487e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 14884cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 14894cfe55166e0173be745c53adb0fecf50d11d1227Daniel MicayJEMALLOC_ALWAYS_INLINE void 1490c1e00ef2a6442d1d047950247c757821560db329Jason Evansarena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, 14913ef51d7f733ac6432e80fa902a779ab5b98d74f6Jason Evans bool slow_path) 14924cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay{ 149388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans arena_chunk_t *chunk; 14944cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 1495c1e00ef2a6442d1d047950247c757821560db329Jason Evans assert(!tsdn_null(tsdn) || tcache == NULL); 1496c1e00ef2a6442d1d047950247c757821560db329Jason Evans 149788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 149888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(chunk != ptr)) { 149988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (config_prof && opt_prof) { 15009c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> 15019c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1Jason Evans LG_PAGE; 1502ef349f3f944b9b40bdeeff6cc322ef753f1ad4beJason Evans assert(arena_mapbits_allocated_get(chunk, pageind) != 1503ef349f3f944b9b40bdeeff6cc322ef753f1ad4beJason Evans 0); 150488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (arena_mapbits_large_get(chunk, pageind) != 0) { 150588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* 150688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * Make sure to use promoted size, not request 150788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans * size. 150888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans */ 150988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size = arena_mapbits_large_size_get(chunk, 15108a03cf039cd06f9fa6972711195055d865673966Jason Evans pageind) - large_pad; 151188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 15124cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay } 1513c1e00ef2a6442d1d047950247c757821560db329Jason Evans assert(s2u(size) == s2u(arena_salloc(tsdn, ptr, false))); 151488fef7ceda6269598cef0cee8b984c8765673c27Jason Evans 151588fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(size <= SMALL_MAXCLASS)) { 151688fef7ceda6269598cef0cee8b984c8765673c27Jason Evans /* Small allocation. */ 151788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans if (likely(tcache != NULL)) { 1518d01fd19755bc0c2f5be3143349016dd0d7de7b36Jason Evans szind_t binind = size2index(size); 1519c1e00ef2a6442d1d047950247c757821560db329Jason Evans tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, 1520c1e00ef2a6442d1d047950247c757821560db329Jason Evans binind, slow_path); 152188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else { 152288fef7ceda6269598cef0cee8b984c8765673c27Jason Evans size_t pageind = ((uintptr_t)ptr - 152388fef7ceda6269598cef0cee8b984c8765673c27Jason Evans (uintptr_t)chunk) >> LG_PAGE; 1524c1e00ef2a6442d1d047950247c757821560db329Jason Evans arena_dalloc_small(tsdn, 1525c1e00ef2a6442d1d047950247c757821560db329Jason Evans extent_node_arena_get(&chunk->node), chunk, 1526c1e00ef2a6442d1d047950247c757821560db329Jason Evans ptr, pageind); 152788fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 152888fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else { 15298a03cf039cd06f9fa6972711195055d865673966Jason Evans assert(config_cache_oblivious || ((uintptr_t)ptr & 15308a03cf039cd06f9fa6972711195055d865673966Jason Evans PAGE_MASK) == 0); 15314cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay 1532ef349f3f944b9b40bdeeff6cc322ef753f1ad4beJason Evans if (likely(tcache != NULL) && size <= tcache_maxclass) { 1533c1e00ef2a6442d1d047950247c757821560db329Jason Evans tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, 1534c1e00ef2a6442d1d047950247c757821560db329Jason Evans size, slow_path); 1535ef349f3f944b9b40bdeeff6cc322ef753f1ad4beJason Evans } else { 1536c1e00ef2a6442d1d047950247c757821560db329Jason Evans arena_dalloc_large(tsdn, 1537c1e00ef2a6442d1d047950247c757821560db329Jason Evans extent_node_arena_get(&chunk->node), chunk, 1538c1e00ef2a6442d1d047950247c757821560db329Jason Evans ptr); 153988fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 154088fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } 154188fef7ceda6269598cef0cee8b984c8765673c27Jason Evans } else 1542c1e00ef2a6442d1d047950247c757821560db329Jason Evans huge_dalloc(tsdn, ptr); 15434cfe55166e0173be745c53adb0fecf50d11d1227Daniel Micay} 1544155bfa7da18cab0d21d87aa2dce4554166836f5dJason Evans# endif /* JEMALLOC_ARENA_INLINE_B */ 1545e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 1546e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1547e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_INLINES */ 1548e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 1549