arena.h revision 597632be188d2bcc135dad2145cc46ef44897aad
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 2e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_TYPES 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Subpages are an artificially designated partitioning of pages. Their only 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * purpose is to support subpage-spaced size classes. 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 8e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * There must be at least 4 subpages per page, due to the way size classes are 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * handled. 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 11e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define LG_SUBPAGE 8 12e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define SUBPAGE ((size_t)(1U << LG_SUBPAGE)) 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define SUBPAGE_MASK (SUBPAGE - 1) 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Return the smallest subpage multiple that is >= s. */ 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define SUBPAGE_CEILING(s) \ 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (((s) + SUBPAGE_MASK) & ~SUBPAGE_MASK) 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Smallest size class to support. */ 211e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans# define LG_TINY_MIN LG_SIZEOF_PTR 2241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans# define TINY_MIN (1U << LG_TINY_MIN) 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 24e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Maximum size class that is a multiple of the quantum, but not (necessarily) 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a power of 2. Above this size, allocations are rounded up to the nearest 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * power of 2. 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define LG_QSPACE_MAX_DEFAULT 7 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 32e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Maximum size class that is a multiple of the cacheline, but not (necessarily) 34e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a power of 2. Above this size, allocations are rounded up to the nearest 35e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * power of 2. 36e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 37e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define LG_CSPACE_MAX_DEFAULT 9 38e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized 41e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * as small as possible such that this setting is still honored, without 42e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * violating other constraints. The goal is to make runs as small as possible 43e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * without exceeding a per run external fragmentation threshold. 44e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * We use binary fixed point math for overhead computations, where the binary 46e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * point is implicitly RUN_BFP bits to the left. 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be 498ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * honored for some/all object sizes, since when heap profiling is enabled 508ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * there is one pointer of header overhead per object (plus a constant). This 518ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * constraint is relaxed (ignored) for runs that are so small that the 528ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * per-region overhead is greater than: 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP)) 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 56e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define RUN_BFP 12 57e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* \/ Implicit binary fixed point. */ 58e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define RUN_MAX_OVRHD 0x0000003dU 59e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define RUN_MAX_OVRHD_RELAX 0x00001800U 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 61e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 62e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The minimum ratio of active:dirty pages per arena is computed as: 63e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 64e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * (nactive >> opt_lg_dirty_mult) >= ndirty 65e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 66e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * So, supposing that opt_lg_dirty_mult is 5, there can be no less than 32 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * times as many active pages as dirty pages. 68e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define LG_DIRTY_MULT_DEFAULT 5 70e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 71e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_chunk_map_s arena_chunk_map_t; 72e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_chunk_s arena_chunk_t; 73e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_run_s arena_run_t; 7449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evanstypedef struct arena_bin_info_s arena_bin_info_t; 75e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_bin_s arena_bin_t; 76e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_s arena_t; 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_TYPES */ 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_STRUCTS 81e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 82e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Each element of the chunk map corresponds to one page within the chunk. */ 83e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_chunk_map_s { 8405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans union { 8505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 8605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * Linkage for run trees. There are two disjoint uses: 8705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * 8819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 1) arena_t's runs_avail_{clean,dirty} trees. 8905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * 2) arena_run_t conceptually uses this linkage for in-use 9005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * non-full runs, rather than directly embedding linkage. 9105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 9205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans rb_node(arena_chunk_map_t) rb_link; 9305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 9405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * List of runs currently in purgatory. arena_chunk_purge() 9505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * temporarily allocates runs that contain dirty pages while 9605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * purging, so that other threads cannot use the runs while the 9705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * purging thread is operating without the arena lock held. 9805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 9905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_elm(arena_chunk_map_t) ql_link; 10005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } u; 101e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1026109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 1036109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Profile counters, used for large object runs. */ 1045065156f3f90e421ba2b1a914e47eeb30d83d994Jason Evans prof_ctx_t *prof_ctx; 1056109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 1066109fe07a14b7a619365977d9523db9f8b333792Jason Evans 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 108e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Run address (or size) and various flags are stored together. The bit 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * layout looks like (assuming 32-bit system): 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 1118ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * ???????? ???????? ????---- ----dula 112e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * ? : Unallocated: Run address for first/last pages, unset for internal 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * pages. 11519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Small: Run page offset. 116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: Run size for first page, unset for trailing pages. 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * - : Unused. 118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * d : dirty? 1198ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * u : unzeroed? 120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * l : large? 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a : allocated? 122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Following are example bit patterns for the three types of runs. 124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * p : run page offset 126e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * s : run size 127e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans * c : (binind+1) for size class (used only if prof_promote is true) 128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * x : don't care 129e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * - : 0 1300b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * + : 1 1313377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * [DULA] : bit set 1323377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * [dula] : bit unset 133e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 13419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (clean): 1358ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * ssssssss ssssssss ssss---- ----du-a 1363377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans * xxxxxxxx xxxxxxxx xxxx---- -----Uxx 1378ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * ssssssss ssssssss ssss---- ----dU-a 13819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 13919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (dirty): 1408ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * ssssssss ssssssss ssss---- ----D--a 14119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * xxxxxxxx xxxxxxxx xxxx---- ----xxxx 1428ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * ssssssss ssssssss ssss---- ----D--a 143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 144dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Small: 1458ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * pppppppp pppppppp pppp---- ----d--A 1468ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * pppppppp pppppppp pppp---- -------A 1478ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * pppppppp pppppppp pppp---- ----d--A 148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: 1508ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * ssssssss ssssssss ssss---- ----D-LA 15119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * xxxxxxxx xxxxxxxx xxxx---- ----xxxx 1528ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * -------- -------- -------- ----D-LA 1530b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 1540b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * Large (sampled, size <= PAGE_SIZE): 1558ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * ssssssss ssssssss sssscccc ccccD-LA 1560b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * 1570b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans * Large (not sampled, size == PAGE_SIZE): 1588ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * ssssssss ssssssss ssss---- ----D-LA 159e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 160e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t bits; 1610b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#ifdef JEMALLOC_PROF 1620b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_CLASS_SHIFT 4 1630b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_CLASS_MASK ((size_t)0xff0U) 1640b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#endif 1650b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_FLAGS_MASK ((size_t)0xfU) 1660b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_DIRTY ((size_t)0x8U) 1673377ffa1f4f8e67bce1e36624285e5baf5f9ecefJason Evans#define CHUNK_MAP_UNZEROED ((size_t)0x4U) 1680b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_LARGE ((size_t)0x2U) 1690b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_ALLOCATED ((size_t)0x1U) 1700b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evans#define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED 171e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 172e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef rb_tree(arena_chunk_map_t) arena_avail_tree_t; 173e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef rb_tree(arena_chunk_map_t) arena_run_tree_t; 174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 175e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Arena chunk header. */ 176e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_chunk_s { 177e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Arena that owns the chunk. */ 178e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 179e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1802caa4715ed4f787f263239ff97dd824636289286Jason Evans /* Linkage for the arena's chunks_dirty list. */ 1812caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_elm(arena_chunk_t) link_dirty; 182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 183e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1842caa4715ed4f787f263239ff97dd824636289286Jason Evans * True if the chunk is currently in the chunks_dirty list, due to 185e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * having at some point contained one or more dirty pages. Removal 186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * from chunks_dirty is lazy, so (dirtied && ndirty == 0) is possible. 187e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool dirtied; 189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Number of dirty pages. */ 191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty; 19213668262d17fb5950e2441bc9d56a15db9c93877Jason Evans 1937393f44ff025ca67716fc53b68003fd65122fd97Jason Evans /* 1947393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * Map of pages within chunk that keeps track of free/large/small. The 1957393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * first map_bias entries are omitted, since the chunk header does not 1967393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * need to be tracked in the map. This omission saves a header page 1977393f44ff025ca67716fc53b68003fd65122fd97Jason Evans * for common chunk sizes (e.g. 4 MiB). 1987393f44ff025ca67716fc53b68003fd65122fd97Jason Evans */ 199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t map[1]; /* Dynamically sized. */ 200e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 201e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef rb_tree(arena_chunk_t) arena_chunk_tree_t; 202e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 203e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_run_s { 204e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 205e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uint32_t magic; 206e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define ARENA_RUN_MAGIC 0x384adf93 207e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 208e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 209e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Bin this run is associated with. */ 210e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 211e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 21284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* Index of next region that has never been allocated, or nregs. */ 21384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans uint32_t nextind; 214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 215e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Number of free regions in run. */ 216e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned nfree; 217e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 21949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans/* 22084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Read-only information associated with each element of arena_t's bins array 22149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * is stored separately, partly to reduce memory usage (only one copy, rather 22249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * than one per arena), but mainly to avoid false cacheline sharing. 22349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 22449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansstruct arena_bin_info_s { 22549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Size of regions in a run for this bin's size class. */ 22649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t reg_size; 22749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 22849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total size of a run for this bin's size class. */ 22949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t run_size; 23049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 23149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Total number of regions in a run for this bin's size class. */ 23249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t nregs; 23349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 23484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 23584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Offset of first bitmap_t element in a run header for this bin's size 23684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * class. 23784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 23884c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans uint32_t bitmap_offset; 23984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 24084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 24184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Metadata used to manipulate bitmaps for runs associated with this 24284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * bin. 24384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 24484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans bitmap_info_t bitmap_info; 24584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans 24649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#ifdef JEMALLOC_PROF 24749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* 24849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * Offset of first (prof_ctx_t *) in a run header for this bin's size 24949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans * class, or 0 if (opt_prof == false). 25049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans */ 25149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t ctx0_offset; 25249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#endif 25349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 25449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans /* Offset of first region in a run for this bin's size class. */ 25549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans uint32_t reg0_offset; 25649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans}; 25749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 258e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_bin_s { 259e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 26086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * All operations on runcur, runs, and stats require that lock be 26186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. Run allocation/deallocation are protected by the arena lock, 26286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * which may be acquired while holding one or more bin locks, but not 26386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * vise versa. 26486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 26586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_t lock; 26686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 26786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current run being used to service allocations of this bin's size 269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * class. 270e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *runcur; 272e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Tree of non-full runs. This tree is used when looking for an 275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * existing run when runcur is no longer usable. We choose the 276e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * non-full run that is lowest in memory; this policy tends to keep 277e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * objects packed well, and it can also help reduce the number of 278e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * almost-empty chunks. 279e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 280e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_t runs; 281e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 282e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 283e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Bin statistics. */ 284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_bin_stats_t stats; 285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 287e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 288e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_s { 289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uint32_t magic; 291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define ARENA_MAGIC 0x947d3d24 292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 293e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2946109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* This arena's index within the arenas array. */ 2956109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned ind; 2966109fe07a14b7a619365977d9523db9f8b333792Jason Evans 29786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 298597632be188d2bcc135dad2145cc46ef44897aadJason Evans * Number of threads currently assigned to this arena. This field is 299597632be188d2bcc135dad2145cc46ef44897aadJason Evans * protected by arenas_lock. 300597632be188d2bcc135dad2145cc46ef44897aadJason Evans */ 301597632be188d2bcc135dad2145cc46ef44897aadJason Evans unsigned nthreads; 302597632be188d2bcc135dad2145cc46ef44897aadJason Evans 303597632be188d2bcc135dad2145cc46ef44897aadJason Evans /* 304597632be188d2bcc135dad2145cc46ef44897aadJason Evans * There are three classes of arena operations from a locking 305597632be188d2bcc135dad2145cc46ef44897aadJason Evans * perspective: 306597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 1) Thread asssignment (modifies nthreads) is protected by 307597632be188d2bcc135dad2145cc46ef44897aadJason Evans * arenas_lock. 308597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 2) Bin-related operations are protected by bin locks. 309597632be188d2bcc135dad2145cc46ef44897aadJason Evans * 3) Chunk- and run-related operations are protected by this mutex. 31086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 311e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_t lock; 312e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 313e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 314e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_stats_t stats; 315e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 316e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 317e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * List of tcaches for extant threads associated with this arena. 318e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Stats from these are merged incrementally, and at exit. 319e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 320e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_head(tcache_t) tcache_ql; 321e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 322e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 323e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 324d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 325d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans uint64_t prof_accumbytes; 326d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 327d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 3282caa4715ed4f787f263239ff97dd824636289286Jason Evans /* List of dirty-page-containing chunks this arena manages. */ 3292caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_head(arena_chunk_t) chunks_dirty; 330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * In order to avoid rapid chunk allocation/deallocation when an arena 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * oscillates right on the cusp of needing a new chunk, cache the most 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * recently freed chunk. The spare is left in the arena's chunk trees 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * until it is deleted. 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * There is one spare chunk per arena, rather than one spare total, in 338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * order to avoid interactions between multiple threads that could make 339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a single spare inadequate. 340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *spare; 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Number of pages in active runs. */ 344bc25a47ee0e2ac8e10a94d5fa070f0dbbdeb7e7eJason Evans size_t nactive; 345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current count of pages within unused runs that are potentially 348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * dirty, and for which madvise(... MADV_DONTNEED) has not been called. 349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * By tracking this, we can institute a limit on how much dirty unused 350e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * memory is mapped for each arena. 351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty; 353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 355799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * Approximate number of pages being purged. It is possible for 356799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * multiple threads to purge dirty pages concurrently, and they use 357799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * npurgatory to indicate the total number of pages all threads are 358799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans * attempting to purge. 35905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 360799ca0b68dd2a87eb7696b7a83ed85378cc6721cJason Evans size_t npurgatory; 36105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 36205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 36319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Size/address-ordered trees of this arena's available runs. The trees 36419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * are used for first-best-fit run allocation. The dirty tree contains 36519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * runs with dirty pages (i.e. very likely to have been touched and 36619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * therefore have associated physical pages), whereas the clean tree 36719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * contains runs with pages that either have no associated physical 36819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * pages, or have pages that the kernel may recycle at any time due to 36919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * previous madvise(2) calls. The dirty tree is used in preference to 37019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * the clean tree for allocations, because using dirty pages reduces 37119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * the amount of dirty purging necessary to keep the active:dirty page 37219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * ratio below the purge threshold. 373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 37419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t runs_avail_clean; 37519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t runs_avail_dirty; 376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * bins is used to store trees of free regions of the following sizes, 3798ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * assuming a 64-bit system with 16-byte quantum, 4 KiB page size, and 3808ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * default MALLOC_CONF. 381e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 382e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * bins[i] | size | 383e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * --------+--------+ 3848ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 0 | 8 | 385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * --------+--------+ 3868ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 1 | 16 | 3878ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 2 | 32 | 3888ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 3 | 48 | 389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * : : 3908ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 6 | 96 | 3918ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 7 | 112 | 3928ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 8 | 128 | 393e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * --------+--------+ 3948ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 9 | 192 | 3958ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 10 | 256 | 3968ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 11 | 320 | 3978ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 12 | 384 | 3988ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 13 | 448 | 3998ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 14 | 512 | 400e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * --------+--------+ 4018ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 15 | 768 | 4028ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 16 | 1024 | 4038ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 17 | 1280 | 404e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * : : 4058ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 25 | 3328 | 4068ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 26 | 3584 | 4078ad0eacfb351af990ae021f219b7b88d70aede41Jason Evans * 27 | 3840 | 408e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * --------+--------+ 409e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 410e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t bins[1]; /* Dynamically sized. */ 411e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 412e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 413e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_STRUCTS */ 414e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 415e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_EXTERNS 416e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 417e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t opt_lg_qspace_max; 418e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t opt_lg_cspace_max; 41984c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evansextern ssize_t opt_lg_dirty_mult; 42041ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans/* 42141ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans * small_size2bin is a compact lookup table that rounds request sizes up to 42241ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans * size classes. In order to reduce cache footprint, the table is compressed, 42341ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans * and all accesses are via the SMALL_SIZE2BIN macro. 42441ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans */ 425e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern uint8_t const *small_size2bin; 42641ade967c29ea9312c0b7390ee43bc0c63373f39Jason Evans#define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN]) 427e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 42849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansextern arena_bin_info_t *arena_bin_info; 42949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 430e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Various bin-related settings. */ 431e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY /* Number of (2^n)-spaced tiny bins. */ 432e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define ntbins ((unsigned)(LG_QUANTUM - LG_TINY_MIN)) 433e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#else 434e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define ntbins 0 435e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 436e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern unsigned nqbins; /* Number of quantum-spaced bins. */ 437e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern unsigned ncbins; /* Number of cacheline-spaced bins. */ 438e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern unsigned nsbins; /* Number of subpage-spaced bins. */ 439e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern unsigned nbins; 440e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define tspace_max ((size_t)(QUANTUM >> 1)) 442e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 443e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define qspace_min QUANTUM 444e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t qspace_max; 445e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t cspace_min; 446e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t cspace_max; 447e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t sspace_min; 448e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t sspace_max; 449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define small_maxclass sspace_max 450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4517393f44ff025ca67716fc53b68003fd65122fd97Jason Evans#define nlclasses (chunk_npages - map_bias) 4523c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans 4536005f0710cf07d60659d91b20b7ff5592d310027Jason Evansvoid arena_purge_all(arena_t *arena); 4546005f0710cf07d60659d91b20b7ff5592d310027Jason Evans#ifdef JEMALLOC_PROF 4556005f0710cf07d60659d91b20b7ff5592d310027Jason Evansvoid arena_prof_accum(arena_t *arena, uint64_t accumbytes); 4566005f0710cf07d60659d91b20b7ff5592d310027Jason Evans#endif 457e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 458dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, 459dafde14e08ddfda747aabb2045b350848b601b2eJason Evans size_t binind 460d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans# ifdef JEMALLOC_PROF 461d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , uint64_t prof_accumbytes 462d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans# endif 463d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ); 464d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 465e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid *arena_malloc_small(arena_t *arena, size_t size, bool zero); 466dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid *arena_malloc_large(arena_t *arena, size_t size, bool zero); 467e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid *arena_malloc(size_t size, bool zero); 4688e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansvoid *arena_palloc(arena_t *arena, size_t size, size_t alloc_size, 4698e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t alignment, bool zero); 470e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t arena_salloc(const void *ptr); 4716109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 4720b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evansvoid arena_prof_promoted(const void *ptr, size_t size); 4730b270a991dd7822b7c7e13d9219f235a5dde9fdfJason Evanssize_t arena_salloc_demote(const void *ptr); 4746109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 475e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 476e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm); 477e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); 478e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 479b34e8684ec025aa780997c11f847c19fb269755bJason Evansvoid arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, 480b34e8684ec025aa780997c11f847c19fb269755bJason Evans arena_stats_t *astats, malloc_bin_stats_t *bstats, 481b34e8684ec025aa780997c11f847c19fb269755bJason Evans malloc_large_stats_t *lstats); 482e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 4838e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansvoid *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, 4848e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t extra, bool zero); 4858e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evansvoid *arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, 4868e3c3c61b5bb676a705450708e7e79698cdc9e0cJason Evans size_t alignment, bool zero); 487e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool arena_new(arena_t *arena, unsigned ind); 488a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evansbool arena_boot(void); 489e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 490e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_EXTERNS */ 491e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 492e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_INLINES 493e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 494e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifndef JEMALLOC_ENABLE_INLINE 49549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evanssize_t arena_bin_index(arena_t *arena, arena_bin_t *bin); 49649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansunsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, 497b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans const void *ptr); 49881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans# ifdef JEMALLOC_PROF 49981b4e6eb6f06cac048e3743787a70676f1534269Jason Evansprof_ctx_t *arena_prof_ctx_get(const void *ptr); 500e4f7846f1fd279a039ffa2a41707348187219de4Jason Evansvoid arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); 50181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans# endif 502e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr); 503e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 504e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 505e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) 50649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason EvansJEMALLOC_INLINE size_t 50749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evansarena_bin_index(arena_t *arena, arena_bin_t *bin) 50849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans{ 50949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = bin - arena->bins; 51049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(binind < nbins); 51149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans return (binind); 51249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans} 51349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans 51481b4e6eb6f06cac048e3743787a70676f1534269Jason EvansJEMALLOC_INLINE unsigned 515b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evansarena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) 51681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 51781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans unsigned shift, diff, regind; 518b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans size_t size; 51981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 52084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(run->magic == ARENA_RUN_MAGIC); 52184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans /* 52284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * Freeing a pointer lower than region zero can cause assertion 52384c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans * failure. 52484c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans */ 52584c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans assert((uintptr_t)ptr >= (uintptr_t)run + 52684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans (uintptr_t)bin_info->reg0_offset); 52781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 52881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 52981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * Avoid doing division with a variable divisor if possible. Using 53081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * actual division here can reduce allocator throughput by over 20%! 53181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 53249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - 53349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg0_offset); 53481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 53581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* Rescale (factor powers of 2 out of the numerator and denominator). */ 536b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans size = bin_info->reg_size; 53781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans shift = ffs(size) - 1; 53881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans diff >>= shift; 53981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans size >>= shift; 54081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 54181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans if (size == 1) { 54281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* The divisor was a power of 2. */ 54381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans regind = diff; 54481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } else { 54581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans /* 54681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * To divide by a number D that is not a power of two we 54781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * multiply by (2^21 / D) and then right shift by 21 positions. 54881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 54981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * X / D 55081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 55181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * becomes 55281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 55381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * (X * size_invs[D - 3]) >> SIZE_INV_SHIFT 55481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * 55581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * We can omit the first three elements, because we never 55681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * divide by 0, and 1 and 2 are both powers of two, which are 55781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans * handled above. 55881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans */ 55981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#define SIZE_INV_SHIFT 21 56081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) 56181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans static const unsigned size_invs[] = { 56281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(3), 56381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 56481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 56581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 56681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 56781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 56881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 56981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 57081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans }; 57181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 57281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans if (size <= ((sizeof(size_invs) / sizeof(unsigned)) + 2)) 57381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans regind = (diff * size_invs[size - 3]) >> SIZE_INV_SHIFT; 57481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans else 57581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans regind = diff / size; 57681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV 57781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#undef SIZE_INV_SHIFT 57881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } 57981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert(diff == regind * size); 58049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(regind < bin_info->nregs); 58181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 58281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (regind); 58381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 58481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 58581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#ifdef JEMALLOC_PROF 58681b4e6eb6f06cac048e3743787a70676f1534269Jason EvansJEMALLOC_INLINE prof_ctx_t * 58781b4e6eb6f06cac048e3743787a70676f1534269Jason Evansarena_prof_ctx_get(const void *ptr) 58881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans{ 58981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans prof_ctx_t *ret; 59081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans arena_chunk_t *chunk; 59181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans size_t pageind, mapbits; 59281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 59381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert(ptr != NULL); 59481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 59581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 59681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 59781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 59881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans mapbits = chunk->map[pageind-map_bias].bits; 59981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 60081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 60181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans if (prof_promote) 60281b4e6eb6f06cac048e3743787a70676f1534269Jason Evans ret = (prof_ctx_t *)(uintptr_t)1U; 60381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans else { 60481b4e6eb6f06cac048e3743787a70676f1534269Jason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 60581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << 60681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans PAGE_SHIFT)); 60749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(chunk->arena, run->bin); 60849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = &arena_bin_info[binind]; 60981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans unsigned regind; 61081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 61184c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(run->magic == ARENA_RUN_MAGIC); 612b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans regind = arena_run_regind(run, bin_info, ptr); 61381b4e6eb6f06cac048e3743787a70676f1534269Jason Evans ret = *(prof_ctx_t **)((uintptr_t)run + 61449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->ctx0_offset + (regind * 61581b4e6eb6f06cac048e3743787a70676f1534269Jason Evans sizeof(prof_ctx_t *))); 61681b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } 61781b4e6eb6f06cac048e3743787a70676f1534269Jason Evans } else 61881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans ret = chunk->map[pageind-map_bias].prof_ctx; 61981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 62081b4e6eb6f06cac048e3743787a70676f1534269Jason Evans return (ret); 62181b4e6eb6f06cac048e3743787a70676f1534269Jason Evans} 622e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 623e4f7846f1fd279a039ffa2a41707348187219de4Jason EvansJEMALLOC_INLINE void 624e4f7846f1fd279a039ffa2a41707348187219de4Jason Evansarena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) 625e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans{ 626e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans arena_chunk_t *chunk; 627e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans size_t pageind, mapbits; 628e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 629e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert(ptr != NULL); 630e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 631e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 632e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 633e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 634e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans mapbits = chunk->map[pageind-map_bias].bits; 635e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 636e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans if ((mapbits & CHUNK_MAP_LARGE) == 0) { 637e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans if (prof_promote == false) { 638e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 639e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << 640e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans PAGE_SHIFT)); 641e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans arena_bin_t *bin = run->bin; 64249f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind; 64349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info; 644b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans unsigned regind; 645e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 64684c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(run->magic == ARENA_RUN_MAGIC); 64749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans binind = arena_bin_index(chunk->arena, bin); 64849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info = &arena_bin_info[binind]; 649b602daa6710dab61d8e1ca0cd3c44ac8a564fd9fJason Evans regind = arena_run_regind(run, bin_info, ptr); 650e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans 65149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans *((prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset 652e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans + (regind * sizeof(prof_ctx_t *)))) = ctx; 653e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans } else 654e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans assert((uintptr_t)ctx == (uintptr_t)1U); 655e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans } else 656e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans chunk->map[pageind-map_bias].prof_ctx = ctx; 657e4f7846f1fd279a039ffa2a41707348187219de4Jason Evans} 65881b4e6eb6f06cac048e3743787a70676f1534269Jason Evans#endif 65981b4e6eb6f06cac048e3743787a70676f1534269Jason Evans 660e476f8a161d445211fd6e54fe370275196e66bcbJason EvansJEMALLOC_INLINE void 661e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr) 662e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 663e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 664e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm; 665e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 666e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(arena != NULL); 66784c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(arena->magic == ARENA_MAGIC); 668e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(chunk->arena == arena); 669e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 670e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 671e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 6727393f44ff025ca67716fc53b68003fd65122fd97Jason Evans pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; 6737393f44ff025ca67716fc53b68003fd65122fd97Jason Evans mapelm = &chunk->map[pageind-map_bias]; 674e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0); 675e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) { 676e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Small allocation. */ 677e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 678e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache; 679e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 680e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((tcache = tcache_get()) != NULL) 681dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_dalloc_small(tcache, ptr); 682e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 683e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 68486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_run_t *run; 68586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin; 68686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 68786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans run = (arena_run_t *)((uintptr_t)chunk + 68819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans (uintptr_t)((pageind - (mapelm->bits >> 68919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)) << PAGE_SHIFT)); 69084c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans dassert(run->magic == ARENA_RUN_MAGIC); 69186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin = run->bin; 69284c8eefeffa246607790ad12e28b0f6a24ecc59dJason Evans#ifdef JEMALLOC_DEBUG 69349f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans { 69449f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans size_t binind = arena_bin_index(arena, bin); 69549f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans arena_bin_info_t *bin_info = 69649f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans &arena_bin_info[binind]; 69749f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 69849f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans (uintptr_t)bin_info->reg0_offset)) % 69949f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans bin_info->reg_size == 0); 70049f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans } 70149f7e8f35ac63d0dd526cf68791dc0ca29538ac9Jason Evans#endif 70286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 703e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_bin(arena, chunk, ptr, mapelm); 70486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 705e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 706e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 707e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 708f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans } else { 709dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_TCACHE 710dafde14e08ddfda747aabb2045b350848b601b2eJason Evans size_t size = mapelm->bits & ~PAGE_MASK; 711dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 712f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 713dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (size <= tcache_maxclass) { 714dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_t *tcache; 715dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 716dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if ((tcache = tcache_get()) != NULL) 717dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_dalloc_large(tcache, ptr, size); 718dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else { 719dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 720dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_dalloc_large(arena, chunk, ptr); 721dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 722dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 723dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } else { 724dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 725dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_dalloc_large(arena, chunk, ptr); 726dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 727dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 728dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#else 729dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 730dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 731e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_large(arena, chunk, ptr); 732dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 733dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 734f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans } 735e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 736e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 737e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 738e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_INLINES */ 739e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 740