arena.h revision 19b3d618924b3542a264612f906bc53bbcec8b70
1e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 2e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_TYPES 3e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 4e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 5e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Subpages are an artificially designated partitioning of pages. Their only 6e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * purpose is to support subpage-spaced size classes. 7e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 8e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * There must be at least 4 subpages per page, due to the way size classes are 9e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * handled. 10e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 11e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define LG_SUBPAGE 8 12e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define SUBPAGE ((size_t)(1U << LG_SUBPAGE)) 13e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define SUBPAGE_MASK (SUBPAGE - 1) 14e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 15e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Return the smallest subpage multiple that is >= s. */ 16e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define SUBPAGE_CEILING(s) \ 17e476f8a161d445211fd6e54fe370275196e66bcbJason Evans (((s) + SUBPAGE_MASK) & ~SUBPAGE_MASK) 18e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 19e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 20e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Smallest size class to support. */ 211e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans# define LG_TINY_MIN LG_SIZEOF_PTR 22e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 23e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 24e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 25e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Maximum size class that is a multiple of the quantum, but not (necessarily) 26e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a power of 2. Above this size, allocations are rounded up to the nearest 27e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * power of 2. 28e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 29e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define LG_QSPACE_MAX_DEFAULT 7 30e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 31e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 32e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Maximum size class that is a multiple of the cacheline, but not (necessarily) 33e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a power of 2. Above this size, allocations are rounded up to the nearest 34e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * power of 2. 35e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 36e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define LG_CSPACE_MAX_DEFAULT 9 37e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 38e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 39e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized 40e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * as small as possible such that this setting is still honored, without 41e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * violating other constraints. The goal is to make runs as small as possible 42e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * without exceeding a per run external fragmentation threshold. 43e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 44e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * We use binary fixed point math for overhead computations, where the binary 45e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * point is implicitly RUN_BFP bits to the left. 46e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 47e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be 48e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * honored for some/all object sizes, since there is one bit of header overhead 49e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * per object (plus a constant). This constraint is relaxed (ignored) for runs 50e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * that are so small that the per-region overhead is greater than: 51e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 52e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP)) 53e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 54e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define RUN_BFP 12 55e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* \/ Implicit binary fixed point. */ 56e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define RUN_MAX_OVRHD 0x0000003dU 57e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define RUN_MAX_OVRHD_RELAX 0x00001800U 58e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 59e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* 60e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * The minimum ratio of active:dirty pages per arena is computed as: 61e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 62e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * (nactive >> opt_lg_dirty_mult) >= ndirty 63e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 64e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * So, supposing that opt_lg_dirty_mult is 5, there can be no less than 32 65e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * times as many active pages as dirty pages. 66e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 67e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define LG_DIRTY_MULT_DEFAULT 5 68e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 69e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_chunk_map_s arena_chunk_map_t; 70e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_chunk_s arena_chunk_t; 71e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_run_s arena_run_t; 72e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_bin_s arena_bin_t; 73e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef struct arena_s arena_t; 74e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 75e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_TYPES */ 76e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 77e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_STRUCTS 78e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 79e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Each element of the chunk map corresponds to one page within the chunk. */ 80e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_chunk_map_s { 8105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans union { 8205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 8305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * Linkage for run trees. There are two disjoint uses: 8405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * 8519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 1) arena_t's runs_avail_{clean,dirty} trees. 8605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * 2) arena_run_t conceptually uses this linkage for in-use 8705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * non-full runs, rather than directly embedding linkage. 8805b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 8905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans rb_node(arena_chunk_map_t) rb_link; 9005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 9105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * List of runs currently in purgatory. arena_chunk_purge() 9205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * temporarily allocates runs that contain dirty pages while 9305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * purging, so that other threads cannot use the runs while the 9405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * purging thread is operating without the arena lock held. 9505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 9605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans ql_elm(arena_chunk_map_t) ql_link; 9705b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans } u; 98e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 996109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 1006109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* Profile counters, used for large object runs. */ 1016109fe07a14b7a619365977d9523db9f8b333792Jason Evans prof_thr_cnt_t *prof_cnt; 1026109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 1036109fe07a14b7a619365977d9523db9f8b333792Jason Evans 104e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 105e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Run address (or size) and various flags are stored together. The bit 106e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * layout looks like (assuming 32-bit system): 107e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 10819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * ???????? ???????? ????---- ----dzla 109e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 110e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * ? : Unallocated: Run address for first/last pages, unset for internal 111e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * pages. 11219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Small: Run page offset. 113e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: Run size for first page, unset for trailing pages. 114e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * - : Unused. 115e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * d : dirty? 116e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * z : zeroed? 117e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * l : large? 118e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a : allocated? 119e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 120e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Following are example bit patterns for the three types of runs. 121e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 122e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * p : run page offset 123e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * s : run size 124e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * x : don't care 125e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * - : 0 12619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * [DZLA] : bit set 12719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * [dzla] : bit unset 128e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 12919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (clean): 13019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * ssssssss ssssssss ssss---- ----dz-- 13119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * xxxxxxxx xxxxxxxx xxxx---- -----Zxx 13219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * ssssssss ssssssss ssss---- ----dZ-- 13319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * 13419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Unallocated (dirty): 13519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * ssssssss ssssssss ssss---- ----D--- 13619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * xxxxxxxx xxxxxxxx xxxx---- ----xxxx 13719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * ssssssss ssssssss ssss---- ----D--- 138e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 139dafde14e08ddfda747aabb2045b350848b601b2eJason Evans * Small: 14019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * pppppppp pppppppp pppp---- ----d--a 1411e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans * pppppppp pppppppp pppp---- -------a 14219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * pppppppp pppppppp pppp---- ----d--a 143e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 144e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Large: 14519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * ssssssss ssssssss ssss---- ----D-la 14619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * xxxxxxxx xxxxxxxx xxxx---- ----xxxx 14719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * -------- -------- -------- ----D-la 148e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 149e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t bits; 15019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans#define CHUNK_MAP_FLAGS_MASK ((size_t)0x1fU) 15119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans#define CHUNK_MAP_KEY ((size_t)0x10U) 15219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans#define CHUNK_MAP_DIRTY ((size_t)0x08U) 15319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans#define CHUNK_MAP_ZEROED ((size_t)0x04U) 15419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans#define CHUNK_MAP_LARGE ((size_t)0x02U) 15519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans#define CHUNK_MAP_ALLOCATED ((size_t)0x01U) 156e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 157e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef rb_tree(arena_chunk_map_t) arena_avail_tree_t; 158e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef rb_tree(arena_chunk_map_t) arena_run_tree_t; 159e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 160e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Arena chunk header. */ 161e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_chunk_s { 162e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Arena that owns the chunk. */ 163e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_t *arena; 164e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1652caa4715ed4f787f263239ff97dd824636289286Jason Evans /* Linkage for the arena's chunks_dirty list. */ 1662caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_elm(arena_chunk_t) link_dirty; 167e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 168e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 1692caa4715ed4f787f263239ff97dd824636289286Jason Evans * True if the chunk is currently in the chunks_dirty list, due to 170e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * having at some point contained one or more dirty pages. Removal 171e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * from chunks_dirty is lazy, so (dirtied && ndirty == 0) is possible. 172e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 173e476f8a161d445211fd6e54fe370275196e66bcbJason Evans bool dirtied; 174e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 175e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Number of dirty pages. */ 176e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty; 17713668262d17fb5950e2441bc9d56a15db9c93877Jason Evans 178e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Map of pages within chunk that keeps track of free/large/small. */ 179e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t map[1]; /* Dynamically sized. */ 180e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 181e476f8a161d445211fd6e54fe370275196e66bcbJason Evanstypedef rb_tree(arena_chunk_t) arena_chunk_tree_t; 182e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 183e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_run_s { 184e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 185e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uint32_t magic; 186e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define ARENA_RUN_MAGIC 0x384adf93 187e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 188e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 189e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Bin this run is associated with. */ 190e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t *bin; 191e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 1921e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Stack of available freed regions, or NULL. */ 1931e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans void *avail; 1941e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans 1951e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans /* Next region that has never been allocated, or run boundary. */ 1961e0a636c11e694b4b157f40198fd463fcfc6c57aJason Evans void *next; 197e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 198e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Number of free regions in run. */ 199e476f8a161d445211fd6e54fe370275196e66bcbJason Evans unsigned nfree; 200e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 201e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 202e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_bin_s { 203e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 20486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * All operations on runcur, runs, and stats require that lock be 20586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. Run allocation/deallocation are protected by the arena lock, 20686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * which may be acquired while holding one or more bin locks, but not 20786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * vise versa. 20886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 20986815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_t lock; 21086815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 21186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 212e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current run being used to service allocations of this bin's size 213e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * class. 214e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 215e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_t *runcur; 216e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 217e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 218e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Tree of non-full runs. This tree is used when looking for an 219e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * existing run when runcur is no longer usable. We choose the 220e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * non-full run that is lowest in memory; this policy tends to keep 221e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * objects packed well, and it can also help reduce the number of 222e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * almost-empty chunks. 223e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 224e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_run_tree_t runs; 225e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 226e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Size of regions in a run for this bin's size class. */ 227e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t reg_size; 228e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 229e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Total size of a run for this bin's size class. */ 230e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t run_size; 231e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 232e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Total number of regions in a run for this bin's size class. */ 233e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uint32_t nregs; 234e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2356109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 2366109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* 2376109fe07a14b7a619365977d9523db9f8b333792Jason Evans * Offset of first (prof_cnt_t *) in a run header for this bin's size 2386109fe07a14b7a619365977d9523db9f8b333792Jason Evans * class, or 0 if (opt_prof == false). 2396109fe07a14b7a619365977d9523db9f8b333792Jason Evans */ 2406109fe07a14b7a619365977d9523db9f8b333792Jason Evans uint32_t cnt0_offset; 2416109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 2426109fe07a14b7a619365977d9523db9f8b333792Jason Evans 243e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Offset of first region in a run for this bin's size class. */ 244e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uint32_t reg0_offset; 245e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 246e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 247e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Bin statistics. */ 248e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_bin_stats_t stats; 249e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 250e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 251e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 252e476f8a161d445211fd6e54fe370275196e66bcbJason Evansstruct arena_s { 253e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_DEBUG 254e476f8a161d445211fd6e54fe370275196e66bcbJason Evans uint32_t magic; 255e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define ARENA_MAGIC 0x947d3d24 256e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 257e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 2586109fe07a14b7a619365977d9523db9f8b333792Jason Evans /* This arena's index within the arenas array. */ 2596109fe07a14b7a619365977d9523db9f8b333792Jason Evans unsigned ind; 2606109fe07a14b7a619365977d9523db9f8b333792Jason Evans 26186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans /* 26286815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * All non-bin-related operations on this arena require that lock be 26386815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans * locked. 26486815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans */ 265e476f8a161d445211fd6e54fe370275196e66bcbJason Evans malloc_mutex_t lock; 266e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 267e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 268e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_stats_t stats; 269e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# ifdef JEMALLOC_TCACHE 270e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 271e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * List of tcaches for extant threads associated with this arena. 272e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Stats from these are merged incrementally, and at exit. 273e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 274e476f8a161d445211fd6e54fe370275196e66bcbJason Evans ql_head(tcache_t) tcache_ql; 275e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# endif 276e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 277e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 278d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 279d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans uint64_t prof_accumbytes; 280d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 281d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans 2822caa4715ed4f787f263239ff97dd824636289286Jason Evans /* List of dirty-page-containing chunks this arena manages. */ 2832caa4715ed4f787f263239ff97dd824636289286Jason Evans ql_head(arena_chunk_t) chunks_dirty; 284e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 285e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 286e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * In order to avoid rapid chunk allocation/deallocation when an arena 287e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * oscillates right on the cusp of needing a new chunk, cache the most 288e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * recently freed chunk. The spare is left in the arena's chunk trees 289e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * until it is deleted. 290e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 291e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * There is one spare chunk per arena, rather than one spare total, in 292e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * order to avoid interactions between multiple threads that could make 293e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * a single spare inadequate. 294e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 295e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_t *spare; 296e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 297e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Number of pages in active runs. */ 298bc25a47ee0e2ac8e10a94d5fa070f0dbbdeb7e7eJason Evans size_t nactive; 299e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 300e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 301e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * Current count of pages within unused runs that are potentially 302e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * dirty, and for which madvise(... MADV_DONTNEED) has not been called. 303e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * By tracking this, we can institute a limit on how much dirty unused 304e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * memory is mapped for each arena. 305e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 306e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t ndirty; 307e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 308e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 30905b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * Approximate number of pages being purged. It is possible for 31005b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * multiple threads to purge dirty pages concurrently, and they use 31105b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * npurgatory to indicate the total number of pages all threads are 31205b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans * attempting to purge. 31305b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans */ 31405b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans size_t npurgatory; 31505b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans 31605b21be347d66b4afd5147ecd26e8cb22f6ea3feJason Evans /* 31719b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * Size/address-ordered trees of this arena's available runs. The trees 31819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * are used for first-best-fit run allocation. The dirty tree contains 31919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * runs with dirty pages (i.e. very likely to have been touched and 32019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * therefore have associated physical pages), whereas the clean tree 32119b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * contains runs with pages that either have no associated physical 32219b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * pages, or have pages that the kernel may recycle at any time due to 32319b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * previous madvise(2) calls. The dirty tree is used in preference to 32419b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * the clean tree for allocations, because using dirty pages reduces 32519b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * the amount of dirty purging necessary to keep the active:dirty page 32619b3d618924b3542a264612f906bc53bbcec8b70Jason Evans * ratio below the purge threshold. 327e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 32819b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t runs_avail_clean; 32919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans arena_avail_tree_t runs_avail_dirty; 330e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 331e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* 332e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * bins is used to store trees of free regions of the following sizes, 333e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * assuming a 16-byte quantum, 4 KiB page size, and default 334e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * JEMALLOC_OPTIONS. 335e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 336e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * bins[i] | size | 337e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * --------+--------+ 338e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 0 | 2 | 339e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 1 | 4 | 340e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 2 | 8 | 341e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * --------+--------+ 342e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 3 | 16 | 343e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 4 | 32 | 344e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 5 | 48 | 345e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * : : 346e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 8 | 96 | 347e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 9 | 112 | 348e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 10 | 128 | 349e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * --------+--------+ 350e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 11 | 192 | 351e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 12 | 256 | 352e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 13 | 320 | 353e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 14 | 384 | 354e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 15 | 448 | 355e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 16 | 512 | 356e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * --------+--------+ 357e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 17 | 768 | 358e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 18 | 1024 | 359e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 19 | 1280 | 360e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * : : 361e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 27 | 3328 | 362e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 28 | 3584 | 363e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 29 | 3840 | 364e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * --------+--------+ 365e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 30 | 4 KiB | 366e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 31 | 6 KiB | 367e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 33 | 8 KiB | 368e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * : : 369e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 43 | 28 KiB | 370e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 44 | 30 KiB | 371e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * 45 | 32 KiB | 372e476f8a161d445211fd6e54fe370275196e66bcbJason Evans * --------+--------+ 373e476f8a161d445211fd6e54fe370275196e66bcbJason Evans */ 374e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_bin_t bins[1]; /* Dynamically sized. */ 375e476f8a161d445211fd6e54fe370275196e66bcbJason Evans}; 376e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 377e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_STRUCTS */ 378e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 379e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_EXTERNS 380e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 381e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t opt_lg_qspace_max; 382e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t opt_lg_cspace_max; 383e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern ssize_t opt_lg_dirty_mult; 384e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern uint8_t const *small_size2bin; 385e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 386e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/* Various bin-related settings. */ 387e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY /* Number of (2^n)-spaced tiny bins. */ 388e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define ntbins ((unsigned)(LG_QUANTUM - LG_TINY_MIN)) 389e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#else 390e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define ntbins 0 391e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 392e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern unsigned nqbins; /* Number of quantum-spaced bins. */ 393e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern unsigned ncbins; /* Number of cacheline-spaced bins. */ 394e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern unsigned nsbins; /* Number of subpage-spaced bins. */ 395e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern unsigned nbins; 396e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TINY 397e476f8a161d445211fd6e54fe370275196e66bcbJason Evans# define tspace_max ((size_t)(QUANTUM >> 1)) 398e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 399e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define qspace_min QUANTUM 400e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t qspace_max; 401e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t cspace_min; 402e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t cspace_max; 403e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t sspace_min; 404e476f8a161d445211fd6e54fe370275196e66bcbJason Evansextern size_t sspace_max; 405e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#define small_maxclass sspace_max 406e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 407dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#define nlclasses (chunk_npages - arena_chunk_header_npages) 4083c2343518c2b1fbbd66065c75a3c19f908de1d78Jason Evans 409e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 410dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, 411dafde14e08ddfda747aabb2045b350848b601b2eJason Evans size_t binind 412d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans# ifdef JEMALLOC_PROF 413d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans , uint64_t prof_accumbytes 414d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans# endif 415d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans ); 416d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#endif 417d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evans#ifdef JEMALLOC_PROF 418d34f9e7e9306698e298a703c28526cd6bfc073ecJason Evansvoid arena_prof_accum(arena_t *arena, uint64_t accumbytes); 419e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 420e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid *arena_malloc_small(arena_t *arena, size_t size, bool zero); 421dafde14e08ddfda747aabb2045b350848b601b2eJason Evansvoid *arena_malloc_large(arena_t *arena, size_t size, bool zero); 422e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid *arena_malloc(size_t size, bool zero); 423e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid *arena_palloc(arena_t *arena, size_t alignment, size_t size, 424e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t alloc_size); 425e476f8a161d445211fd6e54fe370275196e66bcbJason Evanssize_t arena_salloc(const void *ptr); 4266109fe07a14b7a619365977d9523db9f8b333792Jason Evans#ifdef JEMALLOC_PROF 4276109fe07a14b7a619365977d9523db9f8b333792Jason Evansprof_thr_cnt_t *arena_prof_cnt_get(const void *ptr); 4286109fe07a14b7a619365977d9523db9f8b333792Jason Evansvoid arena_prof_cnt_set(const void *ptr, prof_thr_cnt_t *cnt); 4296109fe07a14b7a619365977d9523db9f8b333792Jason Evans#endif 430e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 431e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm); 432e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); 433e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_STATS 434b34e8684ec025aa780997c11f847c19fb269755bJason Evansvoid arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, 435b34e8684ec025aa780997c11f847c19fb269755bJason Evans arena_stats_t *astats, malloc_bin_stats_t *bstats, 436b34e8684ec025aa780997c11f847c19fb269755bJason Evans malloc_large_stats_t *lstats); 437e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 438e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid *arena_ralloc(void *ptr, size_t size, size_t oldsize); 439e476f8a161d445211fd6e54fe370275196e66bcbJason Evansbool arena_new(arena_t *arena, unsigned ind); 440a0bf242230be117a3e54a7d1fc3f11e5a83606ecJason Evansbool arena_boot(void); 441e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 442e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_EXTERNS */ 443e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 444e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_H_INLINES 445e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 446e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifndef JEMALLOC_ENABLE_INLINE 447e476f8a161d445211fd6e54fe370275196e66bcbJason Evansvoid arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr); 448e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 449e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 450e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) 451e476f8a161d445211fd6e54fe370275196e66bcbJason EvansJEMALLOC_INLINE void 452e476f8a161d445211fd6e54fe370275196e66bcbJason Evansarena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr) 453e476f8a161d445211fd6e54fe370275196e66bcbJason Evans{ 454e476f8a161d445211fd6e54fe370275196e66bcbJason Evans size_t pageind; 455e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_chunk_map_t *mapelm; 456e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 457e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(arena != NULL); 458e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(arena->magic == ARENA_MAGIC); 459e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(chunk->arena == arena); 460e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(ptr != NULL); 461e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert(CHUNK_ADDR2BASE(ptr) != ptr); 462e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 463e476f8a161d445211fd6e54fe370275196e66bcbJason Evans pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT); 464e476f8a161d445211fd6e54fe370275196e66bcbJason Evans mapelm = &chunk->map[pageind]; 465e476f8a161d445211fd6e54fe370275196e66bcbJason Evans assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0); 466e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) { 467e476f8a161d445211fd6e54fe370275196e66bcbJason Evans /* Small allocation. */ 468e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 469e476f8a161d445211fd6e54fe370275196e66bcbJason Evans tcache_t *tcache; 470e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 471e476f8a161d445211fd6e54fe370275196e66bcbJason Evans if ((tcache = tcache_get()) != NULL) 472dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_dalloc_small(tcache, ptr); 473e476f8a161d445211fd6e54fe370275196e66bcbJason Evans else { 474e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 47586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_run_t *run; 47686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans arena_bin_t *bin; 47786815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans 47886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans run = (arena_run_t *)((uintptr_t)chunk + 47919b3d618924b3542a264612f906bc53bbcec8b70Jason Evans (uintptr_t)((pageind - (mapelm->bits >> 48019b3d618924b3542a264612f906bc53bbcec8b70Jason Evans PAGE_SHIFT)) << PAGE_SHIFT)); 48186815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans assert(run->magic == ARENA_RUN_MAGIC); 482f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans assert(((uintptr_t)ptr - ((uintptr_t)run + 483f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans (uintptr_t)run->bin->reg0_offset)) % 484f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans run->bin->reg_size == 0); 48586815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans bin = run->bin; 48686815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_lock(&bin->lock); 487e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_bin(arena, chunk, ptr, mapelm); 48886815df9dc7d2418a21c87b3dc9747ab42dea73dJason Evans malloc_mutex_unlock(&bin->lock); 489e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#ifdef JEMALLOC_TCACHE 490e476f8a161d445211fd6e54fe370275196e66bcbJason Evans } 491e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 492f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans } else { 493dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#ifdef JEMALLOC_TCACHE 494dafde14e08ddfda747aabb2045b350848b601b2eJason Evans size_t size = mapelm->bits & ~PAGE_MASK; 495dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 496f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 497dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if (size <= tcache_maxclass) { 498dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_t *tcache; 499dafde14e08ddfda747aabb2045b350848b601b2eJason Evans 500dafde14e08ddfda747aabb2045b350848b601b2eJason Evans if ((tcache = tcache_get()) != NULL) 501dafde14e08ddfda747aabb2045b350848b601b2eJason Evans tcache_dalloc_large(tcache, ptr, size); 502dafde14e08ddfda747aabb2045b350848b601b2eJason Evans else { 503dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 504dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_dalloc_large(arena, chunk, ptr); 505dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 506dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 507dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } else { 508dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 509dafde14e08ddfda747aabb2045b350848b601b2eJason Evans arena_dalloc_large(arena, chunk, ptr); 510dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 511dafde14e08ddfda747aabb2045b350848b601b2eJason Evans } 512dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#else 513dafde14e08ddfda747aabb2045b350848b601b2eJason Evans assert(((uintptr_t)ptr & PAGE_MASK) == 0); 514dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_lock(&arena->lock); 515e476f8a161d445211fd6e54fe370275196e66bcbJason Evans arena_dalloc_large(arena, chunk, ptr); 516dafde14e08ddfda747aabb2045b350848b601b2eJason Evans malloc_mutex_unlock(&arena->lock); 517dafde14e08ddfda747aabb2045b350848b601b2eJason Evans#endif 518f00bb7f132e3b74cb36c34223217df0c4394ada4Jason Evans } 519e476f8a161d445211fd6e54fe370275196e66bcbJason Evans} 520e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif 521e476f8a161d445211fd6e54fe370275196e66bcbJason Evans 522e476f8a161d445211fd6e54fe370275196e66bcbJason Evans#endif /* JEMALLOC_H_INLINES */ 523e476f8a161d445211fd6e54fe370275196e66bcbJason Evans/******************************************************************************/ 524